From d6950c27af31908363c5c815e3b7eb4f9ff41643 Mon Sep 17 00:00:00 2001 From: sepro Date: Wed, 27 Aug 2025 15:34:44 +0200 Subject: [PATCH 001/175] [ie/skeb] Support wav files (#14147) Closes #14146 Authored by: seproDev --- yt_dlp/extractor/skeb.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/skeb.py b/yt_dlp/extractor/skeb.py index 70111d0944..33c90a79d8 100644 --- a/yt_dlp/extractor/skeb.py +++ b/yt_dlp/extractor/skeb.py @@ -51,6 +51,20 @@ class SkebIE(InfoExtractor): }, 'playlist_count': 2, 'expected_warnings': ['Skipping unsupported extension'], + }, { + 'url': 'https://skeb.jp/@Yossshy_Music/works/13', + 'info_dict': { + 'ext': 'wav', + 'id': '5566495', + 'title': '13-1', + 'description': 'md5:1026b8b9ae38c67c2d995970ec196550', + 'uploader': 'Yossshy', + 'uploader_id': 'Yossshy_Music', + 'duration': 336, + 'thumbnail': r're:https?://.+', + 'tags': 'count:59', + 'genres': ['music'], + }, }] def _call_api(self, uploader_id, work_id): @@ -87,7 +101,7 @@ class SkebIE(InfoExtractor): entries = [] for idx, preview in enumerate(traverse_obj(works, ('previews', lambda _, v: url_or_none(v['url']))), 1): ext = traverse_obj(preview, ('information', 'extension', {str})) - if ext not in ('mp3', 'mp4'): + if ext not in ('mp3', 'mp4', 'wav'): self.report_warning(f'Skipping unsupported extension "{ext}"') continue @@ -100,7 +114,7 @@ class SkebIE(InfoExtractor): 'url': preview['vtt_url'], }], } if url_or_none(preview.get('vtt_url')) else None, - 'vcodec': 'none' if ext == 'mp3' else None, + 'vcodec': 'none' if ext in ('mp3', 'wav') else None, **info, **traverse_obj(preview, { 'id': ('id', {str_or_none}), From fec30c56f0e97e573ace659104ff0d72c4cc9809 Mon Sep 17 00:00:00 2001 From: sepro Date: Wed, 27 Aug 2025 22:25:35 +0200 Subject: [PATCH 002/175] [ie/generic] Use https as fallback protocol (#14160) Authored by: seproDev --- yt_dlp/extractor/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index b3a27f31e8..b05d6ca590 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -772,8 +772,8 @@ class GenericIE(InfoExtractor): if default_search in ('auto', 'auto_warning', 'fixup_error'): if re.match(r'[^\s/]+\.[^\s/]+/', url): - self.report_warning('The url doesn\'t specify the protocol, trying with http') - return self.url_result('http://' + url) + self.report_warning('The url doesn\'t specify the protocol, trying with https') + return self.url_result('https://' + url) elif default_search != 'fixup_error': if default_search == 'auto_warning': if re.match(r'^(?:url|URL)$', url): From 1ddbd033f0fd65917526b1271cea66913ac8647f Mon Sep 17 00:00:00 2001 From: sepro Date: Wed, 27 Aug 2025 23:27:57 +0200 Subject: [PATCH 003/175] [ie/generic] Simplify invalid URL error message (#14167) Authored by: seproDev --- yt_dlp/extractor/generic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index b05d6ca590..d44e6d3c4b 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -786,9 +786,7 @@ class GenericIE(InfoExtractor): return self.url_result('ytsearch:' + url) if default_search in ('error', 'fixup_error'): - raise ExtractorError( - f'{url!r} is not a valid URL. ' - f'Set --default-search "ytsearch" (or run yt-dlp "ytsearch:{url}" ) to search YouTube', expected=True) + raise ExtractorError(f'{url!r} is not a valid URL', expected=True) else: if ':' not in default_search: default_search += ':' From 5c7ad68ff1643ad80d18cef8be9db8fcab05ee6c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 27 Aug 2025 17:31:51 -0500 Subject: [PATCH 004/175] [ie/youtube] Deprioritize `web_safari` m3u8 formats (#14168) Authored by: bashonly --- yt_dlp/extractor/youtube/_video.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 6a1476f759..afb1226cfa 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -3611,6 +3611,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if f.get('source_preference') is None: f['source_preference'] = -1 + # Deprioritize since its pre-merged m3u8 formats may have lower quality audio streams + if client_name == 'web_safari' and proto == 'hls' and live_status != 'is_live': + f['source_preference'] -= 1 + if missing_pot: f['format_note'] = join_nonempty(f.get('format_note'), 'MISSING POT', delim=' ') f['source_preference'] -= 20 From 8cd37b85d492edb56a4f7506ea05527b85a6b02b Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 27 Aug 2025 18:00:03 -0500 Subject: [PATCH 005/175] [ie/youtube] Use alternative `tv` user-agent when authenticated (#14169) Authored by: bashonly --- yt_dlp/extractor/youtube/_base.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/youtube/_base.py b/yt_dlp/extractor/youtube/_base.py index 4ee54cc11e..5a0f55c53e 100644 --- a/yt_dlp/extractor/youtube/_base.py +++ b/yt_dlp/extractor/youtube/_base.py @@ -310,6 +310,8 @@ INNERTUBE_CLIENTS = { }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, 'SUPPORTS_COOKIES': True, + # See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506 + 'AUTHENTICATED_USER_AGENT': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)', }, 'tv_simply': { 'INNERTUBE_CONTEXT': { @@ -368,6 +370,7 @@ def build_innertube_clients(): ytcfg.setdefault('REQUIRE_AUTH', False) ytcfg.setdefault('SUPPORTS_COOKIES', False) ytcfg.setdefault('PLAYER_PARAMS', None) + ytcfg.setdefault('AUTHENTICATED_USER_AGENT', None) ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en') _, base_client, variant = _split_innertube_client(client) @@ -655,7 +658,14 @@ class YoutubeBaseInfoExtractor(InfoExtractor): _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=' def _get_default_ytcfg(self, client='web'): - return copy.deepcopy(INNERTUBE_CLIENTS[client]) + ytcfg = copy.deepcopy(INNERTUBE_CLIENTS[client]) + + # Currently, only the tv client needs to use an alternative user-agent when logged-in + if ytcfg.get('AUTHENTICATED_USER_AGENT') and self.is_authenticated: + client_context = ytcfg.setdefault('INNERTUBE_CONTEXT', {}).setdefault('client', {}) + client_context['userAgent'] = ytcfg['AUTHENTICATED_USER_AGENT'] + + return ytcfg def _get_innertube_host(self, client='web'): return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST'] @@ -954,7 +964,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): ytcfg = self.extract_ytcfg(video_id, webpage) or {} # Workaround for https://github.com/yt-dlp/yt-dlp/issues/12563 - if client == 'tv': + # But it's not effective when logged-in + if client == 'tv' and not self.is_authenticated: config_info = traverse_obj(ytcfg, ( 'INNERTUBE_CONTEXT', 'client', 'configInfo', {dict})) or {} config_info.pop('appInstallData', None) From 487a90c8efe74644b14a1324374473960def41ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 23:56:39 +0000 Subject: [PATCH 006/175] Release 2025.08.27 Created by: bashonly :ci skip all --- Changelog.md | 13 +++++++++++++ yt_dlp/version.py | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index d4ac4a5a69..2ad6da30e1 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,19 @@ # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2025.08.27 + +#### Extractor changes +- **generic** + - [Simplify invalid URL error message](https://github.com/yt-dlp/yt-dlp/commit/1ddbd033f0fd65917526b1271cea66913ac8647f) ([#14167](https://github.com/yt-dlp/yt-dlp/issues/14167)) by [seproDev](https://github.com/seproDev) + - [Use https as fallback protocol](https://github.com/yt-dlp/yt-dlp/commit/fec30c56f0e97e573ace659104ff0d72c4cc9809) ([#14160](https://github.com/yt-dlp/yt-dlp/issues/14160)) by [seproDev](https://github.com/seproDev) +- **skeb**: [Support wav files](https://github.com/yt-dlp/yt-dlp/commit/d6950c27af31908363c5c815e3b7eb4f9ff41643) ([#14147](https://github.com/yt-dlp/yt-dlp/issues/14147)) by [seproDev](https://github.com/seproDev) +- **youtube** + - [Add `tcc` player JS variant](https://github.com/yt-dlp/yt-dlp/commit/8f4a908300f55054bc96814bceeaa1034fdf4110) ([#14134](https://github.com/yt-dlp/yt-dlp/issues/14134)) by [bashonly](https://github.com/bashonly) + - [Deprioritize `web_safari` m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/5c7ad68ff1643ad80d18cef8be9db8fcab05ee6c) ([#14168](https://github.com/yt-dlp/yt-dlp/issues/14168)) by [bashonly](https://github.com/bashonly) + - [Player client maintenance](https://github.com/yt-dlp/yt-dlp/commit/3bd91544122142a87863d79e54e995c26cfd7f92) ([#14135](https://github.com/yt-dlp/yt-dlp/issues/14135)) by [bashonly](https://github.com/bashonly) + - [Use alternative `tv` user-agent when authenticated](https://github.com/yt-dlp/yt-dlp/commit/8cd37b85d492edb56a4f7506ea05527b85a6b02b) ([#14169](https://github.com/yt-dlp/yt-dlp/issues/14169)) by [bashonly](https://github.com/bashonly) + ### 2025.08.22 #### Core changes diff --git a/yt_dlp/version.py b/yt_dlp/version.py index fa2a637c06..cde18db454 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,8 +1,8 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2025.08.22' +__version__ = '2025.08.27' -RELEASE_GIT_HEAD = '5c8bcfdbc638dfde13e93157637d8521413ed774' +RELEASE_GIT_HEAD = '8cd37b85d492edb56a4f7506ea05527b85a6b02b' VARIANT = None @@ -12,4 +12,4 @@ CHANNEL = 'stable' ORIGIN = 'yt-dlp/yt-dlp' -_pkg_version = '2025.08.22' +_pkg_version = '2025.08.27' From 18fe696df9d60804a8f5cb8cd74f38111d6eb711 Mon Sep 17 00:00:00 2001 From: Gegham Zakaryan Date: Thu, 28 Aug 2025 05:12:08 +0400 Subject: [PATCH 007/175] [ie/googledrive] Fix subtitles extraction (#14139) Authored by: zakaryan2004 --- yt_dlp/extractor/googledrive.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py index dfba2d3ba1..0c84f0b241 100644 --- a/yt_dlp/extractor/googledrive.py +++ b/yt_dlp/extractor/googledrive.py @@ -12,6 +12,7 @@ from ..utils import ( get_element_html_by_id, int_or_none, lowercase_escape, + parse_qs, try_get, update_url_query, ) @@ -111,14 +112,18 @@ class GoogleDriveIE(InfoExtractor): self._caption_formats_ext.append(f.attrib['fmt_code']) def _get_captions_by_type(self, video_id, subtitles_id, caption_type, - origin_lang_code=None): + origin_lang_code=None, origin_lang_name=None): if not subtitles_id or not caption_type: return captions = {} for caption_entry in self._captions_xml.findall( self._CAPTIONS_ENTRY_TAG[caption_type]): caption_lang_code = caption_entry.attrib.get('lang_code') - if not caption_lang_code: + caption_name = caption_entry.attrib.get('name') or origin_lang_name + if not caption_lang_code or not caption_name: + self.report_warning(f'Missing necessary caption metadata. ' + f'Need lang_code and name attributes. ' + f'Found: {caption_entry.attrib}') continue caption_format_data = [] for caption_format in self._caption_formats_ext: @@ -129,7 +134,7 @@ class GoogleDriveIE(InfoExtractor): 'lang': (caption_lang_code if origin_lang_code is None else origin_lang_code), 'type': 'track', - 'name': '', + 'name': caption_name, 'kind': '', } if origin_lang_code is not None: @@ -155,14 +160,15 @@ class GoogleDriveIE(InfoExtractor): self._download_subtitles_xml(video_id, subtitles_id, hl) if not self._captions_xml: return - track = self._captions_xml.find('track') + track = next((t for t in self._captions_xml.findall('track') if t.attrib.get('cantran') == 'true'), None) if track is None: return origin_lang_code = track.attrib.get('lang_code') - if not origin_lang_code: + origin_lang_name = track.attrib.get('name') + if not origin_lang_code or not origin_lang_name: return return self._get_captions_by_type( - video_id, subtitles_id, 'automatic_captions', origin_lang_code) + video_id, subtitles_id, 'automatic_captions', origin_lang_code, origin_lang_name) def _real_extract(self, url): video_id = self._match_id(url) @@ -268,10 +274,8 @@ class GoogleDriveIE(InfoExtractor): subtitles_id = None ttsurl = get_value('ttsurl') if ttsurl: - # the video Id for subtitles will be the last value in the ttsurl - # query string - subtitles_id = ttsurl.encode().decode( - 'unicode_escape').split('=')[-1] + # the subtitles ID is the vid param of the ttsurl query + subtitles_id = parse_qs(ttsurl).get('vid', [None])[-1] self.cookiejar.clear(domain='.google.com', path='/', name='NID') From 223baa81f6637dcdef108f817180d8d1ae9fa213 Mon Sep 17 00:00:00 2001 From: Abdulmohsen <1621552+arabcoders@users.noreply.github.com> Date: Thu, 28 Aug 2025 04:18:10 +0300 Subject: [PATCH 008/175] [ie/tver] Extract more metadata (#14165) Authored by: arabcoders --- yt_dlp/extractor/tver.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yt_dlp/extractor/tver.py b/yt_dlp/extractor/tver.py index a3dbabfd1e..ffcc6a76b7 100644 --- a/yt_dlp/extractor/tver.py +++ b/yt_dlp/extractor/tver.py @@ -45,6 +45,8 @@ class TVerIE(StreaksBaseIE): 'release_timestamp': 1651453200, 'release_date': '20220502', '_old_archive_ids': ['brightcovenew ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068'], + 'series_id': 'sru35hwdd2', + 'season_id': 'ss2lcn4af6', }, }, { # via Brightcove backend (deprecated) @@ -67,6 +69,8 @@ class TVerIE(StreaksBaseIE): 'upload_date': '20220501', 'release_timestamp': 1651453200, 'release_date': '20220502', + 'series_id': 'sru35hwdd2', + 'season_id': 'ss2lcn4af6', }, 'params': {'extractor_args': {'tver': {'backend': ['brightcove']}}}, }, { @@ -202,6 +206,8 @@ class TVerIE(StreaksBaseIE): 'description': ('description', {str}), 'release_timestamp': ('viewStatus', 'startAt', {int_or_none}), 'episode_number': ('no', {int_or_none}), + 'series_id': ('seriesID', {str}), + 'season_id': ('seasonID', {str}), }), } From 0b51005b4819e7cea222fcbaf8e60391db4f732c Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 28 Aug 2025 02:19:25 +0100 Subject: [PATCH 009/175] [ie/ITVBTCC] Fix extractor (#14161) Closes #14156 Authored by: garret1317 --- yt_dlp/extractor/itv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/itv.py b/yt_dlp/extractor/itv.py index 89e6f189cb..1f4020847c 100644 --- a/yt_dlp/extractor/itv.py +++ b/yt_dlp/extractor/itv.py @@ -18,6 +18,7 @@ from ..utils import ( url_or_none, urljoin, ) +from ..utils.traversal import traverse_obj class ITVIE(InfoExtractor): @@ -223,6 +224,7 @@ class ITVBTCCIE(InfoExtractor): }, 'playlist_count': 12, }, { + # news page, can have absent `data` field 'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike', 'info_dict': { 'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike', @@ -243,7 +245,7 @@ class ITVBTCCIE(InfoExtractor): entries = [] for video in json_map: - if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')): + if not any(traverse_obj(video, ('data', attr)) == 'Brightcove' for attr in ('name', 'type')): continue video_id = video['data']['id'] account_id = video['data']['accountId'] From 1e28f6bf743627b909135bb9a88537ad2deccaf0 Mon Sep 17 00:00:00 2001 From: InvalidUsernameException Date: Thu, 28 Aug 2025 03:26:49 +0200 Subject: [PATCH 010/175] [ie/kick:vod] Support ongoing livestream VODs (#14154) Authored by: InvalidUsernameException --- yt_dlp/extractor/kick.py | 46 +++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py index 8049e1e342..06bd8bf591 100644 --- a/yt_dlp/extractor/kick.py +++ b/yt_dlp/extractor/kick.py @@ -95,26 +95,47 @@ class KickVODIE(KickBaseIE): IE_NAME = 'kick:vod' _VALID_URL = r'https?://(?:www\.)?kick\.com/[\w-]+/videos/(?P[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' _TESTS = [{ - 'url': 'https://kick.com/xqc/videos/8dd97a8d-e17f-48fb-8bc3-565f88dbc9ea', - 'md5': '3870f94153e40e7121a6e46c068b70cb', + # Regular VOD + 'url': 'https://kick.com/xqc/videos/5c697a87-afce-4256-b01f-3c8fe71ef5cb', 'info_dict': { - 'id': '8dd97a8d-e17f-48fb-8bc3-565f88dbc9ea', + 'id': '5c697a87-afce-4256-b01f-3c8fe71ef5cb', 'ext': 'mp4', - 'title': '18+ #ad 🛑LIVE🛑CLICK🛑DRAMA🛑NEWS🛑STUFF🛑REACT🛑GET IN HHERE🛑BOP BOP🛑WEEEE WOOOO🛑', + 'title': '🐗LIVE🐗CLICK🐗HERE🐗DRAMA🐗ALL DAY🐗NEWS🐗VIDEOS🐗CLIPS🐗GAMES🐗STUFF🐗WOW🐗IM HERE🐗LETS GO🐗COOL🐗VERY NICE🐗', 'description': 'THE BEST AT ABSOLUTELY EVERYTHING. THE JUICER. LEADER OF THE JUICERS.', - 'channel': 'xqc', - 'channel_id': '668', 'uploader': 'xQc', 'uploader_id': '676', - 'upload_date': '20240909', - 'timestamp': 1725919141, - 'duration': 10155.0, - 'thumbnail': r're:^https?://.*\.jpg', + 'channel': 'xqc', + 'channel_id': '668', 'view_count': int, - 'categories': ['Just Chatting'], - 'age_limit': 0, + 'age_limit': 18, + 'duration': 22278.0, + 'thumbnail': r're:^https?://.*\.jpg', + 'categories': ['Deadlock'], + 'timestamp': 1756082443, + 'upload_date': '20250825', }, 'params': {'skip_download': 'm3u8'}, + }, { + # VOD of ongoing livestream (at the time of writing the test, ID rotates every two days) + 'url': 'https://kick.com/a-log-burner/videos/5230df84-ea38-46e1-be4f-f5949ae55641', + 'info_dict': { + 'id': '5230df84-ea38-46e1-be4f-f5949ae55641', + 'ext': 'mp4', + 'title': r're:😴 Cozy Fireplace ASMR 🔥 | Relax, Focus, Sleep 💤', + 'description': 'md5:080bc713eac0321a7b376a1b53816d1b', + 'uploader': 'A_Log_Burner', + 'uploader_id': '65114691', + 'channel': 'a-log-burner', + 'channel_id': '63967687', + 'view_count': int, + 'age_limit': 18, + 'thumbnail': r're:^https?://.*\.jpg', + 'categories': ['Other, Watch Party'], + 'timestamp': int, + 'upload_date': str, + 'live_status': 'is_live', + }, + 'skip': 'live', }] def _real_extract(self, url): @@ -137,6 +158,7 @@ class KickVODIE(KickBaseIE): 'categories': ('livestream', 'categories', ..., 'name', {str}), 'view_count': ('views', {int_or_none}), 'age_limit': ('livestream', 'is_mature', {bool}, {lambda x: 18 if x else 0}), + 'is_live': ('livestream', 'is_live', {bool}), }), } From 76bb46002c9a9655f2b1d29d4840e75e79037cfa Mon Sep 17 00:00:00 2001 From: sepro Date: Fri, 29 Aug 2025 22:06:53 +0200 Subject: [PATCH 011/175] Fix `--id` deprecation warning (#14190) Authored by: seproDev --- yt_dlp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 3277cbfa1a..25f8d93f8b 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -979,6 +979,7 @@ def parse_options(argv=None): 'geo_bypass': opts.geo_bypass, 'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_ip_block': opts.geo_bypass_ip_block, + 'useid': opts.useid or None, 'warn_when_outdated': opts.update_self is None, '_warnings': warnings, '_deprecation_warnings': deprecation_warnings, From ed24640943872c4cf30d7cc4601bec87b50ba03c Mon Sep 17 00:00:00 2001 From: sepro Date: Sat, 30 Aug 2025 00:28:44 +0200 Subject: [PATCH 012/175] [ie/lrt] Fix extractors (#14193) Closes #13501 Authored by: seproDev --- yt_dlp/extractor/lrt.py | 114 +++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 49 deletions(-) diff --git a/yt_dlp/extractor/lrt.py b/yt_dlp/extractor/lrt.py index 34c9ece2d1..e05600f8ba 100644 --- a/yt_dlp/extractor/lrt.py +++ b/yt_dlp/extractor/lrt.py @@ -1,22 +1,14 @@ from .common import InfoExtractor from ..utils import ( clean_html, - merge_dicts, - traverse_obj, unified_timestamp, url_or_none, urljoin, ) +from ..utils.traversal import traverse_obj -class LRTBaseIE(InfoExtractor): - def _extract_js_var(self, webpage, var_name, default=None): - return self._search_regex( - fr'{var_name}\s*=\s*(["\'])((?:(?!\1).)+)\1', - webpage, var_name.replace('_', ' '), default, group=2) - - -class LRTStreamIE(LRTBaseIE): +class LRTStreamIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/tiesiogiai/(?P[\w-]+)' _TESTS = [{ 'url': 'https://www.lrt.lt/mediateka/tiesiogiai/lrt-opus', @@ -31,86 +23,110 @@ class LRTStreamIE(LRTBaseIE): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - streams_data = self._download_json(self._extract_js_var(webpage, 'tokenURL'), video_id) + + # TODO: Use _search_nextjs_v13_data once fixed + get_stream_url = self._search_regex( + r'\\"get_streams_url\\":\\"([^"]+)\\"', webpage, 'stream URL') + streams_data = self._download_json(get_stream_url, video_id) formats, subtitles = [], {} for stream_url in traverse_obj(streams_data, ( - 'response', 'data', lambda k, _: k.startswith('content')), expected_type=url_or_none): - fmts, subs = self._extract_m3u8_formats_and_subtitles(stream_url, video_id, 'mp4', m3u8_id='hls', live=True) + 'response', 'data', lambda k, _: k.startswith('content'), {url_or_none})): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + stream_url, video_id, 'mp4', m3u8_id='hls', live=True) formats.extend(fmts) subtitles = self._merge_subtitles(subtitles, subs) - stream_title = self._extract_js_var(webpage, 'video_title', 'LRT') return { 'id': video_id, 'formats': formats, 'subtitles': subtitles, 'is_live': True, - 'title': f'{self._og_search_title(webpage)} - {stream_title}', + 'title': self._og_search_title(webpage), } -class LRTVODIE(LRTBaseIE): - _VALID_URL = r'https?://(?:www\.)?lrt\.lt(?P/mediateka/irasas/(?P[0-9]+))' +class LRTVODIE(InfoExtractor): + _VALID_URL = [ + r'https?://(?:(?:www|archyvai)\.)?lrt\.lt/mediateka/irasas/(?P[0-9]+)', + r'https?://(?:(?:www|archyvai)\.)?lrt\.lt/mediateka/video/[^?#]+\?(?:[^#]*&)?episode=(?P[0-9]+)', + ] _TESTS = [{ # m3u8 download 'url': 'https://www.lrt.lt/mediateka/irasas/2000127261/greita-ir-gardu-sicilijos-ikvepta-klasikiniu-makaronu-su-baklazanais-vakariene', 'info_dict': { 'id': '2000127261', 'ext': 'mp4', - 'title': 'Greita ir gardu: Sicilijos įkvėpta klasikinių makaronų su baklažanais vakarienė', + 'title': 'Nustebinkite svečius klasikiniu makaronų su baklažanais receptu', 'description': 'md5:ad7d985f51b0dc1489ba2d76d7ed47fa', - 'duration': 3035, - 'timestamp': 1604079000, + 'timestamp': 1604086200, 'upload_date': '20201030', 'tags': ['LRT TELEVIZIJA', 'Beatos virtuvė', 'Beata Nicholson', 'Makaronai', 'Baklažanai', 'Vakarienė', 'Receptas'], 'thumbnail': 'https://www.lrt.lt/img/2020/10/30/764041-126478-1287x836.jpg', + 'channel': 'Beatos virtuvė', }, }, { - # direct mp3 download - 'url': 'http://www.lrt.lt/mediateka/irasas/1013074524/', - 'md5': '389da8ca3cad0f51d12bed0c844f6a0a', + # audio download + 'url': 'https://www.lrt.lt/mediateka/irasas/1013074524/kita-tema', + 'md5': 'fc982f10274929c66fdff65f75615cb0', 'info_dict': { 'id': '1013074524', - 'ext': 'mp3', - 'title': 'Kita tema 2016-09-05 15:05', + 'ext': 'mp4', + 'title': 'Kita tema', 'description': 'md5:1b295a8fc7219ed0d543fc228c931fb5', - 'duration': 3008, - 'view_count': int, - 'like_count': int, + 'channel': 'Kita tema', + 'timestamp': 1473087900, + 'upload_date': '20160905', }, + }, { + 'url': 'https://www.lrt.lt/mediateka/video/auksinis-protas-vasara?episode=2000420320&season=%2Fmediateka%2Fvideo%2Fauksinis-protas-vasara%2F2025', + 'info_dict': { + 'id': '2000420320', + 'ext': 'mp4', + 'title': 'Kuris senovės romėnų poetas aprašė Narcizo mitą?', + 'description': 'Intelektinė viktorina. Ved. Arūnas Valinskas ir Andrius Tapinas.', + 'channel': 'Auksinis protas. Vasara', + 'thumbnail': 'https://www.lrt.lt/img/2025/06/09/2094343-987905-1287x836.jpg', + 'tags': ['LRT TELEVIZIJA', 'Auksinis protas'], + 'timestamp': 1749851040, + 'upload_date': '20250613', + }, + }, { + 'url': 'https://archyvai.lrt.lt/mediateka/video/ziniu-riteriai-ir-damos?episode=49685&season=%2Fmediateka%2Fvideo%2Fziniu-riteriai-ir-damos%2F2013', + 'only_matching': True, + }, { + 'url': 'https://archyvai.lrt.lt/mediateka/irasas/2000077058/panorama-1989-baltijos-kelias', + 'only_matching': True, }] def _real_extract(self, url): - path, video_id = self._match_valid_url(url).group('path', 'id') + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - media_url = self._extract_js_var(webpage, 'main_url', path) - media = self._download_json(self._extract_js_var( - webpage, 'media_info_url', - 'https://www.lrt.lt/servisai/stream_url/vod/media_info/'), - video_id, query={'url': media_url}) + # TODO: Use _search_nextjs_v13_data once fixed + canonical_url = ( + self._search_regex(r'\\"(?:article|data)\\":{[^}]*\\"url\\":\\"(/[^"]+)\\"', webpage, 'content URL', fatal=False) + or self._search_regex(r'\d+)/(?P[^?#/]+)' _TESTS = [{ # m3u8 download From d925e92b710153d0d51d030f115b3c87226bc0f0 Mon Sep 17 00:00:00 2001 From: sepro Date: Sun, 31 Aug 2025 00:41:52 +0200 Subject: [PATCH 013/175] [ie/vevo] Restore extractors (#14203) Partially reverts 6f4c1bb593da92f0ce68229d0c813cdbaf1314da Authored by: seproDev --- yt_dlp/extractor/_extractors.py | 4 + yt_dlp/extractor/disney.py | 4 + yt_dlp/extractor/myspace.py | 6 +- yt_dlp/extractor/vevo.py | 352 ++++++++++++++++++++++++++++++++ 4 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 yt_dlp/extractor/vevo.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 25bad4f0dc..cb41381275 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2288,6 +2288,10 @@ from .varzesh3 import Varzesh3IE from .vbox7 import Vbox7IE from .veo import VeoIE from .vesti import VestiIE +from .vevo import ( + VevoIE, + VevoPlaylistIE, +) from .vgtv import ( VGTVIE, BTArticleIE, diff --git a/yt_dlp/extractor/disney.py b/yt_dlp/extractor/disney.py index d5cf6bbb38..a90f12389e 100644 --- a/yt_dlp/extractor/disney.py +++ b/yt_dlp/extractor/disney.py @@ -90,6 +90,10 @@ class DisneyIE(InfoExtractor): webpage, 'embed data'), video_id) video_data = page_data['video'] + for external in video_data.get('externals', []): + if external.get('source') == 'vevo': + return self.url_result('vevo:' + external['data_id'], 'Vevo') + video_id = video_data['id'] title = video_data['title'] diff --git a/yt_dlp/extractor/myspace.py b/yt_dlp/extractor/myspace.py index 701c2f447f..fa2ef14e13 100644 --- a/yt_dlp/extractor/myspace.py +++ b/yt_dlp/extractor/myspace.py @@ -111,8 +111,12 @@ class MySpaceIE(InfoExtractor): search_data('stream-url'), search_data('hls-stream-url'), search_data('http-stream-url')) if not formats: + vevo_id = search_data('vevo-id') youtube_id = search_data('youtube-id') - if youtube_id: + if vevo_id: + self.to_screen(f'Vevo video detected: {vevo_id}') + return self.url_result(f'vevo:{vevo_id}', ie='Vevo') + elif youtube_id: self.to_screen(f'Youtube video detected: {youtube_id}') return self.url_result(youtube_id, ie='Youtube') else: diff --git a/yt_dlp/extractor/vevo.py b/yt_dlp/extractor/vevo.py new file mode 100644 index 0000000000..8552a609c9 --- /dev/null +++ b/yt_dlp/extractor/vevo.py @@ -0,0 +1,352 @@ +import json +import re + +from .common import InfoExtractor +from ..networking.exceptions import HTTPError +from ..utils import ( + ExtractorError, + int_or_none, + parse_iso8601, + parse_qs, +) + + +class VevoBaseIE(InfoExtractor): + def _extract_json(self, webpage, video_id): + return self._parse_json( + self._search_regex( + r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*', + webpage, 'initial store'), + video_id) + + +class VevoIE(VevoBaseIE): + """ + Accepts urls from vevo.com or in the format 'vevo:{id}' + (currently used by MTVIE and MySpaceIE) + """ + _VALID_URL = r'''(?x) + (?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?| + https?://cache\.vevo\.com/m/html/embed\.html\?video=| + https?://videoplayer\.vevo\.com/embed/embedded\?videoId=| + https?://embed\.vevo\.com/.*?[?&]isrc=| + https?://tv\.vevo\.com/watch/artist/(?:[^/]+)/| + vevo:) + (?P[^&?#]+)''' + _EMBED_REGEX = [r']+?src=(["\'])(?P(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1'] + + _TESTS = [{ + 'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280', + 'md5': '95ee28ee45e70130e3ab02b0f579ae23', + 'info_dict': { + 'id': 'GB1101300280', + 'ext': 'mp4', + 'title': 'Hurts - Somebody to Die For', + 'timestamp': 1372057200, + 'upload_date': '20130624', + 'uploader': 'Hurts', + 'track': 'Somebody to Die For', + 'artist': 'Hurts', + 'genre': 'Pop', + }, + 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'], + }, { + 'note': 'v3 SMIL format', + 'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923', + 'md5': 'f6ab09b034f8c22969020b042e5ac7fc', + 'info_dict': { + 'id': 'USUV71302923', + 'ext': 'mp4', + 'title': 'Cassadee Pope - I Wish I Could Break Your Heart', + 'timestamp': 1392796919, + 'upload_date': '20140219', + 'uploader': 'Cassadee Pope', + 'track': 'I Wish I Could Break Your Heart', + 'artist': 'Cassadee Pope', + 'genre': 'Country', + }, + 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'], + }, { + 'note': 'Age-limited video', + 'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282', + 'info_dict': { + 'id': 'USRV81300282', + 'ext': 'mp4', + 'title': 'Justin Timberlake - Tunnel Vision (Explicit)', + 'age_limit': 18, + 'timestamp': 1372888800, + 'upload_date': '20130703', + 'uploader': 'Justin Timberlake', + 'track': 'Tunnel Vision (Explicit)', + 'artist': 'Justin Timberlake', + 'genre': 'Pop', + }, + 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'], + }, { + 'note': 'No video_info', + 'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000', + 'md5': '8b83cc492d72fc9cf74a02acee7dc1b0', + 'info_dict': { + 'id': 'USUV71503000', + 'ext': 'mp4', + 'title': 'K Camp ft. T.I. - Till I Die', + 'age_limit': 18, + 'timestamp': 1449468000, + 'upload_date': '20151207', + 'uploader': 'K Camp', + 'track': 'Till I Die', + 'artist': 'K Camp', + 'genre': 'Hip-Hop', + }, + 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'], + }, { + 'note': 'Featured test', + 'url': 'https://www.vevo.com/watch/lemaitre/Wait/USUV71402190', + 'md5': 'd28675e5e8805035d949dc5cf161071d', + 'info_dict': { + 'id': 'USUV71402190', + 'ext': 'mp4', + 'title': 'Lemaitre ft. LoLo - Wait', + 'age_limit': 0, + 'timestamp': 1413432000, + 'upload_date': '20141016', + 'uploader': 'Lemaitre', + 'track': 'Wait', + 'artist': 'Lemaitre', + 'genre': 'Electronic', + }, + 'expected_warnings': ['Unable to download SMIL file', 'Unable to download info'], + }, { + 'note': 'Only available via webpage', + 'url': 'http://www.vevo.com/watch/GBUV71600656', + 'md5': '67e79210613865b66a47c33baa5e37fe', + 'info_dict': { + 'id': 'GBUV71600656', + 'ext': 'mp4', + 'title': 'ABC - Viva Love', + 'age_limit': 0, + 'timestamp': 1461830400, + 'upload_date': '20160428', + 'uploader': 'ABC', + 'track': 'Viva Love', + 'artist': 'ABC', + 'genre': 'Pop', + }, + 'expected_warnings': ['Failed to download video versions info'], + }, { + # no genres available + 'url': 'http://www.vevo.com/watch/INS171400764', + 'only_matching': True, + }, { + # Another case available only via the webpage; using streams/streamsV3 formats + # Geo-restricted to Netherlands/Germany + 'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909', + 'only_matching': True, + }, { + 'url': 'https://embed.vevo.com/?isrc=USH5V1923499&partnerId=4d61b777-8023-4191-9ede-497ed6c24647&partnerAdCode=', + 'only_matching': True, + }, { + 'url': 'https://tv.vevo.com/watch/artist/janet-jackson/US0450100550', + 'only_matching': True, + }] + _VERSIONS = { + 0: 'youtube', # only in AuthenticateVideo videoVersions + 1: 'level3', + 2: 'akamai', + 3: 'level3', + 4: 'amazon', + } + + def _initialize_api(self, video_id): + webpage = self._download_webpage( + 'https://accounts.vevo.com/token', None, + note='Retrieving oauth token', + errnote='Unable to retrieve oauth token', + data=json.dumps({ + 'client_id': 'SPupX1tvqFEopQ1YS6SS', + 'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous', + }).encode(), + headers={ + 'Content-Type': 'application/json', + }) + + if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage): + self.raise_geo_restricted( + f'{self.IE_NAME} said: This page is currently unavailable in your region') + + auth_info = self._parse_json(webpage, video_id) + self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token'] + + def _call_api(self, path, *args, **kwargs): + try: + data = self._download_json(self._api_url_template % path, *args, **kwargs) + except ExtractorError as e: + if isinstance(e.cause, HTTPError): + errors = self._parse_json(e.cause.response.read().decode(), None)['errors'] + error_message = ', '.join([error['message'] for error in errors]) + raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True) + raise + return data + + def _real_extract(self, url): + video_id = self._match_id(url) + + self._initialize_api(video_id) + + video_info = self._call_api( + f'video/{video_id}', video_id, 'Downloading api video info', + 'Failed to download video info') + + video_versions = self._call_api( + f'video/{video_id}/streams', video_id, + 'Downloading video versions info', + 'Failed to download video versions info', + fatal=False) + + # Some videos are only available via webpage (e.g. + # https://github.com/ytdl-org/youtube-dl/issues/9366) + if not video_versions: + webpage = self._download_webpage(url, video_id) + json_data = self._extract_json(webpage, video_id) + if 'streams' in json_data.get('default', {}): + video_versions = json_data['default']['streams'][video_id][0] + else: + video_versions = [ + value + for key, value in json_data['apollo']['data'].items() + if key.startswith(f'{video_id}.streams')] + + uploader = None + artist = None + featured_artist = None + artists = video_info.get('artists') + for curr_artist in artists: + if curr_artist.get('role') == 'Featured': + featured_artist = curr_artist['name'] + else: + artist = uploader = curr_artist['name'] + + formats = [] + for video_version in video_versions: + version = self._VERSIONS.get(video_version.get('version'), 'generic') + version_url = video_version.get('url') + if not version_url: + continue + + if '.ism' in version_url: + continue + elif '.mpd' in version_url: + formats.extend(self._extract_mpd_formats( + version_url, video_id, mpd_id=f'dash-{version}', + note=f'Downloading {version} MPD information', + errnote=f'Failed to download {version} MPD information', + fatal=False)) + elif '.m3u8' in version_url: + formats.extend(self._extract_m3u8_formats( + version_url, video_id, 'mp4', 'm3u8_native', + m3u8_id=f'hls-{version}', + note=f'Downloading {version} m3u8 information', + errnote=f'Failed to download {version} m3u8 information', + fatal=False)) + else: + m = re.search(r'''(?xi) + _(?P[a-z0-9]+) + _(?P[0-9]+)x(?P[0-9]+) + _(?P[a-z0-9]+) + _(?P[0-9]+) + _(?P[a-z0-9]+) + _(?P[0-9]+) + \.(?P[a-z0-9]+)''', version_url) + if not m: + continue + + formats.append({ + 'url': version_url, + 'format_id': f'http-{version}-{video_version.get("quality") or m.group("quality")}', + 'vcodec': m.group('vcodec'), + 'acodec': m.group('acodec'), + 'vbr': int(m.group('vbr')), + 'abr': int(m.group('abr')), + 'ext': m.group('ext'), + 'width': int(m.group('width')), + 'height': int(m.group('height')), + }) + + track = video_info['title'] + if featured_artist: + artist = f'{artist} ft. {featured_artist}' + title = f'{artist} - {track}' if artist else track + + genres = video_info.get('genres') + genre = ( + genres[0] if genres and isinstance(genres, list) + and isinstance(genres[0], str) else None) + + is_explicit = video_info.get('isExplicit') + if is_explicit is True: + age_limit = 18 + elif is_explicit is False: + age_limit = 0 + else: + age_limit = None + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'), + 'timestamp': parse_iso8601(video_info.get('releaseDate')), + 'uploader': uploader, + 'duration': int_or_none(video_info.get('duration')), + 'view_count': int_or_none(video_info.get('views', {}).get('total')), + 'age_limit': age_limit, + 'track': track, + 'artist': uploader, + 'genre': genre, + } + + +class VevoPlaylistIE(VevoBaseIE): + _VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?Pplaylist|genre)/(?P[^/?#&]+)' + + _TESTS = [{ + 'url': 'http://www.vevo.com/watch/genre/rock', + 'info_dict': { + 'id': 'rock', + 'title': 'Rock', + }, + 'playlist_count': 20, + }, { + 'url': 'http://www.vevo.com/watch/genre/rock?index=0', + 'only_matching': True, + }] + + def _real_extract(self, url): + mobj = self._match_valid_url(url) + playlist_id = mobj.group('id') + playlist_kind = mobj.group('kind') + + webpage = self._download_webpage(url, playlist_id) + + qs = parse_qs(url) + index = qs.get('index', [None])[0] + + if index: + video_id = self._search_regex( + r']+content=(["\'])vevo://video/(?P.+?)\1[^>]*>', + webpage, 'video id', default=None, group='id') + if video_id: + return self.url_result(f'vevo:{video_id}', VevoIE.ie_key()) + + playlists = self._extract_json(webpage, playlist_id)['default'][f'{playlist_kind}s'] + + playlist = (next(iter(playlists.values())) + if playlist_kind == 'playlist' else playlists[playlist_id]) + + entries = [ + self.url_result(f'vevo:{src}', VevoIE.ie_key()) + for src in playlist['isrcs']] + + return self.playlist_result( + entries, playlist.get('playlistId') or playlist_id, + playlist.get('name'), playlist.get('description')) From 603acdff07f0226088916886002d2ad8309ff9d3 Mon Sep 17 00:00:00 2001 From: Chase Ryan Date: Fri, 5 Sep 2025 21:28:52 +0000 Subject: [PATCH 014/175] [ie/charlierose] Fix extractor (#14231) Authored by: gitchasing --- yt_dlp/extractor/charlierose.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/charlierose.py b/yt_dlp/extractor/charlierose.py index 8fe6797c6a..9fa71fea17 100644 --- a/yt_dlp/extractor/charlierose.py +++ b/yt_dlp/extractor/charlierose.py @@ -6,7 +6,7 @@ class CharlieRoseIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?charlierose\.com/(?:video|episode)(?:s|/player)/(?P\d+)' _TESTS = [{ 'url': 'https://charlierose.com/videos/27996', - 'md5': 'fda41d49e67d4ce7c2411fd2c4702e09', + 'md5': '4405b662f557f94aa256fa6a7baf7426', 'info_dict': { 'id': '27996', 'ext': 'mp4', @@ -39,12 +39,16 @@ class CharlieRoseIE(InfoExtractor): self._PLAYER_BASE % video_id, webpage, video_id, m3u8_entry_protocol='m3u8_native')[0] self._remove_duplicate_formats(info_dict['formats']) + for fmt in info_dict['formats']: + if fmt.get('protocol') == 'm3u8_native': + fmt['__needs_testing'] = True info_dict.update({ 'id': video_id, 'title': title, 'thumbnail': self._og_search_thumbnail(webpage), 'description': self._og_search_description(webpage), + '_format_sort_fields': ('proto',), }) return info_dict From 50136eeeb3767289b236f140b759f23b39b00888 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 5 Sep 2025 17:38:20 -0500 Subject: [PATCH 015/175] [build] Overhaul Linux builds and refactor release workflow (#13997) - Use `manylinux-shared` images for Linux builds - Discontinue `yt-dlp_linux_armv7l`/`linux_armv7l_exe` release binary - Add `yt-dlp_linux_armv7l.zip`/`linux_armv7l_dir` release binary - Add `yt-dlp_musllinux` and `yt-dlp_musllinux_aarch64` release binaries - Migrate `linux_exe` build strategy from staticx+musl to manylinux2014/glibc2.17 - Rewrite release.yml's "unholy bash monstrosity" as devscripts/setup_variables.py Closes #10072, Closes #10630, Closes #10578, Closes #13976, Closes #13977, Closes #14106 Authored by: bashonly --- .github/workflows/build.yml | 378 ++++++++++++++++---------- .github/workflows/cache-warmer.yml | 22 ++ .github/workflows/release-master.yml | 13 +- .github/workflows/release-nightly.yml | 13 +- .github/workflows/release.yml | 262 +++++++----------- README.md | 20 +- bundle/docker/compose.yml | 155 ++++++++++- bundle/docker/linux/Dockerfile | 16 ++ bundle/docker/linux/build.sh | 46 ++++ bundle/docker/linux/verify.sh | 44 +++ bundle/docker/static/Dockerfile | 21 -- bundle/docker/static/entrypoint.sh | 14 - bundle/pyinstaller.py | 2 + devscripts/setup_variables.py | 157 +++++++++++ devscripts/setup_variables_tests.py | 331 ++++++++++++++++++++++ devscripts/update-version.py | 21 +- devscripts/update_changelog.py | 6 +- devscripts/utils.py | 19 ++ test/test_update.py | 22 +- yt_dlp/update.py | 65 ++--- 20 files changed, 1203 insertions(+), 424 deletions(-) create mode 100644 .github/workflows/cache-warmer.yml create mode 100644 bundle/docker/linux/Dockerfile create mode 100755 bundle/docker/linux/build.sh create mode 100755 bundle/docker/linux/verify.sh delete mode 100644 bundle/docker/static/Dockerfile delete mode 100755 bundle/docker/static/entrypoint.sh create mode 100644 devscripts/setup_variables.py create mode 100644 devscripts/setup_variables_tests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a0220d702..510edb1e72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,10 +12,13 @@ on: unix: default: true type: boolean - linux_static: + linux: default: true type: boolean - linux_arm: + linux_armv7l: + default: true + type: boolean + musllinux: default: true type: boolean macos: @@ -37,7 +40,9 @@ on: version: description: | VERSION: yyyy.mm.dd[.rev] or rev - required: true + (default: auto-generated) + required: false + default: '' type: string channel: description: | @@ -49,12 +54,16 @@ on: description: yt-dlp, yt-dlp.tar.gz default: true type: boolean - linux_static: - description: yt-dlp_linux + linux: + description: yt-dlp_linux, yt-dlp_linux.zip, yt-dlp_linux_aarch64, yt-dlp_linux_aarch64.zip default: true type: boolean - linux_arm: - description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + linux_armv7l: + description: yt-dlp_linux_armv7l.zip + default: true + type: boolean + musllinux: + description: yt-dlp_musllinux, yt-dlp_musllinux.zip, yt-dlp_musllinux_aarch64, yt-dlp_musllinux_aarch64.zip default: true type: boolean macos: @@ -81,16 +90,51 @@ jobs: runs-on: ubuntu-latest outputs: origin: ${{ steps.process_origin.outputs.origin }} + timestamp: ${{ steps.process_origin.outputs.timestamp }} + version: ${{ steps.process_origin.outputs.version }} steps: - name: Process origin id: process_origin + env: + ORIGIN: ${{ inputs.origin }} + REPOSITORY: ${{ github.repository }} + VERSION: ${{ inputs.version }} + shell: python run: | - echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT" + import datetime as dt + import json + import os + import re + origin = os.environ['ORIGIN'] + timestamp = dt.datetime.now(tz=dt.timezone.utc).strftime('%Y.%m.%d.%H%M%S.%f') + version = os.getenv('VERSION') + if version and '.' not in version: + # build.yml was dispatched with only a revision as the version input value + version_parts = [*timestamp.split('.')[:3], version] + elif not version: + # build.yml was dispatched without any version input value, so include .HHMMSS revision + version_parts = timestamp.split('.')[:4] + else: + # build.yml was called or dispatched with a complete version input value + version_parts = version.split('.') + assert all(re.fullmatch(r'[0-9]+', part) for part in version_parts), 'Version must be numeric' + outputs = { + 'origin': os.environ['REPOSITORY'] if origin == 'current repo' else origin, + 'timestamp': timestamp, + 'version': '.'.join(version_parts), + } + print(json.dumps(outputs, indent=2)) + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write('\n'.join(f'{key}={value}' for key, value in outputs.items())) unix: needs: process if: inputs.unix runs-on: ubuntu-latest + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} steps: - uses: actions/checkout@v4 with: @@ -103,7 +147,7 @@ jobs: sudo apt -y install zip pandoc man sed - name: Prepare run: | - python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" python devscripts/update_changelog.py -vv python devscripts/make_lazy_extractors.py - name: Build Unix platform-independent binary @@ -117,7 +161,7 @@ jobs: version="$(./yt-dlp --version)" ./yt-dlp_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 downgraded_version="$(./yt-dlp_downgraded --version)" - [[ "$version" != "$downgraded_version" ]] + [[ "${version}" != "${downgraded_version}" ]] - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -127,99 +171,156 @@ jobs: yt-dlp.tar.gz compression-level: 0 - linux_static: + linux: needs: process - if: inputs.linux_static - runs-on: ubuntu-latest + if: inputs.linux + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - exe: yt-dlp_linux + platform: x86_64 + runner: ubuntu-24.04 + - exe: yt-dlp_linux_aarch64 + platform: aarch64 + runner: ubuntu-24.04-arm + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} + EXE_NAME: ${{ matrix.exe }} steps: - uses: actions/checkout@v4 - - name: Build static executable + - name: Build executable env: - channel: ${{ inputs.channel }} - origin: ${{ needs.process.outputs.origin }} - version: ${{ inputs.version }} + SERVICE: linux_${{ matrix.platform }} + run: | + mkdir -p ./dist + pushd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + popd + sudo chown "${USER}:docker" "./dist/${EXE_NAME}" + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + SERVICE: linux_${{ matrix.platform }}_verify run: | - mkdir ~/build cd bundle/docker - docker compose up --build static - sudo chown "${USER}:docker" ~/build/yt-dlp_linux + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - name: Verify --update-to if: vars.UPDATE_TO_VERIFICATION run: | - chmod +x ~/build/yt-dlp_linux - cp ~/build/yt-dlp_linux ~/build/yt-dlp_linux_downgraded - version="$(~/build/yt-dlp_linux --version)" - ~/build/yt-dlp_linux_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 - downgraded_version="$(~/build/yt-dlp_linux_downgraded --version)" - [[ "$version" != "$downgraded_version" ]] + chmod +x "./dist/${EXE_NAME}" + mkdir -p ~/testing + cp "./dist/${EXE_NAME}" ~/testing/"${EXE_NAME}_downgraded" + version="$("./dist/${EXE_NAME}" --version)" + ~/testing/"${EXE_NAME}_downgraded" -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(~/testing/"${EXE_NAME}_downgraded" --version)" + [[ "${version}" != "${downgraded_version}" ]] + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }}_${{ matrix.platform }} + path: | + dist/${{ matrix.exe }}* + compression-level: 0 + + linux_armv7l: + needs: process + if: inputs.linux_armv7l + permissions: + contents: read + runs-on: ubuntu-24.04-arm + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} + EXE_NAME: yt-dlp_linux_armv7l + steps: + - uses: actions/checkout@v4 + - name: Cache requirements + id: cache-venv + uses: actions/cache@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 + with: + path: | + ~/yt-dlp-build-venv + key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} + restore-keys: | + cache-reqs-${{ github.job }}-${{ github.ref }}- + cache-reqs-${{ github.job }}- + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: Build executable + env: + SERVICE: linux_armv7l + run: | + mkdir -p ./dist + mkdir -p ~/yt-dlp-build-venv + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + SERVICE: linux_armv7l_verify + run: | + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: build-bin-${{ github.job }} path: | - ~/build/yt-dlp_linux + dist/yt-dlp_linux_armv7l.zip compression-level: 0 - linux_arm: + musllinux: needs: process - if: inputs.linux_arm - permissions: - contents: read - packages: write # for creating cache - runs-on: ubuntu-latest + if: inputs.musllinux + runs-on: ${{ matrix.runner }} strategy: + fail-fast: false matrix: - architecture: - - armv7 - - aarch64 - + include: + - exe: yt-dlp_musllinux + platform: x86_64 + runner: ubuntu-24.04 + - exe: yt-dlp_musllinux_aarch64 + platform: aarch64 + runner: ubuntu-24.04-arm + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} + EXE_NAME: ${{ matrix.exe }} steps: - uses: actions/checkout@v4 - with: - path: ./repo - - name: Virtualized Install, Prepare & Build - uses: yt-dlp/run-on-arch-action@v3 - with: - # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 - env: | - GITHUB_WORKFLOW: build - githubToken: ${{ github.token }} # To cache image - arch: ${{ matrix.architecture }} - distro: ubuntu20.04 # Standalone executable should be built on minimum supported OS - dockerRunArgs: --volume "${PWD}/repo:/repo" - install: | # Installing Python 3.10 from the Deadsnakes repo raises errors - apt update - apt -y install zlib1g-dev libffi-dev python3.9 python3.9-dev python3.9-distutils python3-pip \ - python3-secretstorage # Cannot build cryptography wheel in virtual armv7 environment - python3.9 -m pip install -U pip wheel 'setuptools>=71.0.2' - # XXX: Keep this in sync with pyproject.toml (it can't be accessed at this stage) and exclude secretstorage - python3.9 -m pip install -U Pyinstaller mutagen pycryptodomex brotli certifi cffi \ - 'requests>=2.32.2,<3' 'urllib3>=2.0.2,<3' 'websockets>=13.0' - - run: | - cd repo - python3.9 devscripts/install_deps.py -o --include build - python3.9 devscripts/install_deps.py --include pyinstaller # Cached versions may be out of date - python3.9 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" - python3.9 devscripts/make_lazy_extractors.py - python3.9 -m bundle.pyinstaller - - if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then - arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}" - chmod +x ./dist/yt-dlp_linux_${arch} - cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded - version="$(./dist/yt-dlp_linux_${arch} --version)" - ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 - downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)" - [[ "$version" != "$downgraded_version" ]] - fi - + - name: Build executable + env: + SERVICE: musllinux_${{ matrix.platform }} + run: | + mkdir -p ./dist + pushd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + popd + sudo chown "${USER}:docker" "./dist/${EXE_NAME}" + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + SERVICE: musllinux_${{ matrix.platform }}_verify + run: | + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-bin-linux_${{ matrix.architecture }} - path: | # run-on-arch-action designates armv7l as armv7 - repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + name: build-bin-${{ github.job }}_${{ matrix.platform }} + path: | + dist/${{ matrix.exe }}* compression-level: 0 macos: @@ -227,22 +328,28 @@ jobs: if: inputs.macos permissions: contents: read - actions: write # For cleaning up cache runs-on: macos-14 + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} steps: - uses: actions/checkout@v4 # NB: Building universal2 does not work with python from actions/setup-python - - name: Restore cached requirements - id: restore-cache - uses: actions/cache/restore@v4 + - name: Cache requirements + id: cache-venv + uses: actions/cache@v4 env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 with: path: | ~/yt-dlp-build-venv - key: cache-reqs-${{ github.job }}-${{ github.ref }} + key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} + restore-keys: | + cache-reqs-${{ github.job }}-${{ github.ref }}- + cache-reqs-${{ github.job }}- - name: Install Requirements run: | @@ -287,7 +394,7 @@ jobs: - name: Prepare run: | - python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3 devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" python3 devscripts/make_lazy_extractors.py - name: Build run: | @@ -315,27 +422,11 @@ jobs: dist/yt-dlp_macos.zip compression-level: 0 - - name: Cleanup cache - if: steps.restore-cache.outputs.cache-hit == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - cache_key: cache-reqs-${{ github.job }}-${{ github.ref }} - run: | - gh cache delete "${cache_key}" - - - name: Cache requirements - uses: actions/cache/save@v4 - with: - path: | - ~/yt-dlp-build-venv - key: cache-reqs-${{ github.job }}-${{ github.ref }} - windows: needs: process if: inputs.windows permissions: contents: read - actions: write # For cleaning up cache runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -353,6 +444,14 @@ jobs: runner: windows-11-arm python_version: '3.13' # arm64 only has Python >= 3.11 available suffix: '_arm64' + env: + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ needs.process.outputs.version }} + SUFFIX: ${{ matrix.suffix }} + BASE_CACHE_KEY: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }} + # Use custom PyInstaller built with https://github.com/yt-dlp/Pyinstaller-builds + PYINSTALLER_URL: https://yt-dlp.github.io/Pyinstaller-Builds/${{ matrix.arch }}/pyinstaller-6.15.0-py3-none-any.whl steps: - uses: actions/checkout@v4 @@ -361,49 +460,60 @@ jobs: python-version: ${{ matrix.python_version }} architecture: ${{ matrix.arch }} - - name: Restore cached requirements - id: restore-cache + - name: Cache requirements + id: cache-venv if: matrix.arch == 'arm64' - uses: actions/cache/restore@v4 + uses: actions/cache@v4 env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 with: path: | /yt-dlp-build-venv - key: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }}-${{ github.ref }} + key: ${{ env.BASE_CACHE_KEY }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} + restore-keys: | + ${{ env.BASE_CACHE_KEY }}-${{ github.ref }}- + ${{ env.BASE_CACHE_KEY }}- - name: Install Requirements + env: + ARCH: ${{ matrix.arch }} + shell: pwsh run: | python -m venv /yt-dlp-build-venv /yt-dlp-build-venv/Scripts/Activate.ps1 python devscripts/install_deps.py -o --include build - python devscripts/install_deps.py ${{ (matrix.arch != 'x86' && '--include curl-cffi') || '' }} - # Use custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds - python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/${{ matrix.arch }}/pyinstaller-6.15.0-py3-none-any.whl" + if ("${Env:ARCH}" -eq "x86") { + python devscripts/install_deps.py + } else { + python devscripts/install_deps.py --include curl-cffi + } + python -m pip install -U "${Env:PYINSTALLER_URL}" - name: Prepare + shell: pwsh run: | - python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/update-version.py -c "${Env:CHANNEL}" -r "${Env:ORIGIN}" "${Env:VERSION}" python devscripts/make_lazy_extractors.py - name: Build + shell: pwsh run: | /yt-dlp-build-venv/Scripts/Activate.ps1 python -m bundle.pyinstaller python -m bundle.pyinstaller --onedir - Compress-Archive -Path ./dist/yt-dlp${{ matrix.suffix }}/* -DestinationPath ./dist/yt-dlp_win${{ matrix.suffix }}.zip + Compress-Archive -Path ./dist/yt-dlp${Env:SUFFIX}/* -DestinationPath ./dist/yt-dlp_win${Env:SUFFIX}.zip - name: Verify --update-to if: vars.UPDATE_TO_VERIFICATION + shell: pwsh run: | - foreach ($name in @("yt-dlp${{ matrix.suffix }}")) { - Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" - $version = & "./dist/${name}.exe" --version - & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2025.08.20 - $downgraded_version = & "./dist/${name}_downgraded.exe" --version - if ($version -eq $downgraded_version) { - exit 1 - } + $name = "yt-dlp${Env:SUFFIX}" + Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" + $version = & "./dist/${name}.exe" --version + & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2025.08.20 + $downgraded_version = & "./dist/${name}_downgraded.exe" --version + if ($version -eq $downgraded_version) { + exit 1 } - name: Upload artifacts @@ -415,30 +525,14 @@ jobs: dist/yt-dlp_win${{ matrix.suffix }}.zip compression-level: 0 - - name: Cleanup cache - if: | - matrix.arch == 'arm64' && steps.restore-cache.outputs.cache-hit == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - cache_key: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }}-${{ github.ref }} - run: | - gh cache delete "${cache_key}" - - - name: Cache requirements - if: matrix.arch == 'arm64' - uses: actions/cache/save@v4 - with: - path: | - /yt-dlp-build-venv - key: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }}-${{ github.ref }} - meta_files: if: always() && !cancelled() needs: - process - unix - - linux_static - - linux_arm + - linux + - linux_armv7l + - musllinux - macos - windows runs-on: ubuntu-latest @@ -469,38 +563,38 @@ jobs: lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7 lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) lock 2024.10.22 py2exe .+ - lock 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b lock 2024.10.22 zip Python 3\.8 lock 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lock 2025.08.11 darwin_legacy_exe .+ + lock 2025.08.27 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6 lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) lockV2 yt-dlp/yt-dlp 2024.10.22 py2exe .+ - lockV2 yt-dlp/yt-dlp 2024.10.22 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b lockV2 yt-dlp/yt-dlp 2024.10.22 zip Python 3\.8 lockV2 yt-dlp/yt-dlp 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp 2025.08.11 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp 2025.08.27 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server) lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 py2exe .+ - lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 zip Python 3\.8 lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp-nightly-builds 2025.08.12.233030 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp-nightly-builds 2025.08.30.232839 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server) lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.045052 py2exe .+ - lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 linux_(?:armv7l|aarch64)_exe .+-glibc2\.(?:[12]?\d|30)\b lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 zip Python 3\.8 lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp-master-builds 2025.08.12.232447 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp-master-builds 2025.09.05.212910 linux_armv7l_exe .+ EOF - name: Sign checksum files env: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} - if: env.GPG_SIGNING_KEY != '' + if: env.GPG_SIGNING_KEY run: | gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}" for signfile in ./SHA*SUMS; do diff --git a/.github/workflows/cache-warmer.yml b/.github/workflows/cache-warmer.yml new file mode 100644 index 0000000000..0b2daa8897 --- /dev/null +++ b/.github/workflows/cache-warmer.yml @@ -0,0 +1,22 @@ +name: Keep cache warm +on: + workflow_dispatch: + schedule: + - cron: '0 22 1,6,11,16,21,27 * *' + +jobs: + build: + if: | + vars.KEEP_CACHE_WARM || github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/build.yml + with: + version: '999999' + channel: stable + unix: false + linux: false + linux_armv7l: true + musllinux: false + macos: true + windows: true + permissions: + contents: read diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 78445e417e..7dfda9f842 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -6,10 +6,12 @@ on: paths: - "yt_dlp/**.py" - "!yt_dlp/version.py" - - "bundle/*.py" + - "bundle/**" - "pyproject.toml" - "Makefile" - ".github/workflows/build.yml" + - ".github/workflows/release.yml" + - ".github/workflows/release-master.yml" concurrency: group: release-master permissions: @@ -17,21 +19,20 @@ permissions: jobs: release: - if: vars.BUILD_MASTER != '' + if: vars.BUILD_MASTER uses: ./.github/workflows/release.yml with: prerelease: true - source: master + source: ${{ (github.repository != 'yt-dlp/yt-dlp' && vars.MASTER_ARCHIVE_REPO) || 'master' }} + target: 'master' permissions: contents: write - packages: write # For package cache - actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit publish_pypi: needs: [release] - if: vars.MASTER_PYPI_PROJECT != '' + if: vars.MASTER_PYPI_PROJECT runs-on: ubuntu-latest permissions: id-token: write # mandatory for trusted publishing diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 8f72844058..13cce8c33f 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -7,7 +7,7 @@ permissions: jobs: check_nightly: - if: vars.BUILD_NIGHTLY != '' + if: vars.BUILD_NIGHTLY runs-on: ubuntu-latest outputs: commit: ${{ steps.check_for_new_commits.outputs.commit }} @@ -22,9 +22,13 @@ jobs: "yt_dlp/*.py" ':!yt_dlp/version.py' "bundle/*.py" + "bundle/docker/compose.yml" + "bundle/docker/linux/*" "pyproject.toml" "Makefile" ".github/workflows/build.yml" + ".github/workflows/release.yml" + ".github/workflows/release-nightly.yml" ) echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT" @@ -34,17 +38,16 @@ jobs: uses: ./.github/workflows/release.yml with: prerelease: true - source: nightly + source: ${{ (github.repository != 'yt-dlp/yt-dlp' && vars.NIGHTLY_ARCHIVE_REPO) || 'nightly' }} + target: 'nightly' permissions: contents: write - packages: write # For package cache - actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit publish_pypi: needs: [release] - if: vars.NIGHTLY_PYPI_PROJECT != '' + if: vars.NIGHTLY_PYPI_PROJECT runs-on: ubuntu-latest permissions: id-token: write # mandatory for trusted publishing diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26b93e429c..acedebd306 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,10 @@ on: required: false default: '' type: string + linux_armv7l: + required: false + default: false + type: boolean prerelease: required: false default: true @@ -43,6 +47,10 @@ on: required: false default: '' type: string + linux_armv7l: + description: Include linux_armv7l + default: true + type: boolean prerelease: description: Pre-release default: false @@ -77,135 +85,57 @@ jobs: - name: Process inputs id: process_inputs + env: + INPUTS: ${{ toJSON(inputs) }} run: | - cat << EOF - ::group::Inputs - prerelease=${{ inputs.prerelease }} - source=${{ inputs.source }} - target=${{ inputs.target }} - version=${{ inputs.version }} - ::endgroup:: - EOF - IFS='@' read -r source_repo source_tag <<<"${{ inputs.source }}" - IFS='@' read -r target_repo target_tag <<<"${{ inputs.target }}" - cat << EOF >> "$GITHUB_OUTPUT" - source_repo=${source_repo} - source_tag=${source_tag} - target_repo=${target_repo} - target_tag=${target_tag} - EOF + python -m devscripts.setup_variables process_inputs - name: Setup variables id: setup_variables env: - source_repo: ${{ steps.process_inputs.outputs.source_repo }} - source_tag: ${{ steps.process_inputs.outputs.source_tag }} - target_repo: ${{ steps.process_inputs.outputs.target_repo }} - target_tag: ${{ steps.process_inputs.outputs.target_tag }} + INPUTS: ${{ toJSON(inputs) }} + PROCESSED: ${{ toJSON(steps.process_inputs.outputs) }} + REPOSITORY: ${{ github.repository }} + PUSH_VERSION_COMMIT: ${{ vars.PUSH_VERSION_COMMIT }} + PYPI_PROJECT: ${{ vars.PYPI_PROJECT }} + SOURCE_PYPI_PROJECT: ${{ vars[format('{0}_pypi_project', steps.process_inputs.outputs.source_repo)] }} + SOURCE_PYPI_SUFFIX: ${{ vars[format('{0}_pypi_suffix', steps.process_inputs.outputs.source_repo)] }} + TARGET_PYPI_PROJECT: ${{ vars[format('{0}_pypi_project', steps.process_inputs.outputs.target_repo)] }} + TARGET_PYPI_SUFFIX: ${{ vars[format('{0}_pypi_suffix', steps.process_inputs.outputs.target_repo)] }} + SOURCE_ARCHIVE_REPO: ${{ vars[format('{0}_archive_repo', steps.process_inputs.outputs.source_repo)] }} + TARGET_ARCHIVE_REPO: ${{ vars[format('{0}_archive_repo', steps.process_inputs.outputs.target_repo)] }} + HAS_SOURCE_ARCHIVE_REPO_TOKEN: ${{ !!secrets[format('{0}_archive_repo_token', steps.process_inputs.outputs.source_repo)] }} + HAS_TARGET_ARCHIVE_REPO_TOKEN: ${{ !!secrets[format('{0}_archive_repo_token', steps.process_inputs.outputs.target_repo)] }} + HAS_ARCHIVE_REPO_TOKEN: ${{ !!secrets.ARCHIVE_REPO_TOKEN }} run: | - # unholy bash monstrosity (sincere apologies) - fallback_token () { - if ${{ !secrets.ARCHIVE_REPO_TOKEN }}; then - echo "::error::Repository access secret ${target_repo_token^^} not found" - exit 1 - fi - target_repo_token=ARCHIVE_REPO_TOKEN - return 0 - } + python -m devscripts.setup_variables - source_is_channel=0 - [[ "${source_repo}" == 'stable' ]] && source_repo='yt-dlp/yt-dlp' - if [[ -z "${source_repo}" ]]; then - source_repo='${{ github.repository }}' - elif [[ '${{ vars[format('{0}_archive_repo', env.source_repo)] }}' ]]; then - source_is_channel=1 - source_channel='${{ vars[format('{0}_archive_repo', env.source_repo)] }}' - elif [[ -z "${source_tag}" && "${source_repo}" != */* ]]; then - source_tag="${source_repo}" - source_repo='${{ github.repository }}' - fi - resolved_source="${source_repo}" - if [[ "${source_tag}" ]]; then - resolved_source="${resolved_source}@${source_tag}" - elif [[ "${source_repo}" == 'yt-dlp/yt-dlp' ]]; then - resolved_source='stable' - fi - - revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}" - version="$( - python devscripts/update-version.py \ - -c "${resolved_source}" -r "${{ github.repository }}" ${{ inputs.version || '$revision' }} | \ - grep -Po "version=\K\d+\.\d+\.\d+(\.\d+)?")" - - if [[ "${target_repo}" ]]; then - if [[ -z "${target_tag}" ]]; then - if [[ '${{ vars[format('{0}_archive_repo', env.target_repo)] }}' ]]; then - target_tag="${source_tag:-${version}}" - else - target_tag="${target_repo}" - target_repo='${{ github.repository }}' - fi - fi - if [[ "${target_repo}" != '${{ github.repository}}' ]]; then - target_repo='${{ vars[format('{0}_archive_repo', env.target_repo)] }}' - target_repo_token='${{ env.target_repo }}_archive_repo_token' - ${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token - pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}' - pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}' - fi - else - target_tag="${source_tag:-${version}}" - if ((source_is_channel)); then - target_repo="${source_channel}" - target_repo_token='${{ env.source_repo }}_archive_repo_token' - ${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token - pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}' - pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}' - else - target_repo='${{ github.repository }}' - fi - fi - - if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then - pypi_project='${{ vars.PYPI_PROJECT }}' - fi - - echo "::group::Output variables" - cat << EOF | tee -a "$GITHUB_OUTPUT" - channel=${resolved_source} - version=${version} - target_repo=${target_repo} - target_repo_token=${target_repo_token} - target_tag=${target_tag} - pypi_project=${pypi_project} - pypi_suffix=${pypi_suffix} - EOF - echo "::endgroup::" - - - name: Update documentation + - name: Update version & documentation env: - version: ${{ steps.setup_variables.outputs.version }} - target_repo: ${{ steps.setup_variables.outputs.target_repo }} - if: | - !inputs.prerelease && env.target_repo == github.repository + CHANNEL: ${{ steps.setup_variables.outputs.channel }} + # Use base repo since this could be committed; build jobs will call this again with true origin + REPOSITORY: ${{ github.repository }} + VERSION: ${{ steps.setup_variables.outputs.version }} run: | + python devscripts/update-version.py -c "${CHANNEL}" -r "${REPOSITORY}" "${VERSION}" python devscripts/update_changelog.py -vv make doc - name: Push to release id: push_release env: - version: ${{ steps.setup_variables.outputs.version }} - target_repo: ${{ steps.setup_variables.outputs.target_repo }} + VERSION: ${{ steps.setup_variables.outputs.version }} + GITHUB_EVENT_SENDER_LOGIN: ${{ github.event.sender.login }} + GITHUB_EVENT_REF: ${{ github.event.ref }} if: | - !inputs.prerelease && env.target_repo == github.repository + !inputs.prerelease && steps.setup_variables.outputs.target_repo == github.repository run: | git config --global user.name "github-actions[bot]" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -u - git commit -m "Release ${{ env.version }}" \ - -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all" - git push origin --force ${{ github.event.ref }}:release + git commit -m "Release ${VERSION}" \ + -m "Created by: ${GITHUB_EVENT_SENDER_LOGIN}" -m ":ci skip all" + git push origin --force "${GITHUB_EVENT_REF}:release" - name: Get target commitish id: get_target @@ -214,10 +144,10 @@ jobs: - name: Update master env: - target_repo: ${{ steps.setup_variables.outputs.target_repo }} + GITHUB_EVENT_REF: ${{ github.event.ref }} if: | - vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease && env.target_repo == github.repository - run: git push origin ${{ github.event.ref }} + vars.PUSH_VERSION_COMMIT && !inputs.prerelease && steps.setup_variables.outputs.target_repo == github.repository + run: git push origin "${GITHUB_EVENT_REF}" build: needs: prepare @@ -226,10 +156,9 @@ jobs: version: ${{ needs.prepare.outputs.version }} channel: ${{ needs.prepare.outputs.channel }} origin: ${{ needs.prepare.outputs.target_repo }} + linux_armv7l: ${{ inputs.linux_armv7l }} permissions: contents: read - packages: write # For package cache - actions: write # For cleaning up cache secrets: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} @@ -255,16 +184,16 @@ jobs: - name: Prepare env: - version: ${{ needs.prepare.outputs.version }} - suffix: ${{ needs.prepare.outputs.pypi_suffix }} - channel: ${{ needs.prepare.outputs.channel }} - target_repo: ${{ needs.prepare.outputs.target_repo }} - pypi_project: ${{ needs.prepare.outputs.pypi_project }} + VERSION: ${{ needs.prepare.outputs.version }} + SUFFIX: ${{ needs.prepare.outputs.pypi_suffix }} + CHANNEL: ${{ needs.prepare.outputs.channel }} + TARGET_REPO: ${{ needs.prepare.outputs.target_repo }} + PYPI_PROJECT: ${{ needs.prepare.outputs.pypi_project }} run: | - python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}" + python devscripts/update-version.py -c "${CHANNEL}" -r "${TARGET_REPO}" -s "${SUFFIX}" "${VERSION}" python devscripts/update_changelog.py -vv python devscripts/make_lazy_extractors.py - sed -i -E '0,/(name = ")[^"]+(")/s//\1${{ env.pypi_project }}\2/' pyproject.toml + sed -i -E '0,/(name = ")[^"]+(")/s//\1'"${PYPI_PROJECT}"'\2/' pyproject.toml - name: Build run: | @@ -298,7 +227,11 @@ jobs: permissions: contents: write runs-on: ubuntu-latest - + env: + TARGET_REPO: ${{ needs.prepare.outputs.target_repo }} + TARGET_TAG: ${{ needs.prepare.outputs.target_tag }} + VERSION: ${{ needs.prepare.outputs.version }} + HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} steps: - uses: actions/checkout@v4 with: @@ -314,81 +247,80 @@ jobs: - name: Generate release notes env: - head_sha: ${{ needs.prepare.outputs.head_sha }} - target_repo: ${{ needs.prepare.outputs.target_repo }} - target_tag: ${{ needs.prepare.outputs.target_tag }} + REPOSITORY: ${{ github.repository }} + BASE_REPO: yt-dlp/yt-dlp + NIGHTLY_REPO: yt-dlp/yt-dlp-nightly-builds + MASTER_REPO: yt-dlp/yt-dlp-master-builds + DOCS_PATH: ${{ env.TARGET_REPO == github.repository && format('/tree/{0}', env.TARGET_TAG) || '' }} run: | printf '%s' \ - '[![Installation](https://img.shields.io/badge/-Which%20file%20to%20download%3F-white.svg?style=for-the-badge)]' \ - '(https://github.com/${{ github.repository }}#installation "Installation instructions") ' \ - '[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]' \ - '(https://discord.gg/H5MNcFW63r "Discord") ' \ - '[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]' \ - '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \ - '[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]' \ - '(https://github.com/${{ github.repository }}' \ - '${{ env.target_repo == github.repository && format('/tree/{0}', env.target_tag) || '' }}#readme "Documentation") ' \ - ${{ env.target_repo == 'yt-dlp/yt-dlp' && '\ + "[![Installation](https://img.shields.io/badge/-Which%20file%20to%20download%3F-white.svg?style=for-the-badge)]" \ + "(https://github.com/${REPOSITORY}#installation \"Installation instructions\") " \ + "[![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)]" \ + "(https://discord.gg/H5MNcFW63r \"Discord\") " \ + "[![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)]" \ + "(https://github.com/${BASE_REPO}/blob/master/Collaborators.md#collaborators \"Donate\") " \ + "[![Documentation](https://img.shields.io/badge/-Docs-brightgreen.svg?style=for-the-badge&logo=GitBook&labelColor=555555)]" \ + "(https://github.com/${REPOSITORY}${DOCS_PATH}#readme \"Documentation\") " > ./RELEASE_NOTES + if [[ "${TARGET_REPO}" == "${BASE_REPO}" ]]; then + printf '%s' \ "[![Nightly](https://img.shields.io/badge/Nightly%20builds-purple.svg?style=for-the-badge)]" \ - "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\") " \ + "(https://github.com/${NIGHTLY_REPO}/releases/latest \"Nightly builds\") " \ "[![Master](https://img.shields.io/badge/Master%20builds-lightblue.svg?style=for-the-badge)]" \ - "(https://github.com/yt-dlp/yt-dlp-master-builds/releases/latest \"Master builds\")"' || '' }} > ./RELEASE_NOTES + "(https://github.com/${MASTER_REPO}/releases/latest \"Master builds\")" >> ./RELEASE_NOTES + fi printf '\n\n' >> ./RELEASE_NOTES cat >> ./RELEASE_NOTES << EOF - #### A description of the various files is in the [README](https://github.com/${{ github.repository }}#release-files) + #### A description of the various files is in the [README](https://github.com/${REPOSITORY}#release-files) --- $(python ./devscripts/make_changelog.py -vv --collapsible) EOF printf '%s\n\n' '**This is a pre-release build**' >> ./PRERELEASE_NOTES cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES - printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ env.head_sha }}' >> ./ARCHIVE_NOTES + printf '%s\n\n' "Generated from: https://github.com/${REPOSITORY}/commit/${HEAD_SHA}" >> ./ARCHIVE_NOTES cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES - name: Publish to archive repo env: GH_TOKEN: ${{ secrets[needs.prepare.outputs.target_repo_token] }} GH_REPO: ${{ needs.prepare.outputs.target_repo }} - version: ${{ needs.prepare.outputs.version }} - channel: ${{ needs.prepare.outputs.channel }} + TITLE_PREFIX: ${{ startswith(env.TARGET_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }} + TITLE: ${{ inputs.target != env.TARGET_REPO && inputs.target || needs.prepare.outputs.channel }} if: | - inputs.prerelease && env.GH_TOKEN != '' && env.GH_REPO != '' && env.GH_REPO != github.repository + inputs.prerelease && env.GH_TOKEN && env.GH_REPO && env.GH_REPO != github.repository run: | - title="${{ startswith(env.GH_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }}${{ env.channel }}" gh release create \ --notes-file ARCHIVE_NOTES \ - --title "${title} ${{ env.version }}" \ - ${{ env.version }} \ + --title "${TITLE_PREFIX}${TITLE} ${VERSION}" \ + "${VERSION}" \ artifact/* - name: Prune old release env: GH_TOKEN: ${{ github.token }} - version: ${{ needs.prepare.outputs.version }} - target_repo: ${{ needs.prepare.outputs.target_repo }} - target_tag: ${{ needs.prepare.outputs.target_tag }} if: | - env.target_repo == github.repository && env.target_tag != env.version + env.TARGET_REPO == github.repository && env.TARGET_TAG != env.VERSION run: | - gh release delete --yes --cleanup-tag "${{ env.target_tag }}" || true - git tag --delete "${{ env.target_tag }}" || true + gh release delete --yes --cleanup-tag "${TARGET_TAG}" || true + git tag --delete "${TARGET_TAG}" || true sleep 5 # Enough time to cover deletion race condition - name: Publish release env: GH_TOKEN: ${{ github.token }} - version: ${{ needs.prepare.outputs.version }} - target_repo: ${{ needs.prepare.outputs.target_repo }} - target_tag: ${{ needs.prepare.outputs.target_tag }} - head_sha: ${{ needs.prepare.outputs.head_sha }} + NOTES_FILE: ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} + TITLE_PREFIX: ${{ github.repository == 'yt-dlp/yt-dlp' && 'yt-dlp ' || '' }} + TITLE: ${{ env.TARGET_TAG != env.VERSION && format('{0} ', env.TARGET_TAG) || '' }} + PRERELEASE: ${{ inputs.prerelease && '1' || '0' }} if: | - env.target_repo == github.repository + env.TARGET_REPO == github.repository run: | - title="${{ github.repository == 'yt-dlp/yt-dlp' && 'yt-dlp ' || '' }}" - title+="${{ env.target_tag != env.version && format('{0} ', env.target_tag) || '' }}" - gh release create \ - --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \ - --target ${{ env.head_sha }} \ - --title "${title}${{ env.version }}" \ - ${{ inputs.prerelease && '--prerelease' || '' }} \ - ${{ env.target_tag }} \ - artifact/* + gh_options=( + --notes-file "${NOTES_FILE}" + --target "${HEAD_SHA}" + --title "${TITLE_PREFIX}${TITLE}${VERSION}" + ) + if ((PRERELEASE)); then + gh_options+=(--prerelease) + fi + gh release create "${gh_options[@]}" "${TARGET_TAG}" artifact/* diff --git a/README.md b/README.md index f34ac19c8a..e582e1e53c 100644 --- a/README.md +++ b/README.md @@ -105,14 +105,20 @@ File|Description File|Description :---|:--- +[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux (glibc 2.17+) standalone x86_64 binary +[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux (glibc 2.17+) x86_64 executable (no auto-update) +[yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux (glibc 2.17+) standalone aarch64 binary +[yt-dlp_linux_aarch64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64.zip)|Unpackaged Linux (glibc 2.17+) aarch64 executable (no auto-update) +[yt-dlp_linux_armv7l.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l.zip)|Unpackaged Linux (glibc 2.31+) armv7l executable (no auto-update) +[yt-dlp_musllinux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux)|Linux (musl 1.2+) standalone x86_64 binary +[yt-dlp_musllinux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux.zip)|Unpackaged Linux (musl 1.2+) x86_64 executable (no auto-update) +[yt-dlp_musllinux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux_aarch64)|Linux (musl 1.2+) standalone aarch64 binary +[yt-dlp_musllinux_aarch64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux_aarch64.zip)|Unpackaged Linux (musl 1.2+) aarch64 executable (no auto-update) [yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Win8+) standalone x86 (32-bit) binary -[yt-dlp_arm64.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_arm64.exe)|Windows (Win10+) standalone arm64 (64-bit) binary -[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary -[yt-dlp_linux_armv7l](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l)|Linux standalone armv7l (32-bit) binary -[yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux standalone aarch64 (64-bit) binary +[yt-dlp_win_x86.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win_x86.zip)|Unpackaged Windows (Win8+) x86 (32-bit) executable (no auto-update) +[yt-dlp_arm64.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_arm64.exe)|Windows (Win10+) standalone ARM64 binary +[yt-dlp_win_arm64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win_arm64.zip)|Unpackaged Windows (Win10+) ARM64 executable (no auto-update) [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows (Win8+) x64 executable (no auto-update) -[yt-dlp_win_x86.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win_x86.zip)|Unpackaged Windows (Win8+) x86 executable (no auto-update) -[yt-dlp_win_arm64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win_arm64.zip)|Unpackaged Windows (Win10+) arm64 executable (no auto-update) [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update) #### Misc @@ -206,7 +212,7 @@ The following provide support for impersonating browser requests. This may be re * [**curl_cffi**](https://github.com/lexiforest/curl_cffi) (recommended) - Python binding for [curl-impersonate](https://github.com/lexiforest/curl-impersonate). Provides impersonation targets for Chrome, Edge and Safari. Licensed under [MIT](https://github.com/lexiforest/curl_cffi/blob/main/LICENSE) * Can be installed with the `curl-cffi` group, e.g. `pip install "yt-dlp[default,curl-cffi]"` - * Currently included in `yt-dlp.exe`, `yt-dlp_linux` and `yt-dlp_macos` builds + * Currently included in most builds *except* `yt-dlp` (Unix zipimport binary), `yt-dlp_x86` (Windows 32-bit) and `yt-dlp_musllinux_aarch64` ### Metadata diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml index 5f89ca6d09..1216aac5d7 100644 --- a/bundle/docker/compose.yml +++ b/bundle/docker/compose.yml @@ -1,10 +1,153 @@ services: - static: - build: static + + linux_x86_64: + build: + context: linux + target: build + platforms: + - "linux/amd64" + args: + BUILDIMAGE: ghcr.io/yt-dlp/manylinux2014_x86_64-shared:latest environment: - channel: ${channel} - origin: ${origin} - version: ${version} + EXE_NAME: ${EXE_NAME:?} + CHANNEL: ${CHANNEL:?} + ORIGIN: ${ORIGIN:?} + VERSION: volumes: - - ~/build:/build - ../..:/yt-dlp + + linux_x86_64_verify: + build: + context: linux + target: verify + platforms: + - "linux/amd64" + args: + VERIFYIMAGE: quay.io/pypa/manylinux2014_x86_64:latest + environment: + EXE_NAME: ${EXE_NAME:?} + volumes: + - ../../dist:/build + + linux_aarch64: + build: + context: linux + target: build + platforms: + - "linux/arm64" + args: + BUILDIMAGE: ghcr.io/yt-dlp/manylinux2014_aarch64-shared:latest + environment: + EXE_NAME: ${EXE_NAME:?} + CHANNEL: ${CHANNEL:?} + ORIGIN: ${ORIGIN:?} + VERSION: + volumes: + - ../..:/yt-dlp + + linux_aarch64_verify: + build: + context: linux + target: verify + platforms: + - "linux/arm64" + args: + VERIFYIMAGE: quay.io/pypa/manylinux2014_aarch64:latest + environment: + EXE_NAME: ${EXE_NAME:?} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a glibc2.17 aarch64 release to --update-to + volumes: + - ../../dist:/build + + linux_armv7l: + build: + context: linux + target: build + platforms: + - "linux/arm/v7" + args: + BUILDIMAGE: ghcr.io/yt-dlp/manylinux_2_31_armv7l-shared:latest + environment: + EXE_NAME: ${EXE_NAME:?} + CHANNEL: ${CHANNEL:?} + ORIGIN: ${ORIGIN:?} + VERSION: + SKIP_ONEFILE_BUILD: "1" + volumes: + - ../..:/yt-dlp + - ~/yt-dlp-build-venv:/yt-dlp-build-venv + + linux_armv7l_verify: + build: + context: linux + target: verify + platforms: + - "linux/arm/v7" + args: + VERIFYIMAGE: arm32v7/debian:bullseye + environment: + EXE_NAME: ${EXE_NAME:?} + TEST_ONEDIR_BUILD: "1" + volumes: + - ../../dist:/build + + musllinux_x86_64: + build: + context: linux + target: build + platforms: + - "linux/amd64" + args: + BUILDIMAGE: ghcr.io/yt-dlp/musllinux_1_2_x86_64-shared:latest + environment: + EXE_NAME: ${EXE_NAME:?} + CHANNEL: ${CHANNEL:?} + ORIGIN: ${ORIGIN:?} + VERSION: + volumes: + - ../..:/yt-dlp + + musllinux_x86_64_verify: + build: + context: linux + target: verify + platforms: + - "linux/amd64" + args: + VERIFYIMAGE: alpine:3.22 + environment: + EXE_NAME: ${EXE_NAME:?} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + volumes: + - ../../dist:/build + + musllinux_aarch64: + build: + context: linux + target: build + platforms: + - "linux/arm64" + args: + BUILDIMAGE: ghcr.io/yt-dlp/musllinux_1_2_aarch64-shared:latest + environment: + EXE_NAME: ${EXE_NAME:?} + CHANNEL: ${CHANNEL:?} + ORIGIN: ${ORIGIN:?} + VERSION: + EXCLUDE_CURL_CFFI: "1" + volumes: + - ../..:/yt-dlp + + musllinux_aarch64_verify: + build: + context: linux + target: verify + platforms: + - "linux/arm64" + args: + VERIFYIMAGE: alpine:3.22 + environment: + EXE_NAME: ${EXE_NAME:?} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + volumes: + - ../../dist:/build diff --git a/bundle/docker/linux/Dockerfile b/bundle/docker/linux/Dockerfile new file mode 100644 index 0000000000..84bc03d9ca --- /dev/null +++ b/bundle/docker/linux/Dockerfile @@ -0,0 +1,16 @@ +ARG BUILDIMAGE=ghcr.io/yt-dlp/manylinux2014_x86_64-shared:latest +ARG VERIFYIMAGE=alpine:3.22 + + +FROM $BUILDIMAGE AS build + +WORKDIR /yt-dlp +COPY build.sh /build.sh +ENTRYPOINT ["/build.sh"] + + +FROM $VERIFYIMAGE AS verify + +WORKDIR /testing +COPY verify.sh /verify.sh +ENTRYPOINT ["/verify.sh"] diff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.sh new file mode 100755 index 0000000000..1ce330a5b0 --- /dev/null +++ b/bundle/docker/linux/build.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -exuo pipefail + +if [[ -z "${USE_PYTHON_VERSION:-}" ]]; then + USE_PYTHON_VERSION="3.13" +fi + +function runpy { + "/opt/shared-cpython-${USE_PYTHON_VERSION}/bin/python${USE_PYTHON_VERSION}" "$@" +} + +function venvpy { + "python${USE_PYTHON_VERSION}" "$@" +} + +INCLUDES=( + --include pyinstaller + --include secretstorage +) + +if [[ -z "${EXCLUDE_CURL_CFFI:-}" ]]; then + INCLUDES+=(--include curl-cffi) +fi + +runpy -m venv /yt-dlp-build-venv +source /yt-dlp-build-venv/bin/activate +# Inside the venv we use venvpy instead of runpy +venvpy -m ensurepip --upgrade --default-pip +venvpy -m devscripts.install_deps -o --include build +venvpy -m devscripts.install_deps "${INCLUDES[@]}" +venvpy -m devscripts.make_lazy_extractors +venvpy devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" + +if [[ -z "${SKIP_ONEDIR_BUILD:-}" ]]; then + mkdir -p /build + venvpy -m bundle.pyinstaller --onedir --distpath=/build + pushd "/build/${EXE_NAME}" + chmod +x "${EXE_NAME}" + venvpy -m zipfile -c "/yt-dlp/dist/${EXE_NAME}.zip" ./ + popd +fi + +if [[ -z "${SKIP_ONEFILE_BUILD:-}" ]]; then + venvpy -m bundle.pyinstaller + chmod +x "./dist/${EXE_NAME}" +fi diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.sh new file mode 100755 index 0000000000..94de5f3e27 --- /dev/null +++ b/bundle/docker/linux/verify.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then + echo "Extracting zip to verify onedir build" + if command -v python3 >/dev/null 2>&1; then + python3 -m zipfile -e "/build/${EXE_NAME}.zip" ./ + else + echo "Attempting to install unzip" + if command -v dnf >/dev/null 2>&1; then + dnf -y install --allowerasing unzip + elif command -v yum >/dev/null 2>&1; then + yum -y install unzip + elif command -v apt-get >/dev/null 2>&1; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends unzip + elif command -v apk >/dev/null 2>&1; then + apk add --no-cache unzip + else + echo "Unsupported image" + exit 1 + fi + unzip "/build/${EXE_NAME}.zip" -d ./ + fi +else + echo "Verifying onefile build" + cp "/build/${EXE_NAME}" ./ +fi + +chmod +x "./${EXE_NAME}" + +if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then + "./${EXE_NAME}" -v || true + "./${EXE_NAME}" --version + exit 0 +fi + +cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded" +version="$("./${EXE_NAME}" --version)" +"./${EXE_NAME}_downgraded" -v --update-to yt-dlp/yt-dlp@2023.03.04 +downgraded_version="$("./${EXE_NAME}_downgraded" --version)" +if [ "${version}" = "${downgraded_version}" ]; then + exit 1 +fi diff --git a/bundle/docker/static/Dockerfile b/bundle/docker/static/Dockerfile deleted file mode 100644 index dae2dff3d8..0000000000 --- a/bundle/docker/static/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM alpine:3.19 as base - -RUN apk --update add --no-cache \ - build-base \ - python3 \ - pipx \ - ; - -RUN pipx install pyinstaller -# Requires above step to prepare the shared venv -RUN ~/.local/share/pipx/shared/bin/python -m pip install -U wheel -RUN apk --update add --no-cache \ - scons \ - patchelf \ - binutils \ - ; -RUN pipx install staticx - -WORKDIR /yt-dlp -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT /entrypoint.sh diff --git a/bundle/docker/static/entrypoint.sh b/bundle/docker/static/entrypoint.sh deleted file mode 100755 index 8049e68205..0000000000 --- a/bundle/docker/static/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/ash -set -e - -source ~/.local/share/pipx/venvs/pyinstaller/bin/activate -python -m devscripts.install_deps -o --include build -python -m devscripts.install_deps --include secretstorage --include curl-cffi -python -m devscripts.make_lazy_extractors -python devscripts/update-version.py -c "${channel}" -r "${origin}" "${version}" -python -m bundle.pyinstaller -deactivate - -source ~/.local/share/pipx/venvs/staticx/bin/activate -staticx /yt-dlp/dist/yt-dlp_linux /build/yt-dlp_linux -deactivate diff --git a/bundle/pyinstaller.py b/bundle/pyinstaller.py index 0597f602d0..8286f077ab 100755 --- a/bundle/pyinstaller.py +++ b/bundle/pyinstaller.py @@ -13,6 +13,8 @@ from PyInstaller.__main__ import run as run_pyinstaller from devscripts.utils import read_version OS_NAME, MACHINE, ARCH = sys.platform, platform.machine().lower(), platform.architecture()[0][:2] +if OS_NAME == 'linux' and platform.libc_ver()[0] != 'glibc': + OS_NAME = 'musllinux' if MACHINE in ('x86', 'x86_64', 'amd64', 'i386', 'i686'): MACHINE = 'x86' if ARCH == '32' else '' diff --git a/devscripts/setup_variables.py b/devscripts/setup_variables.py new file mode 100644 index 0000000000..a45a36835c --- /dev/null +++ b/devscripts/setup_variables.py @@ -0,0 +1,157 @@ +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import datetime as dt +import json + +from devscripts.utils import calculate_version + + +STABLE_REPOSITORY = 'yt-dlp/yt-dlp' + + +def setup_variables(environment): + """ + `environment` must contain these keys: + REPOSITORY, INPUTS, PROCESSED, + PUSH_VERSION_COMMIT, PYPI_PROJECT, + SOURCE_PYPI_PROJECT, SOURCE_PYPI_SUFFIX, + TARGET_PYPI_PROJECT, TARGET_PYPI_SUFFIX, + SOURCE_ARCHIVE_REPO, TARGET_ARCHIVE_REPO, + HAS_SOURCE_ARCHIVE_REPO_TOKEN, + HAS_TARGET_ARCHIVE_REPO_TOKEN, + HAS_ARCHIVE_REPO_TOKEN + + `INPUTS` must contain these keys: + prerelease + + `PROCESSED` must contain these keys: + source_repo, source_tag, + target_repo, target_tag + """ + REPOSITORY = environment['REPOSITORY'] + INPUTS = json.loads(environment['INPUTS']) + PROCESSED = json.loads(environment['PROCESSED']) + + source_channel = None + does_not_have_needed_token = False + target_repo_token = None + pypi_project = None + pypi_suffix = None + + source_repo = PROCESSED['source_repo'] + source_tag = PROCESSED['source_tag'] + if source_repo == 'stable': + source_repo = STABLE_REPOSITORY + if not source_repo: + source_repo = REPOSITORY + elif environment['SOURCE_ARCHIVE_REPO']: + source_channel = environment['SOURCE_ARCHIVE_REPO'] + elif not source_tag and '/' not in source_repo: + source_tag = source_repo + source_repo = REPOSITORY + + resolved_source = source_repo + if source_tag: + resolved_source = f'{resolved_source}@{source_tag}' + elif source_repo == STABLE_REPOSITORY: + resolved_source = 'stable' + + revision = None + if INPUTS['prerelease'] or not environment['PUSH_VERSION_COMMIT']: + revision = dt.datetime.now(tz=dt.timezone.utc).strftime('%H%M%S') + + version = calculate_version(INPUTS.get('version') or revision) + + target_repo = PROCESSED['target_repo'] + target_tag = PROCESSED['target_tag'] + if target_repo: + if target_repo == 'stable': + target_repo = STABLE_REPOSITORY + if not target_tag: + if target_repo == STABLE_REPOSITORY: + target_tag = version + elif environment['TARGET_ARCHIVE_REPO']: + target_tag = source_tag or version + else: + target_tag = target_repo + target_repo = REPOSITORY + if target_repo != REPOSITORY: + target_repo = environment['TARGET_ARCHIVE_REPO'] + target_repo_token = f'{PROCESSED["target_repo"].upper()}_ARCHIVE_REPO_TOKEN' + if not json.loads(environment['HAS_TARGET_ARCHIVE_REPO_TOKEN']): + does_not_have_needed_token = True + pypi_project = environment['TARGET_PYPI_PROJECT'] or None + pypi_suffix = environment['TARGET_PYPI_SUFFIX'] or None + else: + target_tag = source_tag or version + if source_channel: + target_repo = source_channel + target_repo_token = f'{PROCESSED["source_repo"].upper()}_ARCHIVE_REPO_TOKEN' + if not json.loads(environment['HAS_SOURCE_ARCHIVE_REPO_TOKEN']): + does_not_have_needed_token = True + pypi_project = environment['SOURCE_PYPI_PROJECT'] or None + pypi_suffix = environment['SOURCE_PYPI_SUFFIX'] or None + else: + target_repo = REPOSITORY + + if does_not_have_needed_token: + if not json.loads(environment['HAS_ARCHIVE_REPO_TOKEN']): + print(f'::error::Repository access secret {target_repo_token} not found') + return None + target_repo_token = 'ARCHIVE_REPO_TOKEN' + + if target_repo == REPOSITORY and not INPUTS['prerelease']: + pypi_project = environment['PYPI_PROJECT'] or None + + return { + 'channel': resolved_source, + 'version': version, + 'target_repo': target_repo, + 'target_repo_token': target_repo_token, + 'target_tag': target_tag, + 'pypi_project': pypi_project, + 'pypi_suffix': pypi_suffix, + } + + +def process_inputs(inputs): + outputs = {} + for key in ('source', 'target'): + repo, _, tag = inputs.get(key, '').partition('@') + outputs[f'{key}_repo'] = repo + outputs[f'{key}_tag'] = tag + return outputs + + +if __name__ == '__main__': + if not os.getenv('GITHUB_OUTPUT'): + print('This script is only intended for use with GitHub Actions', file=sys.stderr) + sys.exit(1) + + if 'process_inputs' in sys.argv: + inputs = json.loads(os.environ['INPUTS']) + print('::group::Inputs') + print(json.dumps(inputs, indent=2)) + print('::endgroup::') + outputs = process_inputs(inputs) + print('::group::Processed') + print(json.dumps(outputs, indent=2)) + print('::endgroup::') + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write('\n'.join(f'{key}={value}' for key, value in outputs.items())) + sys.exit(0) + + outputs = setup_variables(dict(os.environ)) + if not outputs: + sys.exit(1) + + print('::group::Output variables') + print(json.dumps(outputs, indent=2)) + print('::endgroup::') + + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write('\n'.join(f'{key}={value or ""}' for key, value in outputs.items())) diff --git a/devscripts/setup_variables_tests.py b/devscripts/setup_variables_tests.py new file mode 100644 index 0000000000..8cb52daa1f --- /dev/null +++ b/devscripts/setup_variables_tests.py @@ -0,0 +1,331 @@ +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import datetime as dt +import json + +from devscripts.setup_variables import STABLE_REPOSITORY, process_inputs, setup_variables +from devscripts.utils import calculate_version + + +def _test(github_repository, note, repo_vars, repo_secrets, inputs, expected=None, ignore_revision=False): + inp = inputs.copy() + inp.setdefault('linux_armv7l', True) + inp.setdefault('prerelease', False) + processed = process_inputs(inp) + source_repo = processed['source_repo'].upper() + target_repo = processed['target_repo'].upper() + variables = {k.upper(): v for k, v in repo_vars.items()} + secrets = {k.upper(): v for k, v in repo_secrets.items()} + + env = { + # Keep this in sync with prepare.setup_variables in release.yml + 'INPUTS': json.dumps(inp), + 'PROCESSED': json.dumps(processed), + 'REPOSITORY': github_repository, + 'PUSH_VERSION_COMMIT': variables.get('PUSH_VERSION_COMMIT') or '', + 'PYPI_PROJECT': variables.get('PYPI_PROJECT') or '', + 'SOURCE_PYPI_PROJECT': variables.get(f'{source_repo}_PYPI_PROJECT') or '', + 'SOURCE_PYPI_SUFFIX': variables.get(f'{source_repo}_PYPI_SUFFIX') or '', + 'TARGET_PYPI_PROJECT': variables.get(f'{target_repo}_PYPI_PROJECT') or '', + 'TARGET_PYPI_SUFFIX': variables.get(f'{target_repo}_PYPI_SUFFIX') or '', + 'SOURCE_ARCHIVE_REPO': variables.get(f'{source_repo}_ARCHIVE_REPO') or '', + 'TARGET_ARCHIVE_REPO': variables.get(f'{target_repo}_ARCHIVE_REPO') or '', + 'HAS_SOURCE_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get(f'{source_repo}_ARCHIVE_REPO_TOKEN'))), + 'HAS_TARGET_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get(f'{target_repo}_ARCHIVE_REPO_TOKEN'))), + 'HAS_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get('ARCHIVE_REPO_TOKEN'))), + } + + result = setup_variables(env) + if not expected: + print(' {\n' + '\n'.join(f' {k!r}: {v!r},' for k, v in result.items()) + '\n }') + return + + exp = expected.copy() + if ignore_revision: + assert len(result['version']) == len(exp['version']), f'revision missing: {github_repository} {note}' + version_is_tag = result['version'] == result['target_tag'] + for dct in (result, exp): + dct['version'] = '.'.join(dct['version'].split('.')[:3]) + if version_is_tag: + dct['target_tag'] = dct['version'] + assert result == exp, f'unexpected result: {github_repository} {note}' + + +def main(): + DEFAULT_VERSION_WITH_REVISION = dt.datetime.now(tz=dt.timezone.utc).strftime('%Y.%m.%d.%H%M%S') + DEFAULT_VERSION = calculate_version() + BASE_REPO_VARS = { + 'MASTER_ARCHIVE_REPO': 'yt-dlp/yt-dlp-master-builds', + 'NIGHTLY_ARCHIVE_REPO': 'yt-dlp/yt-dlp-nightly-builds', + 'NIGHTLY_PYPI_PROJECT': 'yt-dlp', + 'NIGHTLY_PYPI_SUFFIX': 'dev', + 'PUSH_VERSION_COMMIT': '1', + 'PYPI_PROJECT': 'yt-dlp', + } + BASE_REPO_SECRETS = { + 'ARCHIVE_REPO_TOKEN': '1', + } + FORK_REPOSITORY = 'fork/yt-dlp' + FORK_ORG = FORK_REPOSITORY.partition('/')[0] + + _test( + STABLE_REPOSITORY, 'official vars/secrets, stable', + BASE_REPO_VARS, BASE_REPO_SECRETS, {}, { + 'channel': 'stable', + 'version': DEFAULT_VERSION, + 'target_repo': STABLE_REPOSITORY, + 'target_repo_token': None, + 'target_tag': DEFAULT_VERSION, + 'pypi_project': 'yt-dlp', + 'pypi_suffix': None, + }) + _test( + STABLE_REPOSITORY, 'official vars/secrets, nightly (w/o target)', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'source': 'nightly', + 'prerelease': True, + }, { + 'channel': 'nightly', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': 'yt-dlp/yt-dlp-nightly-builds', + 'target_repo_token': 'ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': 'yt-dlp', + 'pypi_suffix': 'dev', + }, ignore_revision=True) + _test( + STABLE_REPOSITORY, 'official vars/secrets, nightly', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'source': 'nightly', + 'target': 'nightly', + 'prerelease': True, + }, { + 'channel': 'nightly', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': 'yt-dlp/yt-dlp-nightly-builds', + 'target_repo_token': 'ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': 'yt-dlp', + 'pypi_suffix': 'dev', + }, ignore_revision=True) + _test( + STABLE_REPOSITORY, 'official vars/secrets, master (w/o target)', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'source': 'master', + 'prerelease': True, + }, { + 'channel': 'master', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': 'yt-dlp/yt-dlp-master-builds', + 'target_repo_token': 'ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + STABLE_REPOSITORY, 'official vars/secrets, master', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'source': 'master', + 'target': 'master', + 'prerelease': True, + }, { + 'channel': 'master', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': 'yt-dlp/yt-dlp-master-builds', + 'target_repo_token': 'ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + STABLE_REPOSITORY, 'official vars/secrets, special tag, updates to stable', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'target': f'{STABLE_REPOSITORY}@experimental', + 'prerelease': True, + }, { + 'channel': 'stable', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': STABLE_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'experimental', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + STABLE_REPOSITORY, 'official vars/secrets, special tag, "stable" as target repo', + BASE_REPO_VARS, BASE_REPO_SECRETS, { + 'target': 'stable@experimental', + 'prerelease': True, + }, { + 'channel': 'stable', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': STABLE_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'experimental', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + + _test( + FORK_REPOSITORY, 'fork w/o vars/secrets, stable', + {}, {}, {}, { + 'channel': FORK_REPOSITORY, + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork w/o vars/secrets, prerelease', + {}, {}, {'prerelease': True}, { + 'channel': FORK_REPOSITORY, + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork w/o vars/secrets, nightly', + {}, {}, { + 'prerelease': True, + 'source': 'nightly', + 'target': 'nightly', + }, { + 'channel': f'{FORK_REPOSITORY}@nightly', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'nightly', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork w/o vars/secrets, master', + {}, {}, { + 'prerelease': True, + 'source': 'master', + 'target': 'master', + }, { + 'channel': f'{FORK_REPOSITORY}@master', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'master', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork w/o vars/secrets, revision', + {}, {}, {'version': '123'}, { + 'channel': FORK_REPOSITORY, + 'version': f'{DEFAULT_VERSION[:10]}.123', + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': f'{DEFAULT_VERSION[:10]}.123', + 'pypi_project': None, + 'pypi_suffix': None, + }) + + _test( + FORK_REPOSITORY, 'fork w/ PUSH_VERSION_COMMIT, stable', + {'PUSH_VERSION_COMMIT': '1'}, {}, {}, { + 'channel': FORK_REPOSITORY, + 'version': DEFAULT_VERSION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': DEFAULT_VERSION, + 'pypi_project': None, + 'pypi_suffix': None, + }) + _test( + FORK_REPOSITORY, 'fork w/ PUSH_VERSION_COMMIT, prerelease', + {'PUSH_VERSION_COMMIT': '1'}, {}, {'prerelease': True}, { + 'channel': FORK_REPOSITORY, + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + + _test( + FORK_REPOSITORY, 'fork w/NIGHTLY_ARCHIVE_REPO_TOKEN, nightly', { + 'NIGHTLY_ARCHIVE_REPO': f'{FORK_ORG}/yt-dlp-nightly-builds', + 'PYPI_PROJECT': 'yt-dlp-test', + }, { + 'NIGHTLY_ARCHIVE_REPO_TOKEN': '1', + }, { + 'source': f'{FORK_ORG}/yt-dlp-nightly-builds', + 'target': 'nightly', + 'prerelease': True, + }, { + 'channel': f'{FORK_ORG}/yt-dlp-nightly-builds', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': f'{FORK_ORG}/yt-dlp-nightly-builds', + 'target_repo_token': 'NIGHTLY_ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork w/MASTER_ARCHIVE_REPO_TOKEN, master', { + 'MASTER_ARCHIVE_REPO': f'{FORK_ORG}/yt-dlp-master-builds', + 'MASTER_PYPI_PROJECT': 'yt-dlp-test', + 'MASTER_PYPI_SUFFIX': 'dev', + }, { + 'MASTER_ARCHIVE_REPO_TOKEN': '1', + }, { + 'source': f'{FORK_ORG}/yt-dlp-master-builds', + 'target': 'master', + 'prerelease': True, + }, { + 'channel': f'{FORK_ORG}/yt-dlp-master-builds', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': f'{FORK_ORG}/yt-dlp-master-builds', + 'target_repo_token': 'MASTER_ARCHIVE_REPO_TOKEN', + 'target_tag': DEFAULT_VERSION_WITH_REVISION, + 'pypi_project': 'yt-dlp-test', + 'pypi_suffix': 'dev', + }, ignore_revision=True) + + _test( + FORK_REPOSITORY, 'fork, non-numeric tag', + {}, {}, {'source': 'experimental'}, { + 'channel': f'{FORK_REPOSITORY}@experimental', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'experimental', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + _test( + FORK_REPOSITORY, 'fork, non-numeric tag, updates to stable', + {}, {}, { + 'prerelease': True, + 'source': 'stable', + 'target': 'experimental', + }, { + 'channel': 'stable', + 'version': DEFAULT_VERSION_WITH_REVISION, + 'target_repo': FORK_REPOSITORY, + 'target_repo_token': None, + 'target_tag': 'experimental', + 'pypi_project': None, + 'pypi_suffix': None, + }, ignore_revision=True) + + print('all tests passed') + + +if __name__ == '__main__': + main() diff --git a/devscripts/update-version.py b/devscripts/update-version.py index 2018ba8440..0811e65b44 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -9,24 +9,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import argparse import contextlib -import datetime as dt import sys -from devscripts.utils import read_version, run_process, write_file - - -def get_new_version(version, revision): - if not version: - version = dt.datetime.now(dt.timezone.utc).strftime('%Y.%m.%d') - - if revision: - assert revision.isdecimal(), 'Revision must be a number' - else: - old_version = read_version().split('.') - if version.split('.') == old_version[:3]: - revision = str(int(([*old_version, 0])[3]) + 1) - - return f'{version}.{revision}' if revision else version +from devscripts.utils import calculate_version, run_process, write_file def get_git_head(): @@ -72,9 +57,7 @@ if __name__ == '__main__': args = parser.parse_args() git_head = get_git_head() - version = ( - args.version if args.version and '.' in args.version - else get_new_version(None, args.version)) + version = calculate_version(args.version) write_file(args.output, VERSION_TEMPLATE.format( version=version, git_head=git_head, channel=args.channel, origin=args.origin, package_version=f'{version}{args.suffix}')) diff --git a/devscripts/update_changelog.py b/devscripts/update_changelog.py index 36b9a8e86e..66ad17e32f 100755 --- a/devscripts/update_changelog.py +++ b/devscripts/update_changelog.py @@ -20,7 +20,9 @@ if __name__ == '__main__': '--changelog-path', type=Path, default=Path(__file__).parent.parent / 'Changelog.md', help='path to the Changelog file') args = parser.parse_args() - new_entry = create_changelog(args) header, sep, changelog = read_file(args.changelog_path).partition('\n### ') - write_file(args.changelog_path, f'{header}{sep}{read_version()}\n{new_entry}\n{sep}{changelog}') + current_version = read_version() + if current_version != changelog.splitlines()[0]: + new_entry = create_changelog(args) + write_file(args.changelog_path, f'{header}{sep}{current_version}\n{new_entry}\n{sep}{changelog}') diff --git a/devscripts/utils.py b/devscripts/utils.py index a952c9fae2..b89d01e415 100644 --- a/devscripts/utils.py +++ b/devscripts/utils.py @@ -1,5 +1,7 @@ import argparse +import datetime as dt import functools +import re import subprocess @@ -20,6 +22,23 @@ def read_version(fname='yt_dlp/version.py', varname='__version__'): return items[varname] +def calculate_version(version=None, fname='yt_dlp/version.py'): + if version and '.' in version: + return version + + revision = version + version = dt.datetime.now(dt.timezone.utc).strftime('%Y.%m.%d') + + if revision: + assert re.fullmatch(r'[0-9]+', revision), 'Revision must be numeric' + else: + old_version = read_version(fname=fname).split('.') + if version.split('.') == old_version[:3]: + revision = str(int(([*old_version, 0])[3]) + 1) + + return f'{version}.{revision}' if revision else version + + def get_filename_args(has_infile=False, default_outfile=None): parser = argparse.ArgumentParser() if has_infile: diff --git a/test/test_update.py b/test/test_update.py index b4979bc92c..980470244f 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -9,7 +9,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL, report_warning -from yt_dlp.update import UpdateInfo, Updater +from yt_dlp.update import UpdateInfo, Updater, UPDATE_SOURCES, _make_label # XXX: Keep in sync with yt_dlp.update.UPDATE_SOURCES @@ -280,6 +280,26 @@ class TestUpdate(unittest.TestCase): test('testing', None, current_commit='9' * 40) test('testing', UpdateInfo('testing', commit='9' * 40)) + def test_make_label(self): + STABLE_REPO = UPDATE_SOURCES['stable'] + NIGHTLY_REPO = UPDATE_SOURCES['nightly'] + MASTER_REPO = UPDATE_SOURCES['master'] + + for inputs, expected in [ + ([STABLE_REPO, '2025.09.02', '2025.09.02'], f'stable@2025.09.02 from {STABLE_REPO}'), + ([NIGHTLY_REPO, '2025.09.02.123456', '2025.09.02.123456'], f'nightly@2025.09.02.123456 from {NIGHTLY_REPO}'), + ([MASTER_REPO, '2025.09.02.987654', '2025.09.02.987654'], f'master@2025.09.02.987654 from {MASTER_REPO}'), + (['fork/yt-dlp', 'experimental', '2025.12.31.000000'], 'fork/yt-dlp@experimental build 2025.12.31.000000'), + (['fork/yt-dlp', '2025.09.02', '2025.09.02'], 'fork/yt-dlp@2025.09.02'), + ([STABLE_REPO, 'experimental', '2025.12.31.000000'], f'{STABLE_REPO}@experimental build 2025.12.31.000000'), + ([STABLE_REPO, 'experimental'], f'{STABLE_REPO}@experimental'), + (['fork/yt-dlp', 'experimental'], 'fork/yt-dlp@experimental'), + ]: + result = _make_label(*inputs) + self.assertEqual( + result, expected, + f'{inputs!r} returned {result!r} instead of {expected!r}') + if __name__ == '__main__': unittest.main() diff --git a/yt_dlp/update.py b/yt_dlp/update.py index a17a806432..e33be3f7b3 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -58,15 +58,28 @@ def _get_variant_and_executable_path(): """@returns (variant, executable_path)""" if getattr(sys, 'frozen', False): path = sys.executable - # py2exe is unsupported but we should still correctly identify it for debugging purposes + + # py2exe: No longer officially supported, but still identify it to block updates if not hasattr(sys, '_MEIPASS'): return 'py2exe', path - if sys._MEIPASS == os.path.dirname(path): - return f'{sys.platform}_dir', path - if sys.platform == 'darwin': + + # staticx builds: sys.executable returns a /tmp/ path + # No longer officially supported, but still identify them to block updates + # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information + if static_exe_path := os.getenv('STATICX_PROG_PATH'): + return 'linux_static_exe', static_exe_path + + # We know it's a PyInstaller bundle, but is it "onedir" or "onefile"? + suffix = 'dir' if sys._MEIPASS == os.path.dirname(path) else 'exe' + system_platform = remove_end(sys.platform, '32') + + if system_platform == 'darwin': # darwin_legacy_exe is no longer supported, but still identify it to block updates machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else '' - return f'darwin{machine}_exe', path + return f'darwin{machine}_{suffix}', path + + if system_platform == 'linux' and platform.libc_ver()[0] != 'glibc': + system_platform = 'musllinux' machine = f'_{platform.machine().lower()}' is_64bits = sys.maxsize > 2**32 @@ -77,12 +90,8 @@ def _get_variant_and_executable_path(): # See: https://github.com/yt-dlp/yt-dlp/issues/11813 elif machine[1:] == 'aarch64' and not is_64bits: machine = '_armv7l' - # sys.executable returns a /tmp/ path for staticx builds (linux_static) - # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information - if static_exe_path := os.getenv('STATICX_PROG_PATH'): - path = static_exe_path - return f'{remove_end(sys.platform, "32")}{machine}_exe', path + return f'{system_platform}{machine}_{suffix}', path path = os.path.dirname(__file__) if isinstance(__loader__, zipimporter): @@ -118,7 +127,8 @@ _FILE_SUFFIXES = { 'darwin_exe': '_macos', 'linux_exe': '_linux', 'linux_aarch64_exe': '_linux_aarch64', - 'linux_armv7l_exe': '_linux_armv7l', + 'musllinux_exe': '_musllinux', + 'musllinux_aarch64_exe': '_musllinux_aarch64', } _NON_UPDATEABLE_REASONS = { @@ -146,21 +156,6 @@ def _get_binary_name(): def _get_system_deprecation(): MIN_SUPPORTED, MIN_RECOMMENDED = (3, 9), (3, 10) - EXE_MSG_TMPL = ('Support for {} has been deprecated. ' - 'See https://github.com/yt-dlp/yt-dlp/{} for details.\n{}') - STOP_MSG = 'You may stop receiving updates on this version at any time!' - variant = detect_variant() - - # Temporary until linux_armv7l executable builds are discontinued - if variant == 'linux_armv7l_exe': - return EXE_MSG_TMPL.format( - f'{variant} (the PyInstaller-bundled executable for the Linux armv7l platform)', - 'issues/13976', STOP_MSG) - - # Temporary until linux_aarch64_exe is built with Python >=3.10 instead of Python 3.9 - if variant == 'linux_aarch64_exe': - return None - if sys.version_info > MIN_RECOMMENDED: return None @@ -199,16 +194,14 @@ def _sha256_file(path): def _make_label(origin, tag, version=None): - if '/' in origin: - channel = _INVERSE_UPDATE_SOURCES.get(origin, origin) - else: - channel = origin - label = f'{channel}@{tag}' - if version and version != tag: - label += f' build {version}' - if channel != origin: - label += f' from {origin}' - return label + if tag != version: + if version: + return f'{origin}@{tag} build {version}' + return f'{origin}@{tag}' + + if channel := _INVERSE_UPDATE_SOURCES.get(origin): + return f'{channel}@{tag} from {origin}' + return f'{origin}@{tag}' @dataclass From 7c27965ff6495f9cb8774a0264bc77b49de57207 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 22:46:00 +0000 Subject: [PATCH 016/175] Release 2025.09.05 Created by: bashonly :ci skip all --- CONTRIBUTORS | 2 ++ Changelog.md | 17 +++++++++++++++++ supportedsites.md | 2 ++ yt_dlp/version.py | 6 +++--- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 37a0e100b9..d0f475d1c1 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -806,3 +806,5 @@ junyilou PierreMesure Randalix runarmod +gitchasing +zakaryan2004 diff --git a/Changelog.md b/Changelog.md index 2ad6da30e1..120516ab2a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,23 @@ # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2025.09.05 + +#### Core changes +- [Fix `--id` deprecation warning](https://github.com/yt-dlp/yt-dlp/commit/76bb46002c9a9655f2b1d29d4840e75e79037cfa) ([#14190](https://github.com/yt-dlp/yt-dlp/issues/14190)) by [seproDev](https://github.com/seproDev) + +#### Extractor changes +- **charlierose**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/603acdff07f0226088916886002d2ad8309ff9d3) ([#14231](https://github.com/yt-dlp/yt-dlp/issues/14231)) by [gitchasing](https://github.com/gitchasing) +- **googledrive**: [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/18fe696df9d60804a8f5cb8cd74f38111d6eb711) ([#14139](https://github.com/yt-dlp/yt-dlp/issues/14139)) by [zakaryan2004](https://github.com/zakaryan2004) +- **itvbtcc**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/0b51005b4819e7cea222fcbaf8e60391db4f732c) ([#14161](https://github.com/yt-dlp/yt-dlp/issues/14161)) by [garret1317](https://github.com/garret1317) +- **kick**: vod: [Support ongoing livestream VODs](https://github.com/yt-dlp/yt-dlp/commit/1e28f6bf743627b909135bb9a88537ad2deccaf0) ([#14154](https://github.com/yt-dlp/yt-dlp/issues/14154)) by [InvalidUsernameException](https://github.com/InvalidUsernameException) +- **lrt**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/ed24640943872c4cf30d7cc4601bec87b50ba03c) ([#14193](https://github.com/yt-dlp/yt-dlp/issues/14193)) by [seproDev](https://github.com/seproDev) +- **tver**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/223baa81f6637dcdef108f817180d8d1ae9fa213) ([#14165](https://github.com/yt-dlp/yt-dlp/issues/14165)) by [arabcoders](https://github.com/arabcoders) +- **vevo**: [Restore extractors](https://github.com/yt-dlp/yt-dlp/commit/d925e92b710153d0d51d030f115b3c87226bc0f0) ([#14203](https://github.com/yt-dlp/yt-dlp/issues/14203)) by [seproDev](https://github.com/seproDev) + +#### Misc. changes +- **build**: [Overhaul Linux builds and refactor release workflow](https://github.com/yt-dlp/yt-dlp/commit/50136eeeb3767289b236f140b759f23b39b00888) ([#13997](https://github.com/yt-dlp/yt-dlp/issues/13997)) by [bashonly](https://github.com/bashonly) + ### 2025.08.27 #### Extractor changes diff --git a/supportedsites.md b/supportedsites.md index db89ccd05d..3937134378 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -1601,6 +1601,8 @@ The only reliable way to check if a site is supported is to try it. - **Vbox7** - **Veo** - **Vesti**: Вести.Ru (**Currently broken**) + - **Vevo** + - **VevoPlaylist** - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet - **vh1.com** - **vhx:embed**: [*vimeo*](## "netrc machine") diff --git a/yt_dlp/version.py b/yt_dlp/version.py index cde18db454..624271cb73 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,8 +1,8 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2025.08.27' +__version__ = '2025.09.05' -RELEASE_GIT_HEAD = '8cd37b85d492edb56a4f7506ea05527b85a6b02b' +RELEASE_GIT_HEAD = '50136eeeb3767289b236f140b759f23b39b00888' VARIANT = None @@ -12,4 +12,4 @@ CHANNEL = 'stable' ORIGIN = 'yt-dlp/yt-dlp' -_pkg_version = '2025.08.27' +_pkg_version = '2025.09.05' From cd94e7004036e0149d7d3fa236c7dd44cf460788 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 6 Sep 2025 00:41:43 -0500 Subject: [PATCH 017/175] [build] Post-release workflow cleanup (#14250) Authored by: bashonly --- .github/workflows/build.yml | 12 ++++++++---- bundle/docker/compose.yml | 8 +++++--- bundle/docker/linux/verify.sh | 6 +++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 510edb1e72..00cd946fee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -135,6 +135,7 @@ jobs: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} + UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 steps: - uses: actions/checkout@v4 with: @@ -159,7 +160,7 @@ jobs: chmod +x ./yt-dlp cp ./yt-dlp ./yt-dlp_downgraded version="$(./yt-dlp --version)" - ./yt-dlp_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + ./yt-dlp_downgraded -v --update-to "${UPDATE_TO}" downgraded_version="$(./yt-dlp_downgraded --version)" [[ "${version}" != "${downgraded_version}" ]] - name: Upload artifacts @@ -190,6 +191,7 @@ jobs: ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} EXE_NAME: ${{ matrix.exe }} + UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 steps: - uses: actions/checkout@v4 - name: Build executable @@ -215,7 +217,7 @@ jobs: mkdir -p ~/testing cp "./dist/${EXE_NAME}" ~/testing/"${EXE_NAME}_downgraded" version="$("./dist/${EXE_NAME}" --version)" - ~/testing/"${EXE_NAME}_downgraded" -v --update-to yt-dlp/yt-dlp@2023.03.04 + ~/testing/"${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}" downgraded_version="$(~/testing/"${EXE_NAME}_downgraded" --version)" [[ "${version}" != "${downgraded_version}" ]] - name: Upload artifacts @@ -333,6 +335,7 @@ jobs: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} + UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 steps: - uses: actions/checkout@v4 @@ -409,7 +412,7 @@ jobs: chmod +x ./dist/yt-dlp_macos cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded version="$(./dist/yt-dlp_macos --version)" - ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + ./dist/yt-dlp_macos_downgraded -v --update-to "${UPDATE_TO}" downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)" [[ "$version" != "$downgraded_version" ]] @@ -449,6 +452,7 @@ jobs: ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} SUFFIX: ${{ matrix.suffix }} + UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 BASE_CACHE_KEY: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }} # Use custom PyInstaller built with https://github.com/yt-dlp/Pyinstaller-builds PYINSTALLER_URL: https://yt-dlp.github.io/Pyinstaller-Builds/${{ matrix.arch }}/pyinstaller-6.15.0-py3-none-any.whl @@ -510,7 +514,7 @@ jobs: $name = "yt-dlp${Env:SUFFIX}" Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" $version = & "./dist/${name}.exe" --version - & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2025.08.20 + & "./dist/${name}_downgraded.exe" -v --update-to "${Env:UPDATE_TO}" $downgraded_version = & "./dist/${name}_downgraded.exe" --version if ($version -eq $downgraded_version) { exit 1 diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml index 1216aac5d7..77062f594a 100644 --- a/bundle/docker/compose.yml +++ b/bundle/docker/compose.yml @@ -26,6 +26,7 @@ services: VERIFYIMAGE: quay.io/pypa/manylinux2014_x86_64:latest environment: EXE_NAME: ${EXE_NAME:?} + UPDATE_TO: volumes: - ../../dist:/build @@ -55,7 +56,7 @@ services: VERIFYIMAGE: quay.io/pypa/manylinux2014_aarch64:latest environment: EXE_NAME: ${EXE_NAME:?} - SKIP_UPDATE_TO: "1" # TODO: remove when there is a glibc2.17 aarch64 release to --update-to + UPDATE_TO: volumes: - ../../dist:/build @@ -87,6 +88,7 @@ services: VERIFYIMAGE: arm32v7/debian:bullseye environment: EXE_NAME: ${EXE_NAME:?} + UPDATE_TO: TEST_ONEDIR_BUILD: "1" volumes: - ../../dist:/build @@ -117,7 +119,7 @@ services: VERIFYIMAGE: alpine:3.22 environment: EXE_NAME: ${EXE_NAME:?} - SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + UPDATE_TO: volumes: - ../../dist:/build @@ -148,6 +150,6 @@ services: VERIFYIMAGE: alpine:3.22 environment: EXE_NAME: ${EXE_NAME:?} - SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + UPDATE_TO: volumes: - ../../dist:/build diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.sh index 94de5f3e27..062a576f9d 100755 --- a/bundle/docker/linux/verify.sh +++ b/bundle/docker/linux/verify.sh @@ -35,9 +35,13 @@ if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then exit 0 fi +if [ -z "${UPDATE_TO:-}" ]; then + UPDATE_TO="yt-dlp/yt-dlp@2025.09.05" +fi + cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded" version="$("./${EXE_NAME}" --version)" -"./${EXE_NAME}_downgraded" -v --update-to yt-dlp/yt-dlp@2023.03.04 +"./${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}" downgraded_version="$("./${EXE_NAME}_downgraded" --version)" if [ "${version}" = "${downgraded_version}" ]; then exit 1 From 7c9b10ebc83907d37f9f65ea9d4bd6f5e3bd1371 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 6 Sep 2025 17:28:11 -0500 Subject: [PATCH 018/175] [ci] Test and lint workflows (#14249) Authored by: bashonly --- .github/actionlint.yml | 28 +++++++++++++++++ .github/workflows/release.yml | 2 +- .github/workflows/test-workflows.yml | 46 ++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 .github/actionlint.yml create mode 100644 .github/workflows/test-workflows.yml diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 0000000000..d6d5b9abd2 --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,28 @@ +self-hosted-runner: + labels: + # Workaround for the outdated runner list in actionlint v1.7.7 + # Ref: https://github.com/rhysd/actionlint/issues/533 + - windows-11-arm + +config-variables: + - KEEP_CACHE_WARM + - PUSH_VERSION_COMMIT + - UPDATE_TO_VERIFICATION + - PYPI_PROJECT + - PYPI_SUFFIX + - NIGHTLY_PYPI_PROJECT + - NIGHTLY_PYPI_SUFFIX + - NIGHTLY_ARCHIVE_REPO + - BUILD_NIGHTLY + - MASTER_PYPI_PROJECT + - MASTER_PYPI_SUFFIX + - MASTER_ARCHIVE_REPO + - BUILD_MASTER + - ISSUE_LOCKDOWN + - SANITIZE_COMMENT + +paths: + .github/workflows/build.yml: + ignore: + # SC1090 "Can't follow non-constant source": ignore when using `source` to activate venv + - '.+SC1090.+' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acedebd306..911bdaf6af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.10" # Keep this in sync with test-workflows.yml - name: Process inputs id: process_inputs diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml new file mode 100644 index 0000000000..5025fe8e62 --- /dev/null +++ b/.github/workflows/test-workflows.yml @@ -0,0 +1,46 @@ +name: Test and lint workflows +on: + push: + paths: + - .github/workflows/* + - devscripts/setup_variables.py + - devscripts/setup_variables_tests.py + - devscripts/utils.py + pull_request: + paths: + - .github/workflows/* + - devscripts/setup_variables.py + - devscripts/setup_variables_tests.py + - devscripts/utils.py +permissions: + contents: read +env: + ACTIONLINT_VERSION: "1.7.7" + ACTIONLINT_SHA256SUM: 023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 + ACTIONLINT_REPO: https://github.com/rhysd/actionlint + +jobs: + check: + name: Check workflows + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" # Keep this in sync with release.yml's prepare job + - name: Install requirements + env: + ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }} + run: | + sudo apt -y install shellcheck + python -m pip install -U pyflakes + curl -LO "${ACTIONLINT_REPO}/releases/download/v${ACTIONLINT_VERSION}/${ACTIONLINT_TARBALL}" + printf '%s %s' "${ACTIONLINT_SHA256SUM}" "${ACTIONLINT_TARBALL}" | sha256sum -c - + tar xvzf "${ACTIONLINT_TARBALL}" actionlint + chmod +x actionlint + - name: Run actionlint + run: | + ./actionlint -color + - name: Test GHA devscripts + run: | + python -m devscripts.setup_variables_tests From e6e6b512141e66b1b36058966804fe59c02a2b4d Mon Sep 17 00:00:00 2001 From: sepro Date: Sun, 7 Sep 2025 01:17:02 +0200 Subject: [PATCH 019/175] [docs] Clarify license of PyInstaller-bundled executables (#14257) Closes #348 Authored by: seproDev --- .github/workflows/release.yml | 11 +- README.md | 11 + THIRD_PARTY_LICENSES.txt | 4433 +++++++++++++++++++ bundle/pyinstaller.py | 1 - devscripts/generate_third_party_licenses.py | 316 ++ 5 files changed, 4765 insertions(+), 7 deletions(-) create mode 100644 THIRD_PARTY_LICENSES.txt create mode 100644 devscripts/generate_third_party_licenses.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 911bdaf6af..ab02f59bee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,12 +269,11 @@ jobs: "[![Master](https://img.shields.io/badge/Master%20builds-lightblue.svg?style=for-the-badge)]" \ "(https://github.com/${MASTER_REPO}/releases/latest \"Master builds\")" >> ./RELEASE_NOTES fi - printf '\n\n' >> ./RELEASE_NOTES - cat >> ./RELEASE_NOTES << EOF - #### A description of the various files is in the [README](https://github.com/${REPOSITORY}#release-files) - --- - $(python ./devscripts/make_changelog.py -vv --collapsible) - EOF + printf '\n\n%s\n\n%s%s\n\n---\n' \ + "#### A description of the various files is in the [README](https://github.com/${REPOSITORY}#release-files)" \ + "The PyInstaller-bundled executables are subject to the licenses described in " \ + "[THIRD_PARTY_LICENSES.txt](https://github.com/${BASE_REPO}/blob/master/THIRD_PARTY_LICENSES.txt)" >> ./RELEASE_NOTES + python ./devscripts/make_changelog.py -vv --collapsible >> ./RELEASE_NOTES printf '%s\n\n' '**This is a pre-release build**' >> ./PRERELEASE_NOTES cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES printf '%s\n\n' "Generated from: https://github.com/${REPOSITORY}/commit/${HEAD_SHA}" >> ./ARCHIVE_NOTES diff --git a/README.md b/README.md index e582e1e53c..7be3266ea5 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,17 @@ curl -L https://github.com/yt-dlp/yt-dlp/raw/master/public.key | gpg --import gpg --verify SHA2-256SUMS.sig SHA2-256SUMS gpg --verify SHA2-512SUMS.sig SHA2-512SUMS ``` + +#### Licensing + +While yt-dlp is licensed under the [Unlicense](LICENSE), many of the release files contain code from other projects with different licenses. + +Most notably, the PyInstaller-bundled executables include GPLv3+ licensed code, and as such the combined work is licensed under [GPLv3+](https://www.gnu.org/licenses/gpl-3.0.html). + +See [THIRD_PARTY_LICENSES.txt](THIRD_PARTY_LICENSES.txt) for details. + +The zipimport binary (`yt-dlp`), the source tarball (`yt-dlp.tar.gz`), and the PyPI source distribution & wheel only contain code licensed under the [Unlicense](LICENSE). + **Note**: The manpages, shell completion (autocomplete) files etc. are available inside the [source tarball](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.tar.gz) diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt new file mode 100644 index 0000000000..1040046541 --- /dev/null +++ b/THIRD_PARTY_LICENSES.txt @@ -0,0 +1,4433 @@ +THIRD-PARTY LICENSES + +This file aggregates license texts of third-party components included with the yt-dlp PyInstaller-bundled executables. +yt-dlp itself is licensed under the Unlicense (see LICENSE file). +Source code for bundled third-party components is available from the original projects. +If you cannot obtain it, the maintainers will provide it as per license obligation; maintainer emails are listed in pyproject.toml. + + +-------------------------------------------------------------------------------- +Python | PSF-2.0 +URL: https://www.python.org/ +-------------------------------------------------------------------------------- +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + + + +-------------------------------------------------------------------------------- +Microsoft Distributable Code +Note: Only included in Windows builds +-------------------------------------------------------------------------------- +Additional Conditions for this Windows binary build +--------------------------------------------------- + +This program is linked with and uses Microsoft Distributable Code, +copyrighted by Microsoft Corporation. The Microsoft Distributable Code +is embedded in each .exe, .dll and .pyd file as a result of running +the code through a linker. + +If you further distribute programs that include the Microsoft +Distributable Code, you must comply with the restrictions on +distribution specified by Microsoft. In particular, you must require +distributors and external end users to agree to terms that protect the +Microsoft Distributable Code at least as much as Microsoft's own +requirements for the Distributable Code. See Microsoft's documentation +(included in its developer tools and on its website at microsoft.com) +for specific details. + +Redistribution of the Windows binary build of the Python interpreter +complies with this agreement, provided that you do not: + +- alter any copyright, trademark or patent notice in Microsoft's +Distributable Code; + +- use Microsoft's trademarks in your programs' names or in a way that +suggests your programs come from or are endorsed by Microsoft; + +- distribute Microsoft's Distributable Code to run on a platform other +than Microsoft operating systems, run-time technologies or application +platforms; or + +- include Microsoft Distributable Code in malicious, deceptive or +unlawful programs. + +These restrictions apply only to the Microsoft Distributable Code as +defined above, not to Python itself or any programs running on the +Python interpreter. The redistribution of the Python interpreter and +libraries is governed by the Python Software License included with this +file, or by other licenses as marked. + + + +-------------------------------------------------------------------------------- +bzip2 | bzip2-1.0.6 +URL: https://sourceware.org/bzip2/ +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2010 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.6 of 6 September 2010 + +-------------------------------------------------------------------------- + + + +-------------------------------------------------------------------------------- +libffi | MIT +URL: https://sourceware.org/libffi/ +-------------------------------------------------------------------------------- +libffi - Copyright (c) 1996-2025 Anthony Green, Red Hat, Inc and others. +See source files for details. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +``Software''), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +OpenSSL 3.0+ | Apache-2.0 +URL: https://www.openssl.org/ +-------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + + +-------------------------------------------------------------------------------- +SQLite | Public Domain +URL: https://www.sqlite.org/ +-------------------------------------------------------------------------------- +License Information +=================== + +SQLite Is Public Domain +----------------------- + +The SQLite source code, including all of the files in the directories +listed in the bullets below are +[Public Domain](https://sqlite.org/copyright.html). +The authors have submitted written affidavits releasing their work to +the public for any use. Every byte of the public-domain code can be +traced back to the original authors. The files of this repository +that are public domain include the following: + + * All of the primary SQLite source code files found in the + [src/ directory](https://sqlite.org/src/tree/src?type=tree&expand) + * All of the test cases and testing code in the + [test/ directory](https://sqlite.org/src/tree/test?type=tree&expand) + * All of the SQLite extension source code and test cases in the + [ext/ directory](https://sqlite.org/src/tree/ext?type=tree&expand) + * All code that ends up in the "sqlite3.c" and "sqlite3.h" build products + that actually implement the SQLite RDBMS. + * All of the code used to compile the + [command-line interface](https://sqlite.org/cli.html) + * All of the code used to build various utility programs such as + "sqldiff", "sqlite3_rsync", and "sqlite3_analyzer". + + +The public domain source files usually contain a header comment +similar to the following to make it clear that the software is +public domain. + +> ~~~ +The author disclaims copyright to this source code. In place of +a legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. +~~~ + +Almost every file you find in this source repository will be +public domain. But there are a small number of exceptions: + +Non-Public-Domain Code Included With This Source Repository AS A Convenience +---------------------------------------------------------------------------- + +This repository contains a (relatively) small amount of non-public-domain +code used to help implement the configuration and build logic. In other +words, there are some non-public-domain files used to implement: + +> ~~~ +./configure && make +~~~ + +In all cases, the non-public-domain files included with this +repository have generous BSD-style licenses. So anyone is free to +use any of the code in this source repository for any purpose, though +attribution may be required to reuse or republish the configure and +build scripts. None of the non-public-domain code ever actually reaches +the build products, such as "sqlite3.c", however, so no attribution is +required to use SQLite itself. The non-public-domain code consists of +scripts used to help compile SQLite. The non-public-domain code is +technically not part of SQLite. The non-public-domain code is +included in this repository as a convenience to developers, so that those +who want to build SQLite do not need to go download a bunch of +third-party build scripts in order to compile SQLite. + +Non-public-domain code included in this respository includes: + + * The ["autosetup"](http://msteveb.github.io/autosetup/) configuration + system that is contained (mostly) the autosetup/ directory, but also + includes the "./configure" script at the top-level of this archive. + Autosetup has a separate BSD-style license. See the + [autosetup/LICENSE](http://msteveb.github.io/autosetup/license/) + for details. + + * There are BSD-style licenses on some of the configuration + software found in the legacy autoconf/ directory and its + subdirectories. + +The following unix shell command is can be run from the top-level +of this source repository in order to remove all non-public-domain +code: + +> ~~~ +rm -rf configure autosetup autoconf +~~~ + +If you unpack this source repository and then run the command above, what +is left will be 100% public domain. + + + +-------------------------------------------------------------------------------- +liblzma | 0BSD +URL: https://tukaani.org/xz/ +-------------------------------------------------------------------------------- +XZ Utils Licensing +================== + + Different licenses apply to different files in this package. Here + is a summary of which licenses apply to which parts of this package: + + - liblzma is under the BSD Zero Clause License (0BSD). + + - The command line tools xz, xzdec, lzmadec, and lzmainfo are + under 0BSD except that, on systems that don't have a usable + getopt_long, GNU getopt_long is compiled and linked in from the + 'lib' directory. The getopt_long code is under GNU LGPLv2.1+. + + - The scripts to grep, diff, and view compressed files have been + adapted from GNU gzip. These scripts (xzgrep, xzdiff, xzless, + and xzmore) are under GNU GPLv2+. The man pages of the scripts + are under 0BSD; they aren't based on the man pages of GNU gzip. + + - Most of the XZ Utils specific documentation that is in + plain text files (like README, INSTALL, PACKAGERS, NEWS, + and ChangeLog) are under 0BSD unless stated otherwise in + the file itself. The files xz-file-format.txt and + lzma-file-format.xt are in the public domain but may + be distributed under the terms of 0BSD too. + + - Translated messages and man pages are under 0BSD except that + some old translations are in the public domain. + + - Test files and test code in the 'tests' directory, and + debugging utilities in the 'debug' directory are under + the BSD Zero Clause License (0BSD). + + - The GNU Autotools based build system contains files that are + under GNU GPLv2+, GNU GPLv3+, and a few permissive licenses. + These files don't affect the licensing of the binaries being + built. + + - The 'extra' directory contains files that are under various + free software licenses. These aren't built or installed as + part of XZ Utils. + + The following command may be helpful in finding per-file license + information. It works on xz.git and on a clean file tree extracted + from a release tarball. + + sh build-aux/license-check.sh -v + + For the files under the BSD Zero Clause License (0BSD), if + a copyright notice is needed, the following is sufficient: + + Copyright (C) The XZ Utils authors and contributors + + If you copy significant amounts of 0BSD-licensed code from XZ Utils + into your project, acknowledging this somewhere in your software is + polite (especially if it is proprietary, non-free software), but + it is not legally required by the license terms. Here is an example + of a good notice to put into "about box" or into documentation: + + This software includes code from XZ Utils . + + The following license texts are included in the following files: + - COPYING.0BSD: BSD Zero Clause License + - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 + - COPYING.GPLv2: GNU General Public License version 2 + - COPYING.GPLv3: GNU General Public License version 3 + + If you have questions, don't hesitate to ask for more information. + The contact information is in the README file. + + + +-------------------------------------------------------------------------------- +mpdecimal | BSD-2-Clause +URL: https://www.bytereef.org/mpdecimal/ +-------------------------------------------------------------------------------- +Copyright (c) 2008-2025 Stefan Krah. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +zlib | zlib +URL: https://zlib.net/ +-------------------------------------------------------------------------------- +Copyright notice: + + (C) 1995-2024 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + +-------------------------------------------------------------------------------- +Expat | MIT +URL: https://libexpat.github.io/ +-------------------------------------------------------------------------------- +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +ncurses | X11-distribute-modifications-variant +Note: Only included in Linux/macOS builds +URL: https://invisible-island.net/ncurses/ +-------------------------------------------------------------------------------- +Copyright 2018-2022,2023 Thomas E. Dickey +Copyright 1998-2017,2018 Free Software Foundation, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, distribute with modifications, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE ABOVE COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name(s) of the above copyright +holders shall not be used in advertising or otherwise to promote the +sale, use or other dealings in this Software without prior written +authorization. + +-- vile:txtmode fc=72 +-- $Id: COPYING,v 1.12 2023/01/07 17:55:53 tom Exp $ + + + +-------------------------------------------------------------------------------- +GNU Readline | GPL-3.0-or-later +Note: Only included in Linux builds +URL: https://www.gnu.org/software/readline/ +-------------------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + + + +-------------------------------------------------------------------------------- +libstdc++ | GPL-3.0-with-GCC-exception +Note: Only included in Linux builds +URL: https://gcc.gnu.org/onlinedocs/libstdc++/ +-------------------------------------------------------------------------------- +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. + + + +-------------------------------------------------------------------------------- +libgcc | GPL-3.0-with-GCC-exception +Note: Only included in Linux builds +URL: https://gcc.gnu.org/ +-------------------------------------------------------------------------------- +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. + + + +-------------------------------------------------------------------------------- +libuuid | BSD-3-Clause +Note: Only included in Linux builds +URL: https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/tree/lib/uuid +-------------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, and the entire permission notice in its entirety, + including the disclaimer of warranties. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF +WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + + + +-------------------------------------------------------------------------------- +libintl | LGPL-2.1-or-later +Note: Only included in macOS builds +URL: https://www.gnu.org/software/gettext/ +-------------------------------------------------------------------------------- + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, see . + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Moe Ghoul, President of Vice + +That's all there is to it! + + + +-------------------------------------------------------------------------------- +libidn2 | LGPL-3.0-or-later +Note: Only included in macOS builds +URL: https://www.gnu.org/software/libidn/ +-------------------------------------------------------------------------------- + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + + + +-------------------------------------------------------------------------------- +libidn2 (Unicode character data files) | Unicode-TOU AND Unicode-DFS-2016 +Note: Only included in macOS builds +URL: https://www.gnu.org/software/libidn/ +-------------------------------------------------------------------------------- +A. Unicode Copyright. + + Copyright 1991-2016 Unicode, Inc. All rights reserved. + Certain documents and files on this website contain a legend indicating that "Modification is permitted." Any person is hereby authorized, without fee, to modify such documents and files to create derivative works conforming to the Unicode Standard, subject to Terms and Conditions herein. + Any person is hereby authorized, without fee, to view, use, reproduce, and distribute all documents and files solely for informational purposes and in the creation of products supporting the Unicode Standard, subject to the Terms and Conditions herein. + Further specifications of rights and restrictions pertaining to the use of the particular set of data files known as the "Unicode Character Database" can be found in the License. + Each version of the Unicode Standard has further specifications of rights and restrictions of use. For the book editions (Unicode 5.0 and earlier), these are found on the back of the title page. The online code charts carry specific restrictions. All other files, including online documentation of the core specification for Unicode 6.0 and later, are covered under these general Terms of Use. + No license is granted to "mirror" the Unicode website where a fee is charged for access to the "mirror" site. + Modification is not permitted with respect to this document. All copies of this document must be verbatim. + +B. Restricted Rights Legend. + Any technical data or software which is licensed to the United States of America, its agencies and/or instrumentalities under this Agreement is commercial technical data or commercial computer software developed exclusively at private expense as defined in FAR 2.101, or DFARS 252.227-7014 (June 1995), as applicable. For technical data, use, duplication, or disclosure by the Government is subject to restrictions as set forth in DFARS 202.227-7015 Technical Data, Commercial and Items (Nov 1995) and this Agreement. For Software, in accordance with FAR 12-212 or DFARS 227-7202, as applicable, use, duplication or disclosure by the Government is subject to the restrictions set forth in this Agreement. + +C. Warranties and Disclaimers. + This publication and/or website may include technical or typographical errors or other inaccuracies . Changes are periodically added to the information herein; these changes will be incorporated in new editions of the publication and/or website. Unicode may make improvements and/or changes in the product(s) and/or program(s) described in this publication and/or website at any time. + If this file has been purchased on magnetic or optical media from Unicode, Inc. the sole and exclusive remedy for any claim will be exchange of the defective media within ninety (90) days of original purchase. + EXCEPT AS PROVIDED IN SECTION C.2, THIS PUBLICATION AND/OR SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND EITHER EXPRESS, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. UNICODE AND ITS LICENSORS ASSUME NO RESPONSIBILITY FOR ERRORS OR OMISSIONS IN THIS PUBLICATION AND/OR SOFTWARE OR OTHER DOCUMENTS WHICH ARE REFERENCED BY OR LINKED TO THIS PUBLICATION OR THE UNICODE WEBSITE. + +D. Waiver of Damages. + In no event shall Unicode or its licensors be liable for any special, incidental, indirect or consequential damages of any kind, or any damages whatsoever, whether or not Unicode was advised of the possibility of the damage, including, without limitation, those resulting from the following: loss of use, data or profits, in connection with the use, modification or distribution of this information or its derivatives. + +E. Trademarks & Logos. + The Unicode Word Mark and the Unicode Logo are trademarks of Unicode, Inc. The Unicode Consortium and Unicode, Inc. are trade names of Unicode, Inc. Use of the information and materials found on this website indicates your acknowledgement of Unicode, Inc.s exclusive worldwide rights in the Unicode Word Mark, the Unicode Logo, and the Unicode trade names. + The Unicode Consortium Name and Trademark Usage Policy (Trademark Policy) are incorporated herein by reference and you agree to abide by the provisions of the Trademark Policy, which may be changed from time to time in the sole discretion of Unicode, Inc. + All third party trademarks referenced herein are the property of their respective owners. + +F. Miscellaneous. + Jurisdiction and Venue. This server is operated from a location in the State of California, United States of America. Unicode makes no representation that the materials are appropriate for use in other locations. If you access this server from other locations, you are responsible for compliance with local laws. This Agreement, all use of this site and any claims and damages resulting from use of this site are governed solely by the laws of the State of California without regard to any principles which would apply the laws of a different jurisdiction. The user agrees that any disputes regarding this site shall be resolved solely in the courts located in Santa Clara County, California. The user agrees said courts have personal jurisdiction and agree to waive any right to transfer the dispute to any other forum. + Modification by Unicode Unicode shall have the right to modify this Agreement at any time by posting it to this site. The user may not assign any part of this Agreement without Unicodes prior written consent. + Taxes. The user agrees to pay any taxes arising from access to this website or use of the information herein, except for those based on Unicodes net income. + Severability. If any provision of this Agreement is declared invalid or unenforceable, the remaining provisions of this Agreement shall remain in effect. + Entire Agreement. This Agreement constitutes the entire agreement between the parties. + + + +EXHIBIT 1 +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +Unicode Data Files do not include PDF online code charts under the +directory http://www.unicode.org/Public/. + +Software includes any source code published in the Unicode Standard +or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +NOTICE TO USER: Carefully read the following legal agreement. +BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S +DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), +YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. +IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE +THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + + + +-------------------------------------------------------------------------------- +libunistring | LGPL-3.0-or-later +Note: Only included in macOS builds +URL: https://www.gnu.org/software/libunistring/ +-------------------------------------------------------------------------------- + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + + + +-------------------------------------------------------------------------------- +librtmp | LGPL-2.1-or-later +Note: Only included in macOS builds +URL: https://rtmpdump.mplayerhq.hu/ +-------------------------------------------------------------------------------- + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + + +-------------------------------------------------------------------------------- +zstd | BSD-3-Clause +Note: Only included in macOS builds +URL: https://facebook.github.io/zstd/ +-------------------------------------------------------------------------------- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +brotli | MIT +URL: https://brotli.org/ +-------------------------------------------------------------------------------- +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +curl_cffi | MIT +Note: Not included in `yt-dlp_x86` and `yt-dlp_musllinux_aarch64` builds +URL: https://curl-cffi.readthedocs.io/ +-------------------------------------------------------------------------------- +MIT License + + +Copyright (c) 2018 multippt +Copyright (c) 2022 curl_cffi developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +curl-impersonate | MIT +Note: Not included in `yt-dlp_x86` and `yt-dlp_musllinux_aarch64` builds +URL: https://github.com/lexiforest/curl-impersonate +-------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2025 curl_cffi developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +cffi | MIT-0 +URL: https://cffi.readthedocs.io/ +-------------------------------------------------------------------------------- +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + MIT No Attribution + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +pycparser | BSD-3-Clause +URL: https://github.com/eliben/pycparser +-------------------------------------------------------------------------------- +pycparser -- A C parser in Python + +Copyright (c) 2008-2022, Eli Bendersky +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the copyright holder nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +mutagen | GPL-2.0-or-later +URL: https://mutagen.readthedocs.io/ +-------------------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + + + +-------------------------------------------------------------------------------- +PyCryptodome | Public Domain and BSD-2-Clause +URL: https://www.pycryptodome.org/ +-------------------------------------------------------------------------------- +The source code in PyCryptodome is partially in the public domain +and partially released under the BSD 2-Clause license. + +In either case, there are minimal if no restrictions on the redistribution, +modification and usage of the software. + +Public domain +============= + +All code originating from PyCrypto is free and unencumbered software +released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +BSD license +=========== + +All direct contributions to PyCryptodome are released under the following +license. The copyright of each piece belongs to the respective author. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +certifi | MPL-2.0 +URL: https://github.com/certifi/python-certifi +-------------------------------------------------------------------------------- +This package contains a modified version of ca-bundle.crt: + +ca-bundle.crt -- Bundle of CA Root Certificates + +This is a bundle of X.509 certificates of public Certificate Authorities +(CA). These were automatically extracted from Mozilla's root certificates +file (certdata.txt). This file can be found in the mozilla source tree: +https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt +It contains the certificates in PEM format and therefore +can be directly used with curl / libcurl / php_curl, or with +an Apache+mod_ssl webserver for SSL client authentication. +Just configure this file as the SSLCACertificateFile.# + +***** BEGIN LICENSE BLOCK ***** +This Source Code Form is subject to the terms of the Mozilla Public License, +v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain +one at http://mozilla.org/MPL/2.0/. + +***** END LICENSE BLOCK ***** +@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $ + + + +-------------------------------------------------------------------------------- +requests | Apache-2.0 +URL: https://requests.readthedocs.io/ +-------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + +-------------------------------------------------------------------------------- +charset-normalizer | MIT +URL: https://charset-normalizer.readthedocs.io/ +-------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2025 TAHRI Ahmed R. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +idna | BSD-3-Clause +URL: https://github.com/kjd/idna +-------------------------------------------------------------------------------- +BSD 3-Clause License + +Copyright (c) 2013-2025, Kim Davies and contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +urllib3 | MIT +URL: https://urllib3.readthedocs.io/ +-------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2008-2020 Andrey Petrov and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +SecretStorage | BSD-3-Clause +Note: Only included in Linux builds +URL: https://secretstorage.readthedocs.io/ +-------------------------------------------------------------------------------- +Copyright 2012-2018 Dmitry Shachnev +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of the University nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +cryptography | Apache-2.0 +Note: Only included in Linux builds +URL: https://cryptography.io/ +-------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +-------------------------------------------------------------------------------- +Jeepney | MIT +Note: Only included in Linux builds +URL: https://jeepney.readthedocs.io/ +-------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017 Thomas Kluyver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +websockets | BSD-3-Clause +URL: https://websockets.readthedocs.io/ +-------------------------------------------------------------------------------- +Copyright (c) Aymeric Augustin and contributors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/pyinstaller.py b/bundle/pyinstaller.py index 8286f077ab..422b9d0c74 100755 --- a/bundle/pyinstaller.py +++ b/bundle/pyinstaller.py @@ -129,7 +129,6 @@ def windows_set_version(exe, version): StringStruct('FileDescription', 'yt-dlp%s' % (MACHINE and f' ({MACHINE})')), StringStruct('FileVersion', version), StringStruct('InternalName', f'yt-dlp{suffix}'), - StringStruct('LegalCopyright', 'pukkandan.ytdlp@gmail.com | UNLICENSE'), StringStruct('OriginalFilename', f'yt-dlp{suffix}.exe'), StringStruct('ProductName', f'yt-dlp{suffix}'), StringStruct( diff --git a/devscripts/generate_third_party_licenses.py b/devscripts/generate_third_party_licenses.py new file mode 100644 index 0000000000..db615d2e35 --- /dev/null +++ b/devscripts/generate_third_party_licenses.py @@ -0,0 +1,316 @@ +import requests +from dataclasses import dataclass +from pathlib import Path +import hashlib + +DEFAULT_OUTPUT = 'THIRD_PARTY_LICENSES.txt' +CACHE_LOCATION = '.license_cache' +HEADER = '''THIRD-PARTY LICENSES + +This file aggregates license texts of third-party components included with the yt-dlp PyInstaller-bundled executables. +yt-dlp itself is licensed under the Unlicense (see LICENSE file). +Source code for bundled third-party components is available from the original projects. +If you cannot obtain it, the maintainers will provide it as per license obligation; maintainer emails are listed in pyproject.toml.''' + + +@dataclass(frozen=True) +class Dependency: + name: str + license_url: str + project_url: str = '' + license: str = '' + comment: str = '' + + +DEPENDENCIES: list[Dependency] = [ + # Core runtime environment components + Dependency( + name='Python', + license='PSF-2.0', + license_url='https://raw.githubusercontent.com/python/cpython/refs/heads/main/LICENSE', + project_url='https://www.python.org/', + ), + Dependency( + name='Microsoft Distributable Code', + license_url='https://raw.githubusercontent.com/python/cpython/refs/heads/main/PC/crtlicense.txt', + comment='Only included in Windows builds', + ), + Dependency( + name='bzip2', + license='bzip2-1.0.6', + license_url='https://gitlab.com/federicomenaquintero/bzip2/-/raw/master/COPYING', + project_url='https://sourceware.org/bzip2/', + ), + Dependency( + name='libffi', + license='MIT', + license_url='https://raw.githubusercontent.com/libffi/libffi/refs/heads/master/LICENSE', + project_url='https://sourceware.org/libffi/', + ), + Dependency( + name='OpenSSL 3.0+', + license='Apache-2.0', + license_url='https://raw.githubusercontent.com/openssl/openssl/refs/heads/master/LICENSE.txt', + project_url='https://www.openssl.org/', + ), + Dependency( + name='SQLite', + license='Public Domain', # Technically does not need to be included + license_url='https://sqlite.org/src/raw/e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d?at=LICENSE.md', + project_url='https://www.sqlite.org/', + ), + Dependency( + name='liblzma', + license='0BSD', # Technically does not need to be included + license_url='https://raw.githubusercontent.com/tukaani-project/xz/refs/heads/master/COPYING', + project_url='https://tukaani.org/xz/', + ), + Dependency( + name='mpdecimal', + license='BSD-2-Clause', + # No official repo URL + license_url='https://gist.githubusercontent.com/seproDev/9e5dbfc08af35c3f2463e64eb9b27161/raw/61f5a98bc1a4ad7d48b1c793fc3314d4d43c2ab1/mpdecimal_COPYRIGHT.txt', + project_url='https://www.bytereef.org/mpdecimal/', + ), + Dependency( + name='zlib', + license='zlib', + license_url='https://raw.githubusercontent.com/madler/zlib/refs/heads/develop/LICENSE', + project_url='https://zlib.net/', + ), + Dependency( + name='Expat', + license='MIT', + license_url='https://raw.githubusercontent.com/libexpat/libexpat/refs/heads/master/COPYING', + project_url='https://libexpat.github.io/', + ), + Dependency( + name='ncurses', + license='X11-distribute-modifications-variant', + license_url='https://raw.githubusercontent.com/mirror/ncurses/refs/heads/master/COPYING', + comment='Only included in Linux/macOS builds', + project_url='https://invisible-island.net/ncurses/', + ), + Dependency( + name='GNU Readline', + license='GPL-3.0-or-later', + license_url='https://tiswww.case.edu/php/chet/readline/COPYING', + comment='Only included in Linux builds', + project_url='https://www.gnu.org/software/readline/', + ), + Dependency( + name='libstdc++', + license='GPL-3.0-with-GCC-exception', + license_url='https://raw.githubusercontent.com/gcc-mirror/gcc/refs/heads/master/COPYING.RUNTIME', + comment='Only included in Linux builds', + project_url='https://gcc.gnu.org/onlinedocs/libstdc++/', + ), + Dependency( + name='libgcc', + license='GPL-3.0-with-GCC-exception', + license_url='https://raw.githubusercontent.com/gcc-mirror/gcc/refs/heads/master/COPYING.RUNTIME', + comment='Only included in Linux builds', + project_url='https://gcc.gnu.org/', + ), + Dependency( + name='libuuid', + license='BSD-3-Clause', + license_url='https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/plain/lib/uuid/COPYING', + comment='Only included in Linux builds', + project_url='https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/tree/lib/uuid', + ), + Dependency( + name='libintl', + license='LGPL-2.1-or-later', + license_url='https://raw.githubusercontent.com/autotools-mirror/gettext/refs/heads/master/gettext-runtime/intl/COPYING.LIB', + comment='Only included in macOS builds', + project_url='https://www.gnu.org/software/gettext/', + ), + Dependency( + name='libidn2', + license='LGPL-3.0-or-later', + license_url='https://gitlab.com/libidn/libidn2/-/raw/master/COPYING.LESSERv3', + comment='Only included in macOS builds', + project_url='https://www.gnu.org/software/libidn/', + ), + Dependency( + name='libidn2 (Unicode character data files)', + license='Unicode-TOU AND Unicode-DFS-2016', + license_url='https://gitlab.com/libidn/libidn2/-/raw/master/COPYING.unicode', + comment='Only included in macOS builds', + project_url='https://www.gnu.org/software/libidn/', + ), + Dependency( + name='libunistring', + license='LGPL-3.0-or-later', + license_url='https://gitweb.git.savannah.gnu.org/gitweb/?p=libunistring.git;a=blob_plain;f=COPYING.LIB;hb=HEAD', + comment='Only included in macOS builds', + project_url='https://www.gnu.org/software/libunistring/', + ), + Dependency( + name='librtmp', + license='LGPL-2.1-or-later', + # No official repo URL + license_url='https://gist.githubusercontent.com/seproDev/31d8c691ccddebe37b8b379307cb232d/raw/053408e98547ea8c7d9ba3a80c965f33e163b881/librtmp_COPYING.txt', + comment='Only included in macOS builds', + project_url='https://rtmpdump.mplayerhq.hu/', + ), + Dependency( + name='zstd', + license='BSD-3-Clause', + license_url='https://raw.githubusercontent.com/facebook/zstd/refs/heads/dev/LICENSE', + comment='Only included in macOS builds', + project_url='https://facebook.github.io/zstd/', + ), + + # Python packages + Dependency( + name='brotli', + license='MIT', + license_url='https://raw.githubusercontent.com/google/brotli/refs/heads/master/LICENSE', + project_url='https://brotli.org/', + ), + Dependency( + name='curl_cffi', + license='MIT', + license_url='https://raw.githubusercontent.com/lexiforest/curl_cffi/refs/heads/main/LICENSE', + comment='Not included in `yt-dlp_x86` and `yt-dlp_musllinux_aarch64` builds', + project_url='https://curl-cffi.readthedocs.io/', + ), + # Dependency of curl_cffi + Dependency( + name='curl-impersonate', + license='MIT', + license_url='https://raw.githubusercontent.com/lexiforest/curl-impersonate/refs/heads/main/LICENSE', + comment='Not included in `yt-dlp_x86` and `yt-dlp_musllinux_aarch64` builds', + project_url='https://github.com/lexiforest/curl-impersonate', + ), + Dependency( + name='cffi', + license='MIT-0', # Technically does not need to be included + license_url='https://raw.githubusercontent.com/python-cffi/cffi/refs/heads/main/LICENSE', + project_url='https://cffi.readthedocs.io/', + ), + # Dependecy of cffi + Dependency( + name='pycparser', + license='BSD-3-Clause', + license_url='https://raw.githubusercontent.com/eliben/pycparser/refs/heads/main/LICENSE', + project_url='https://github.com/eliben/pycparser', + ), + Dependency( + name='mutagen', + license='GPL-2.0-or-later', + license_url='https://raw.githubusercontent.com/quodlibet/mutagen/refs/heads/main/COPYING', + project_url='https://mutagen.readthedocs.io/', + ), + Dependency( + name='PyCryptodome', + license='Public Domain and BSD-2-Clause', + license_url='https://raw.githubusercontent.com/Legrandin/pycryptodome/refs/heads/master/LICENSE.rst', + project_url='https://www.pycryptodome.org/', + ), + Dependency( + name='certifi', + license='MPL-2.0', + license_url='https://raw.githubusercontent.com/certifi/python-certifi/refs/heads/master/LICENSE', + project_url='https://github.com/certifi/python-certifi', + ), + Dependency( + name='requests', + license='Apache-2.0', + license_url='https://raw.githubusercontent.com/psf/requests/refs/heads/main/LICENSE', + project_url='https://requests.readthedocs.io/', + ), + # Dependency of requests + Dependency( + name='charset-normalizer', + license='MIT', + license_url='https://raw.githubusercontent.com/jawah/charset_normalizer/refs/heads/master/LICENSE', + project_url='https://charset-normalizer.readthedocs.io/', + ), + # Dependency of requests + Dependency( + name='idna', + license='BSD-3-Clause', + license_url='https://raw.githubusercontent.com/kjd/idna/refs/heads/master/LICENSE.md', + project_url='https://github.com/kjd/idna', + ), + Dependency( + name='urllib3', + license='MIT', + license_url='https://raw.githubusercontent.com/urllib3/urllib3/refs/heads/main/LICENSE.txt', + project_url='https://urllib3.readthedocs.io/', + ), + Dependency( + name='SecretStorage', + license='BSD-3-Clause', + license_url='https://raw.githubusercontent.com/mitya57/secretstorage/refs/heads/master/LICENSE', + comment='Only included in Linux builds', + project_url='https://secretstorage.readthedocs.io/', + ), + # Dependency of SecretStorage + Dependency( + name='cryptography', + license='Apache-2.0', # Also available as BSD-3-Clause + license_url='https://raw.githubusercontent.com/pyca/cryptography/refs/heads/main/LICENSE.APACHE', + comment='Only included in Linux builds', + project_url='https://cryptography.io/', + ), + # Dependency of SecretStorage + Dependency( + name='Jeepney', + license='MIT', + license_url='https://gitlab.com/takluyver/jeepney/-/raw/master/LICENSE', + comment='Only included in Linux builds', + project_url='https://jeepney.readthedocs.io/', + ), + Dependency( + name='websockets', + license='BSD-3-Clause', + license_url='https://raw.githubusercontent.com/python-websockets/websockets/refs/heads/main/LICENSE', + project_url='https://websockets.readthedocs.io/', + ), +] + + +def fetch_text(dep: Dependency) -> str: + cache_dir = Path(CACHE_LOCATION) + cache_dir.mkdir(exist_ok=True) + url_hash = hashlib.sha256(dep.license_url.encode('utf-8')).hexdigest() + cache_file = cache_dir / f'{url_hash}.txt' + + if cache_file.exists(): + return cache_file.read_text() + + # UA needed since some domains block requests default UA + req = requests.get(dep.license_url, headers={'User-Agent': 'yt-dlp license fetcher'}) + req.raise_for_status() + text = req.text + cache_file.write_text(text) + return text + + +def build_output() -> str: + lines = [HEADER] + for d in DEPENDENCIES: + lines.append('\n') + lines.append('-' * 80) + header = f'{d.name}' + if d.license: + header += f' | {d.license}' + if d.comment: + header += f'\nNote: {d.comment}' + if d.project_url: + header += f'\nURL: {d.project_url}' + lines.append(header) + lines.append('-' * 80) + + text = fetch_text(d) + lines.append(text.strip('\n') + '\n') + return '\n'.join(lines) + + +if __name__ == '__main__': + content = build_output() + Path(DEFAULT_OUTPUT).write_text(content) From 6a763a55d8a93b2a964ecf7699248ad342485412 Mon Sep 17 00:00:00 2001 From: Mozi <29089388+pzhlkj6612@users.noreply.github.com> Date: Sat, 6 Sep 2025 23:48:24 +0000 Subject: [PATCH 020/175] [compat] Add `compat_datetime_from_timestamp` (#11902) Authored by: pzhlkj6612, seproDev Co-authored-by: sepro --- test/test_compat.py | 42 ++++++++++++++++++++++++++++++++++++++- test/test_utils.py | 21 ++++++++++++++++++++ yt_dlp/YoutubeDL.py | 6 +----- yt_dlp/compat/__init__.py | 8 ++++++++ yt_dlp/utils/_utils.py | 15 ++++++-------- 5 files changed, 77 insertions(+), 15 deletions(-) diff --git a/test/test_compat.py b/test/test_compat.py index 3aa9c0c518..6cc27d487c 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # Allow direct execution +import datetime as dt import os import sys import unittest @@ -12,7 +13,7 @@ import struct from yt_dlp import compat from yt_dlp.compat import urllib # isort: split -from yt_dlp.compat import compat_etree_fromstring, compat_expanduser +from yt_dlp.compat import compat_etree_fromstring, compat_expanduser, compat_datetime_from_timestamp from yt_dlp.compat.urllib.request import getproxies @@ -59,6 +60,45 @@ class TestCompat(unittest.TestCase): def test_struct_unpack(self): self.assertEqual(struct.unpack('!B', b'\x00'), (0,)) + def test_compat_datetime_from_timestamp(self): + self.assertEqual( + compat_datetime_from_timestamp(0), + dt.datetime(1970, 1, 1, 0, 0, 0, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(1), + dt.datetime(1970, 1, 1, 0, 0, 1, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(3600), + dt.datetime(1970, 1, 1, 1, 0, 0, tzinfo=dt.timezone.utc)) + + self.assertEqual( + compat_datetime_from_timestamp(-1), + dt.datetime(1969, 12, 31, 23, 59, 59, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(-86400), + dt.datetime(1969, 12, 31, 0, 0, 0, tzinfo=dt.timezone.utc)) + + self.assertEqual( + compat_datetime_from_timestamp(0.5), + dt.datetime(1970, 1, 1, 0, 0, 0, 500000, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(1.000001), + dt.datetime(1970, 1, 1, 0, 0, 1, 1, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(-1.25), + dt.datetime(1969, 12, 31, 23, 59, 58, 750000, tzinfo=dt.timezone.utc)) + + self.assertEqual( + compat_datetime_from_timestamp(-1577923200), + dt.datetime(1920, 1, 1, 0, 0, 0, tzinfo=dt.timezone.utc)) + self.assertEqual( + compat_datetime_from_timestamp(4102444800), + dt.datetime(2100, 1, 1, 0, 0, 0, tzinfo=dt.timezone.utc)) + + self.assertEqual( + compat_datetime_from_timestamp(173568960000), + dt.datetime(7470, 3, 8, 0, 0, 0, tzinfo=dt.timezone.utc)) + if __name__ == '__main__': unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index dce07c3626..9e70ad480c 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -101,11 +101,13 @@ from yt_dlp.utils import ( remove_start, render_table, replace_extension, + datetime_round, rot47, sanitize_filename, sanitize_path, sanitize_url, shell_quote, + strftime_or_none, smuggle_url, str_to_int, strip_jsonp, @@ -409,6 +411,25 @@ class TestUtil(unittest.TestCase): self.assertEqual(datetime_from_str('now+1day', precision='hour'), datetime_from_str('now+24hours', precision='auto')) self.assertEqual(datetime_from_str('now+23hours', precision='hour'), datetime_from_str('now+23hours', precision='auto')) + def test_datetime_round(self): + self.assertEqual(datetime_round(dt.datetime.strptime('1820-05-12T01:23:45Z', '%Y-%m-%dT%H:%M:%SZ')), + dt.datetime(1820, 5, 12, tzinfo=dt.timezone.utc)) + self.assertEqual(datetime_round(dt.datetime.strptime('1969-12-31T23:34:45Z', '%Y-%m-%dT%H:%M:%SZ'), 'hour'), + dt.datetime(1970, 1, 1, 0, tzinfo=dt.timezone.utc)) + self.assertEqual(datetime_round(dt.datetime.strptime('2024-12-25T01:23:45Z', '%Y-%m-%dT%H:%M:%SZ'), 'minute'), + dt.datetime(2024, 12, 25, 1, 24, tzinfo=dt.timezone.utc)) + self.assertEqual(datetime_round(dt.datetime.strptime('2024-12-25T01:23:45.123Z', '%Y-%m-%dT%H:%M:%S.%fZ'), 'second'), + dt.datetime(2024, 12, 25, 1, 23, 45, tzinfo=dt.timezone.utc)) + self.assertEqual(datetime_round(dt.datetime.strptime('2024-12-25T01:23:45.678Z', '%Y-%m-%dT%H:%M:%S.%fZ'), 'second'), + dt.datetime(2024, 12, 25, 1, 23, 46, tzinfo=dt.timezone.utc)) + + def test_strftime_or_none(self): + self.assertEqual(strftime_or_none(-4722192000), '18200512') + self.assertEqual(strftime_or_none(0), '19700101') + self.assertEqual(strftime_or_none(1735084800), '20241225') + # Throws OverflowError + self.assertEqual(strftime_or_none(1735084800000), None) + def test_daterange(self): _20century = DateRange('19000101', '20000101') self.assertFalse('17890714' in _20century) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 76a760a5a8..08a1dc493d 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -2717,11 +2717,7 @@ class YoutubeDL: ('modified_timestamp', 'modified_date'), ): if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None: - # Working around out-of-range timestamp values (e.g. negative ones on Windows, - # see http://bugs.python.org/issue1646728) - with contextlib.suppress(ValueError, OverflowError, OSError): - upload_date = dt.datetime.fromtimestamp(info_dict[ts_key], dt.timezone.utc) - info_dict[date_key] = upload_date.strftime('%Y%m%d') + info_dict[date_key] = strftime_or_none(info_dict[ts_key]) if not info_dict.get('release_year'): info_dict['release_year'] = traverse_obj(info_dict, ('release_date', {lambda x: int(x[:4])})) diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py index d779620688..ad1268143c 100644 --- a/yt_dlp/compat/__init__.py +++ b/yt_dlp/compat/__init__.py @@ -1,3 +1,4 @@ +import datetime as dt import os import xml.etree.ElementTree as etree @@ -27,6 +28,13 @@ def compat_ord(c): return c if isinstance(c, int) else ord(c) +def compat_datetime_from_timestamp(timestamp): + # Calling dt.datetime.fromtimestamp with negative timestamps throws error in Windows + # Ref: https://github.com/yt-dlp/yt-dlp/issues/5185, https://github.com/python/cpython/issues/81708, + # https://github.com/yt-dlp/yt-dlp/issues/6706#issuecomment-1496842642 + return (dt.datetime.fromtimestamp(0, dt.timezone.utc) + dt.timedelta(seconds=timestamp)) + + # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # See https://github.com/yt-dlp/yt-dlp/issues/792 # https://docs.python.org/3/library/os.path.html#os.path.expanduser diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index 3adc1d6be2..6f6d85a7fd 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -47,6 +47,7 @@ import xml.etree.ElementTree from . import traversal from ..compat import ( + compat_datetime_from_timestamp, compat_etree_fromstring, compat_expanduser, compat_HTMLParseError, @@ -1376,6 +1377,7 @@ def datetime_round(dt_, precision='day'): if precision == 'microsecond': return dt_ + time_scale = 1_000_000 unit_seconds = { 'day': 86400, 'hour': 3600, @@ -1383,8 +1385,8 @@ def datetime_round(dt_, precision='day'): 'second': 1, } roundto = lambda x, n: ((x + n / 2) // n) * n - timestamp = roundto(calendar.timegm(dt_.timetuple()), unit_seconds[precision]) - return dt.datetime.fromtimestamp(timestamp, dt.timezone.utc) + timestamp = roundto(calendar.timegm(dt_.timetuple()) + dt_.microsecond / time_scale, unit_seconds[precision]) + return compat_datetime_from_timestamp(timestamp) def hyphenate_date(date_str): @@ -2056,18 +2058,13 @@ def strftime_or_none(timestamp, date_format='%Y%m%d', default=None): datetime_object = None try: if isinstance(timestamp, (int, float)): # unix timestamp - # Using naive datetime here can break timestamp() in Windows - # Ref: https://github.com/yt-dlp/yt-dlp/issues/5185, https://github.com/python/cpython/issues/94414 - # Also, dt.datetime.fromtimestamp breaks for negative timestamps - # Ref: https://github.com/yt-dlp/yt-dlp/issues/6706#issuecomment-1496842642 - datetime_object = (dt.datetime.fromtimestamp(0, dt.timezone.utc) - + dt.timedelta(seconds=timestamp)) + datetime_object = compat_datetime_from_timestamp(timestamp) elif isinstance(timestamp, str): # assume YYYYMMDD datetime_object = dt.datetime.strptime(timestamp, '%Y%m%d') date_format = re.sub( # Support %s on windows r'(?{int(datetime_object.timestamp())}', date_format) return datetime_object.strftime(date_format) - except (ValueError, TypeError, AttributeError): + except (ValueError, TypeError, AttributeError, OverflowError, OSError): return default From 48a214bef4bfd5984362d3d24b09dce50ba449ea Mon Sep 17 00:00:00 2001 From: sepro Date: Sun, 7 Sep 2025 02:02:20 +0200 Subject: [PATCH 021/175] [build] Use SPDX license identifier (#14260) Authored by: cdce8p Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 35f81423a8..57b315feef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.27.0"] build-backend = "hatchling.build" [project] @@ -22,7 +22,8 @@ keywords = [ "sponsorblock", "yt-dlp", ] -license = {file = "LICENSE"} +license = "Unlicense" +license-files = ["LICENSE"] classifiers = [ "Topic :: Multimedia :: Video", "Development Status :: 5 - Production/Stable", @@ -37,7 +38,6 @@ classifiers = [ "Programming Language :: Python :: Implementation", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "License :: OSI Approved :: The Unlicense (Unlicense)", "Operating System :: OS Independent", ] dynamic = ["version"] @@ -63,7 +63,7 @@ secretstorage = [ ] build = [ "build", - "hatchling", + "hatchling>=1.27.0", "pip", "setuptools>=71.0.2,<81", # See https://github.com/pyinstaller/pyinstaller/issues/9149 "wheel", From 8597a4331e8535a246d777bb8397bdcab251766c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 6 Sep 2025 19:57:20 -0500 Subject: [PATCH 022/175] [build] Fix cache warmer (#14261) Fix 50136eeeb3767289b236f140b759f23b39b00888 Authored by: bashonly --- .github/workflows/build.yml | 33 +++++++++++------------------- .github/workflows/cache-warmer.yml | 1 + 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 00cd946fee..baf7a2a265 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ on: required: false default: stable type: string + origin: + required: true + type: string unix: default: true type: boolean @@ -27,10 +30,6 @@ on: windows: default: true type: boolean - origin: - required: false - default: '' - type: string secrets: GPG_SIGNING_KEY: required: false @@ -74,13 +73,6 @@ on: description: yt-dlp.exe, yt-dlp_win.zip, yt-dlp_x86.exe, yt-dlp_win_x86.zip, yt-dlp_arm64.exe, yt-dlp_win_arm64.zip default: true type: boolean - origin: - description: Origin - required: false - default: 'current repo' - type: choice - options: - - 'current repo' permissions: contents: read @@ -89,25 +81,24 @@ jobs: process: runs-on: ubuntu-latest outputs: - origin: ${{ steps.process_origin.outputs.origin }} - timestamp: ${{ steps.process_origin.outputs.timestamp }} - version: ${{ steps.process_origin.outputs.version }} + origin: ${{ steps.process_inputs.outputs.origin }} + timestamp: ${{ steps.process_inputs.outputs.timestamp }} + version: ${{ steps.process_inputs.outputs.version }} steps: - - name: Process origin - id: process_origin + - name: Process inputs + id: process_inputs env: - ORIGIN: ${{ inputs.origin }} + INPUTS: ${{ toJSON(inputs) }} REPOSITORY: ${{ github.repository }} - VERSION: ${{ inputs.version }} shell: python run: | import datetime as dt import json import os import re - origin = os.environ['ORIGIN'] + INPUTS = json.loads(os.environ['INPUTS']) timestamp = dt.datetime.now(tz=dt.timezone.utc).strftime('%Y.%m.%d.%H%M%S.%f') - version = os.getenv('VERSION') + version = INPUTS.get('version') if version and '.' not in version: # build.yml was dispatched with only a revision as the version input value version_parts = [*timestamp.split('.')[:3], version] @@ -119,7 +110,7 @@ jobs: version_parts = version.split('.') assert all(re.fullmatch(r'[0-9]+', part) for part in version_parts), 'Version must be numeric' outputs = { - 'origin': os.environ['REPOSITORY'] if origin == 'current repo' else origin, + 'origin': INPUTS.get('origin') or os.environ['REPOSITORY'], 'timestamp': timestamp, 'version': '.'.join(version_parts), } diff --git a/.github/workflows/cache-warmer.yml b/.github/workflows/cache-warmer.yml index 0b2daa8897..00ec1e1f96 100644 --- a/.github/workflows/cache-warmer.yml +++ b/.github/workflows/cache-warmer.yml @@ -12,6 +12,7 @@ jobs: with: version: '999999' channel: stable + origin: ${{ github.repository }} unix: false linux: false linux_armv7l: true From 067062bb87ac057e453ce9efdac7ca117a6a7da0 Mon Sep 17 00:00:00 2001 From: Sipherdrakon <64430430+Sipherdrakon@users.noreply.github.com> Date: Sun, 7 Sep 2025 15:09:23 -0400 Subject: [PATCH 023/175] [ie/10play] Fix extractor (#14242) Closes #14212 Authored by: Sipherdrakon --- yt_dlp/extractor/tenplay.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py index dd4ea56580..9dc1de6b78 100644 --- a/yt_dlp/extractor/tenplay.py +++ b/yt_dlp/extractor/tenplay.py @@ -2,7 +2,14 @@ import itertools from .common import InfoExtractor from ..networking import HEADRequest -from ..utils import int_or_none, traverse_obj, url_or_none, urljoin +from ..utils import ( + ExtractorError, + int_or_none, + update_url_query, + url_or_none, + urljoin, +) +from ..utils.traversal import traverse_obj class TenPlayIE(InfoExtractor): @@ -102,14 +109,19 @@ class TenPlayIE(InfoExtractor): video_data = self._download_json( f'https://vod.ten.com.au/api/videos/bcquery?command=find_videos_by_id&video_id={data["altId"]}', content_id, 'Downloading video JSON') + # Dash URL 403s, changing the m3u8 format works m3u8_url = self._request_webpage( - HEADRequest(video_data['items'][0]['HLSURL']), + HEADRequest(update_url_query(video_data['items'][0]['dashManifestUrl'], { + 'manifest': 'm3u', + })), content_id, 'Checking stream URL').url if '10play-not-in-oz' in m3u8_url: self.raise_geo_restricted(countries=['AU']) + if '10play_unsupported' in m3u8_url: + raise ExtractorError('Unable to extract stream') # Attempt to get a higher quality stream formats = self._extract_m3u8_formats( - m3u8_url.replace(',150,75,55,0000', ',300,150,75,55,0000'), + m3u8_url.replace(',150,75,55,0000', ',500,300,150,75,55,0000'), content_id, 'mp4', fatal=False) if not formats: formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4') From a183837ec8bb5e28fe6eb3a9d77ea2d0d7a106bd Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sun, 7 Sep 2025 15:43:39 -0500 Subject: [PATCH 024/175] [test:utils] Fix `sanitize_path` test for Windows CPython 3.11 (#13878) Authored by: Grub4K Co-authored-by: Simon Sawicki --- test/test_utils.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 9e70ad480c..83916b46d9 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -12,6 +12,7 @@ import datetime as dt import io import itertools import json +import ntpath import pickle import subprocess import unittest @@ -253,12 +254,6 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#') self.assertEqual(sanitize_path('C:\\abc:%(title)s.%(ext)s'), 'C:\\abc#%(title)s.%(ext)s') - # Check with nt._path_normpath if available - try: - from nt import _path_normpath as nt_path_normpath - except ImportError: - nt_path_normpath = None - for test, expected in [ ('C:\\', 'C:\\'), ('../abc', '..\\abc'), @@ -276,8 +271,7 @@ class TestUtil(unittest.TestCase): result = sanitize_path(test) assert result == expected, f'{test} was incorrectly resolved' assert result == sanitize_path(result), f'{test} changed after sanitizing again' - if nt_path_normpath: - assert result == nt_path_normpath(test), f'{test} does not match nt._path_normpath' + assert result == ntpath.normpath(test), f'{test} does not match ntpath.normpath' def test_sanitize_url(self): self.assertEqual(sanitize_url('//foo.bar'), 'http://foo.bar') From c8ede5f34d6c95c442b936bb01ecbcb724aefdef Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:44:36 -0500 Subject: [PATCH 025/175] [build] Use new PyInstaller builds for Windows (#14273) Authored by: bashonly --- .github/workflows/build.yml | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index baf7a2a265..ace208e8eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -417,6 +417,7 @@ jobs: compression-level: 0 windows: + name: windows (${{ matrix.arch }}) needs: process if: inputs.windows permissions: @@ -429,24 +430,33 @@ jobs: - arch: 'x64' runner: windows-2025 python_version: '3.10' - suffix: '' + platform_tag: win_amd64 + pyi_version: '6.15.0' + pyi_tag: '2025.09.08.215938' + pyi_hash: f70e327d849b29562caf01c30db60e6e8b2facb68124612bb53b04f96ffe6852 - arch: 'x86' runner: windows-2025 python_version: '3.10' - suffix: '_x86' + platform_tag: win32 + pyi_version: '6.15.0' + pyi_tag: '2025.09.08.215938' + pyi_hash: b9af6b49a3556d478935de2632cb7dbef41a0c226f7a8ce36efc3ca2aeab3d51 - arch: 'arm64' runner: windows-11-arm python_version: '3.13' # arm64 only has Python >= 3.11 available - suffix: '_arm64' + platform_tag: win_arm64 + pyi_version: '6.15.0' + pyi_tag: '2025.09.08.215938' + pyi_hash: 5dac9f802085432dd3135708e835ef4c08570c308d07c3ef8154495b76bf2a83 env: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} - SUFFIX: ${{ matrix.suffix }} + SUFFIX: ${{ (matrix.arch != 'x64' && format('_{0}', matrix.arch)) || '' }} UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 BASE_CACHE_KEY: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }} - # Use custom PyInstaller built with https://github.com/yt-dlp/Pyinstaller-builds - PYINSTALLER_URL: https://yt-dlp.github.io/Pyinstaller-Builds/${{ matrix.arch }}/pyinstaller-6.15.0-py3-none-any.whl + PYI_REPO: https://github.com/yt-dlp/Pyinstaller-Builds + PYI_WHEEL: pyinstaller-${{ matrix.pyi_version }}-py3-none-${{ matrix.platform_tag }}.whl steps: - uses: actions/checkout@v4 @@ -472,17 +482,23 @@ jobs: - name: Install Requirements env: ARCH: ${{ matrix.arch }} + PYI_URL: ${{ env.PYI_REPO }}/releases/download/${{ matrix.pyi_tag }}/${{ env.PYI_WHEEL }} + PYI_HASH: ${{ matrix.pyi_hash }} shell: pwsh run: | python -m venv /yt-dlp-build-venv /yt-dlp-build-venv/Scripts/Activate.ps1 + python -m pip install -U pip + # Install custom PyInstaller build and verify hash + mkdir /pyi-wheels + python -m pip download -d /pyi-wheels --no-deps --require-hashes "pyinstaller@${Env:PYI_URL}#sha256=${Env:PYI_HASH}" + python -m pip install --force-reinstall -U "/pyi-wheels/${Env:PYI_WHEEL}" python devscripts/install_deps.py -o --include build if ("${Env:ARCH}" -eq "x86") { python devscripts/install_deps.py } else { python devscripts/install_deps.py --include curl-cffi } - python -m pip install -U "${Env:PYINSTALLER_URL}" - name: Prepare shell: pwsh @@ -516,8 +532,8 @@ jobs: with: name: build-bin-${{ github.job }}-${{ matrix.arch }} path: | - dist/yt-dlp${{ matrix.suffix }}.exe - dist/yt-dlp_win${{ matrix.suffix }}.zip + dist/yt-dlp${{ env.SUFFIX }}.exe + dist/yt-dlp_win${{ env.SUFFIX }}.zip compression-level: 0 meta_files: From a1c98226a4e869a34cc764a9dcf7a4558516308e Mon Sep 17 00:00:00 2001 From: Will Smillie Date: Wed, 10 Sep 2025 14:17:24 -0400 Subject: [PATCH 026/175] [ie/xhamster] Fix extractor (#14286) Closes #14145 Authored by: nicolaasjan, willsmillie Co-authored-by: nicolaasjan <14093220+nicolaasjan@users.noreply.github.com> --- yt_dlp/extractor/xhamster.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py index 9d90f181ba..27efd43129 100644 --- a/yt_dlp/extractor/xhamster.py +++ b/yt_dlp/extractor/xhamster.py @@ -1,3 +1,4 @@ +import base64 import itertools import re @@ -12,6 +13,7 @@ from ..utils import ( int_or_none, parse_duration, str_or_none, + try_call, try_get, unified_strdate, url_or_none, @@ -229,6 +231,11 @@ class XHamsterIE(InfoExtractor): standard_url = standard_format.get(standard_format_key) if not standard_url: continue + decoded = try_call(lambda: base64.b64decode(standard_url)) + if decoded and decoded[:4] == b'xor_': + standard_url = bytes( + a ^ b for a, b in + zip(decoded[4:], itertools.cycle(b'xh7999'))).decode() standard_url = urljoin(url, standard_url) if not standard_url or standard_url in format_urls: continue From 679587dac7cd011a1472255e1f06efb017ba91b6 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:39:07 -0500 Subject: [PATCH 027/175] [ie/vimeo] Fix login error handling (#14280) Closes #14279 Authored by: bashonly --- yt_dlp/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py index ce3f441be7..67cda74292 100644 --- a/yt_dlp/extractor/vimeo.py +++ b/yt_dlp/extractor/vimeo.py @@ -151,7 +151,7 @@ class VimeoBaseInfoExtractor(InfoExtractor): 'Referer': self._LOGIN_URL, }) except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status in (405, 418): + if isinstance(e.cause, HTTPError) and e.cause.status in (404, 405, 418): raise ExtractorError( 'Unable to log in: bad username or password', expected=True) From 9def9a4b0e958285e055eb350e5dd43b5c423336 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Thu, 11 Sep 2025 06:20:03 +0900 Subject: [PATCH 028/175] [ie/newspicks] Warn when only preview is available (#14197) Closes #14137 Authored by: doe1080 --- yt_dlp/extractor/newspicks.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/newspicks.py b/yt_dlp/extractor/newspicks.py index 25be3c7203..6ebdba4282 100644 --- a/yt_dlp/extractor/newspicks.py +++ b/yt_dlp/extractor/newspicks.py @@ -50,7 +50,14 @@ class NewsPicksIE(InfoExtractor): webpage = self._download_webpage(url, video_id) fragment = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['fragment'] - m3u8_url = traverse_obj(fragment, ('movie', 'movieUrl', {url_or_none}, {require('m3u8 URL')})) + movie = fragment['movie'] + + if traverse_obj(movie, ('viewable', {str})) == 'PARTIAL_FREE' and not traverse_obj(movie, ('canWatch', {bool})): + self.report_warning( + 'Full video is for Premium members. Without cookies, ' + f'only the preview is downloaded. {self._login_hint()}', video_id) + + m3u8_url = traverse_obj(movie, ('movieUrl', {url_or_none}, {require('m3u8 URL')})) formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4') return { @@ -59,12 +66,12 @@ class NewsPicksIE(InfoExtractor): 'series': traverse_obj(fragment, ('series', 'title', {str})), 'series_id': series_id, 'subtitles': subtitles, - **traverse_obj(fragment, ('movie', { + **traverse_obj(movie, { 'title': ('title', {str}), 'cast': ('relatedUsers', ..., 'displayName', {str}, filter, all, filter), 'description': ('explanation', {clean_html}), 'release_timestamp': ('onAirStartDate', {parse_iso8601}), 'thumbnail': (('image', 'coverImageUrl'), {url_or_none}, any), 'timestamp': ('published', {parse_iso8601}), - })), + }), } From 3d9a88bd8ef149d781c7e569e48e61551eda395e Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Thu, 11 Sep 2025 06:22:10 +0900 Subject: [PATCH 029/175] [ie/pixivsketch] Remove extractors (#14196) Authored by: doe1080 --- yt_dlp/extractor/_extractors.py | 4 -- yt_dlp/extractor/pixivsketch.py | 119 -------------------------------- 2 files changed, 123 deletions(-) delete mode 100644 yt_dlp/extractor/pixivsketch.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index cb41381275..4f74a91352 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1523,10 +1523,6 @@ from .piramidetv import ( PiramideTVChannelIE, PiramideTVIE, ) -from .pixivsketch import ( - PixivSketchIE, - PixivSketchUserIE, -) from .planetmarathi import PlanetMarathiIE from .platzi import ( PlatziCourseIE, diff --git a/yt_dlp/extractor/pixivsketch.py b/yt_dlp/extractor/pixivsketch.py deleted file mode 100644 index 50b7af5354..0000000000 --- a/yt_dlp/extractor/pixivsketch.py +++ /dev/null @@ -1,119 +0,0 @@ -from .common import InfoExtractor -from ..networking.exceptions import HTTPError -from ..utils import ( - ExtractorError, - traverse_obj, - unified_timestamp, -) - - -class PixivSketchBaseIE(InfoExtractor): - def _call_api(self, video_id, path, referer, note='Downloading JSON metadata'): - response = self._download_json(f'https://sketch.pixiv.net/api/{path}', video_id, note=note, headers={ - 'Referer': referer, - 'X-Requested-With': referer, - }) - errors = traverse_obj(response, ('errors', ..., 'message')) - if errors: - raise ExtractorError(' '.join(f'{e}.' for e in errors)) - return response.get('data') or {} - - -class PixivSketchIE(PixivSketchBaseIE): - IE_NAME = 'pixiv:sketch' - _VALID_URL = r'https?://sketch\.pixiv\.net/@(?P[a-zA-Z0-9_-]+)/lives/(?P\d+)/?' - _TESTS = [{ - 'url': 'https://sketch.pixiv.net/@nuhutya/lives/3654620468641830507', - 'info_dict': { - 'id': '7370666691623196569', - 'title': 'まにあえクリスマス！', - 'uploader': 'ぬふちゃ', - 'uploader_id': 'nuhutya', - 'channel_id': '9844815', - 'age_limit': 0, - 'timestamp': 1640351536, - }, - 'skip': True, - }, { - # these two (age_limit > 0) requires you to login on website, but it's actually not required for download - 'url': 'https://sketch.pixiv.net/@namahyou/lives/4393103321546851377', - 'info_dict': { - 'id': '4907995960957946943', - 'title': 'クリスマスなんて知らん🖕', - 'uploader': 'すゃもり', - 'uploader_id': 'suya2mori2', - 'channel_id': '31169300', - 'age_limit': 15, - 'timestamp': 1640347640, - }, - 'skip': True, - }, { - 'url': 'https://sketch.pixiv.net/@8aki/lives/3553803162487249670', - 'info_dict': { - 'id': '1593420639479156945', - 'title': 'おまけ本作業(リョナ有)', - 'uploader': 'おぶい / Obui', - 'uploader_id': 'oving', - 'channel_id': '17606', - 'age_limit': 18, - 'timestamp': 1640330263, - }, - 'skip': True, - }] - - def _real_extract(self, url): - video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id') - data = self._call_api(video_id, f'lives/{video_id}.json', url) - - if not traverse_obj(data, 'is_broadcasting'): - raise ExtractorError(f'This live is offline. Use https://sketch.pixiv.net/@{uploader_id} for ongoing live.', expected=True) - - m3u8_url = traverse_obj(data, ('owner', 'hls_movie', 'url')) - formats = self._extract_m3u8_formats( - m3u8_url, video_id, ext='mp4', - entry_protocol='m3u8_native', m3u8_id='hls') - - return { - 'id': video_id, - 'title': data.get('name'), - 'formats': formats, - 'uploader': traverse_obj(data, ('user', 'name'), ('owner', 'user', 'name')), - 'uploader_id': traverse_obj(data, ('user', 'unique_name'), ('owner', 'user', 'unique_name')), - 'channel_id': str(traverse_obj(data, ('user', 'pixiv_user_id'), ('owner', 'user', 'pixiv_user_id'))), - 'age_limit': 18 if data.get('is_r18') else 15 if data.get('is_r15') else 0, - 'timestamp': unified_timestamp(data.get('created_at')), - 'is_live': True, - } - - -class PixivSketchUserIE(PixivSketchBaseIE): - IE_NAME = 'pixiv:sketch:user' - _VALID_URL = r'https?://sketch\.pixiv\.net/@(?P[a-zA-Z0-9_-]+)/?' - _TESTS = [{ - 'url': 'https://sketch.pixiv.net/@nuhutya', - 'only_matching': True, - }, { - 'url': 'https://sketch.pixiv.net/@namahyou', - 'only_matching': True, - }, { - 'url': 'https://sketch.pixiv.net/@8aki', - 'only_matching': True, - }] - - @classmethod - def suitable(cls, url): - return super().suitable(url) and not PixivSketchIE.suitable(url) - - def _real_extract(self, url): - user_id = self._match_id(url) - data = self._call_api(user_id, f'lives/users/@{user_id}.json', url) - - if not traverse_obj(data, 'is_broadcasting'): - try: - self._call_api(user_id, 'users/current.json', url, 'Investigating reason for request failure') - except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status == 401: - self.raise_login_required(f'Please log in, or use direct link like https://sketch.pixiv.net/@{user_id}/1234567890', method='cookies') - raise ExtractorError('This user is offline', expected=True) - - return self.url_result(f'https://sketch.pixiv.net/@{user_id}/lives/{data["id"]}') From 5c1abcdc49b9d23e1dcb77b95d063cf2bf93e352 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:26:27 -0500 Subject: [PATCH 030/175] [ie/tiktok:live] Fix room ID extraction (#14287) Closes #9418 Authored by: bashonly --- yt_dlp/extractor/tiktok.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py index 025e7a55d8..e165effd4e 100644 --- a/yt_dlp/extractor/tiktok.py +++ b/yt_dlp/extractor/tiktok.py @@ -1518,19 +1518,22 @@ class TikTokLiveIE(TikTokBaseIE): def _real_extract(self, url): uploader, room_id = self._match_valid_url(url).group('uploader', 'id') - webpage = self._download_webpage( - url, uploader or room_id, headers={'User-Agent': 'Mozilla/5.0'}, fatal=not room_id) + if not room_id: + webpage = self._download_webpage( + format_field(uploader, None, self._UPLOADER_URL_FORMAT), uploader) + room_id = traverse_obj( + self._get_universal_data(webpage, uploader), + ('webapp.user-detail', 'userInfo', 'user', 'roomId', {str})) - if webpage: + if not uploader or not room_id: + webpage = self._download_webpage(url, uploader or room_id, fatal=not room_id) data = self._get_sigi_state(webpage, uploader or room_id) - room_id = ( - traverse_obj(data, (( - ('LiveRoom', 'liveRoomUserInfo', 'user'), - ('UserModule', 'users', ...)), 'roomId', {str}, any)) - or self._search_regex(r'snssdk\d*://live\?room_id=(\d+)', webpage, 'room ID', default=room_id)) - uploader = uploader or traverse_obj( - data, ('LiveRoom', 'liveRoomUserInfo', 'user', 'uniqueId'), - ('UserModule', 'users', ..., 'uniqueId'), get_all=False, expected_type=str) + room_id = room_id or traverse_obj(data, (( + ('LiveRoom', 'liveRoomUserInfo', 'user'), + ('UserModule', 'users', ...)), 'roomId', {str}, any)) + uploader = uploader or traverse_obj(data, (( + ('LiveRoom', 'liveRoomUserInfo', 'user'), + ('UserModule', 'users', ...)), 'uniqueId', {str}, any)) if not room_id: raise UserNotLive(video_id=uploader) From 22ea0688ed6bcdbe4c51401a84239cda3decfc9c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:29:12 -0500 Subject: [PATCH 031/175] [ci] Bump actions/setup-python to v6 (#14282) Authored by: bashonly --- .github/workflows/build.yml | 4 ++-- .github/workflows/core.yml | 2 +- .github/workflows/download.yml | 4 ++-- .github/workflows/quick-test.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- .github/workflows/signature-tests.yml | 2 +- .github/workflows/test-workflows.yml | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ace208e8eb..8e382f9007 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,7 +131,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # Needed for changelog - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.10" - name: Install Requirements @@ -460,7 +460,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python_version }} architecture: ${{ matrix.arch }} diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 86036989c0..1f9814dba6 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -53,7 +53,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install test requirements diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 594a664c9c..c417124162 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.9 - name: Install test requirements @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install test requirements diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index 8a7b24033f..1c0fb4e4da 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.9' - name: Install test requirements @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.9' - name: Install dev dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab02f59bee..103dc0b203 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,7 +79,7 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.10" # Keep this in sync with test-workflows.yml @@ -173,7 +173,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.10" @@ -241,7 +241,7 @@ jobs: path: artifact pattern: build-* merge-multiple: true - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.10" diff --git a/.github/workflows/signature-tests.yml b/.github/workflows/signature-tests.yml index 42c65db353..4304e61dee 100644 --- a/.github/workflows/signature-tests.yml +++ b/.github/workflows/signature-tests.yml @@ -29,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install test requirements diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 5025fe8e62..36ff86e48d 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.10" # Keep this in sync with release.yml's prepare job - name: Install requirements From ba8044685537e8e14adc6826fb4d730856fd2e2b Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:18:01 -0500 Subject: [PATCH 032/175] [cleanup] Bump ruff to 0.13.x (#14293) Authored by: bashonly --- devscripts/make_contributing.py | 2 +- pyproject.toml | 2 +- test/test_InfoExtractor.py | 2 +- test/test_overwrites.py | 4 ++-- test/test_pot/test_pot_framework.py | 2 +- test/test_verbose_output.py | 8 ++++---- yt_dlp/downloader/f4m.py | 10 +++++----- yt_dlp/extractor/bilibili.py | 4 ++-- yt_dlp/extractor/brainpop.py | 2 +- yt_dlp/extractor/cnn.py | 1 + yt_dlp/extractor/fifa.py | 4 ++-- yt_dlp/extractor/nowness.py | 2 +- yt_dlp/extractor/radiofrance.py | 4 ++-- yt_dlp/extractor/rcti.py | 2 +- yt_dlp/extractor/substack.py | 4 ++-- yt_dlp/extractor/vk.py | 2 +- yt_dlp/extractor/youtube/_video.py | 2 +- yt_dlp/extractor/zingmp3.py | 4 ++-- yt_dlp/networking/_helper.py | 4 ++-- yt_dlp/utils/_utils.py | 2 +- 20 files changed, 34 insertions(+), 33 deletions(-) diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index a06f8a616e..e76cf885c8 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -8,7 +8,7 @@ def main(): return # This is unused in yt-dlp parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') - options, args = parser.parse_args() + _, args = parser.parse_args() if len(args) != 2: parser.error('Expected an input and an output filename') diff --git a/pyproject.toml b/pyproject.toml index 57b315feef..d402314bd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ dev = [ ] static-analysis = [ "autopep8~=2.0", - "ruff~=0.12.0", + "ruff~=0.13.0", ] test = [ "pytest~=8.1", diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 40dd05e136..c15dd8a617 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -1945,7 +1945,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ server_thread.daemon = True server_thread.start() - (content, urlh) = self.ie._download_webpage_handle( + content, _ = self.ie._download_webpage_handle( f'http://127.0.0.1:{port}/teapot', None, expected_status=TEAPOT_RESPONSE_STATUS) self.assertEqual(content, TEAPOT_RESPONSE_BODY) diff --git a/test/test_overwrites.py b/test/test_overwrites.py index 0beafdf12e..96a77a0081 100644 --- a/test/test_overwrites.py +++ b/test/test_overwrites.py @@ -29,7 +29,7 @@ class TestOverwrites(unittest.TestCase): '-o', 'test.webm', 'https://www.youtube.com/watch?v=jNQXAC9IVRw', ], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + sout, _ = outp.communicate() self.assertTrue(b'has already been downloaded' in sout) # if the file has no content, it has not been redownloaded self.assertTrue(os.path.getsize(download_file) < 1) @@ -41,7 +41,7 @@ class TestOverwrites(unittest.TestCase): '-o', 'test.webm', 'https://www.youtube.com/watch?v=jNQXAC9IVRw', ], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + sout, _ = outp.communicate() self.assertTrue(b'has already been downloaded' not in sout) # if the file has no content, it has not been redownloaded self.assertTrue(os.path.getsize(download_file) > 1) diff --git a/test/test_pot/test_pot_framework.py b/test/test_pot/test_pot_framework.py index bc94653f4a..d2de1dd290 100644 --- a/test/test_pot/test_pot_framework.py +++ b/test/test_pot/test_pot_framework.py @@ -153,7 +153,7 @@ class TestPoTokenProvider: with pytest.raises( PoTokenProviderRejectedRequest, - match='External requests by "example" provider do not support proxy scheme "socks4". Supported proxy ' + match=r'External requests by "example" provider do not support proxy scheme "socks4"\. Supported proxy ' 'schemes: http, socks5h', ): provider.request_pot(pot_request) diff --git a/test/test_verbose_output.py b/test/test_verbose_output.py index 21ce10a1fb..e9559d33b1 100644 --- a/test/test_verbose_output.py +++ b/test/test_verbose_output.py @@ -22,7 +22,7 @@ class TestVerboseOutput(unittest.TestCase): '--username', 'johnsmith@gmail.com', '--password', 'my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + _, serr = outp.communicate() self.assertTrue(b'--username' in serr) self.assertTrue(b'johnsmith' not in serr) self.assertTrue(b'--password' in serr) @@ -36,7 +36,7 @@ class TestVerboseOutput(unittest.TestCase): '-u', 'johnsmith@gmail.com', '-p', 'my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + _, serr = outp.communicate() self.assertTrue(b'-u' in serr) self.assertTrue(b'johnsmith' not in serr) self.assertTrue(b'-p' in serr) @@ -50,7 +50,7 @@ class TestVerboseOutput(unittest.TestCase): '--username=johnsmith@gmail.com', '--password=my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + _, serr = outp.communicate() self.assertTrue(b'--username' in serr) self.assertTrue(b'johnsmith' not in serr) self.assertTrue(b'--password' in serr) @@ -64,7 +64,7 @@ class TestVerboseOutput(unittest.TestCase): '-u=johnsmith@gmail.com', '-p=my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - sout, serr = outp.communicate() + _, serr = outp.communicate() self.assertTrue(b'-u' in serr) self.assertTrue(b'johnsmith' not in serr) self.assertTrue(b'-p' in serr) diff --git a/yt_dlp/downloader/f4m.py b/yt_dlp/downloader/f4m.py index 22d0ebd265..3c8f0265e6 100644 --- a/yt_dlp/downloader/f4m.py +++ b/yt_dlp/downloader/f4m.py @@ -149,14 +149,14 @@ class FlvReader(io.BytesIO): segments_count = self.read_unsigned_char() segments = [] for _ in range(segments_count): - box_size, box_type, box_data = self.read_box_info() + _box_size, box_type, box_data = self.read_box_info() assert box_type == b'asrt' segment = FlvReader(box_data).read_asrt() segments.append(segment) fragments_run_count = self.read_unsigned_char() fragments = [] for _ in range(fragments_run_count): - box_size, box_type, box_data = self.read_box_info() + _box_size, box_type, box_data = self.read_box_info() assert box_type == b'afrt' fragments.append(FlvReader(box_data).read_afrt()) @@ -167,7 +167,7 @@ class FlvReader(io.BytesIO): } def read_bootstrap_info(self): - total_size, box_type, box_data = self.read_box_info() + _, box_type, box_data = self.read_box_info() assert box_type == b'abst' return FlvReader(box_data).read_abst() @@ -324,9 +324,9 @@ class F4mFD(FragmentFD): if requested_bitrate is None or len(formats) == 1: # get the best format formats = sorted(formats, key=lambda f: f[0]) - rate, media = formats[-1] + _, media = formats[-1] else: - rate, media = next(filter( + _, media = next(filter( lambda f: int(f[0]) == requested_bitrate, formats)) # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec. diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py index cd9bf6b165..8675061d1a 100644 --- a/yt_dlp/extractor/bilibili.py +++ b/yt_dlp/extractor/bilibili.py @@ -1366,7 +1366,7 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE): else: yield self.url_result(f'https://www.bilibili.com/video/{entry["bvid"]}', BiliBiliIE, entry['bvid']) - metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) + _, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) return self.playlist_result(paged_list, playlist_id) @@ -1400,7 +1400,7 @@ class BilibiliSpaceAudioIE(BilibiliSpaceBaseIE): for entry in page_data.get('data') or []: yield self.url_result(f'https://www.bilibili.com/audio/au{entry["id"]}', BilibiliAudioIE, entry['id']) - metadata, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) + _, paged_list = self._extract_playlist(fetch_page, get_metadata, get_entries) return self.playlist_result(paged_list, playlist_id) diff --git a/yt_dlp/extractor/brainpop.py b/yt_dlp/extractor/brainpop.py index df10299a0c..1e4fb2c8f5 100644 --- a/yt_dlp/extractor/brainpop.py +++ b/yt_dlp/extractor/brainpop.py @@ -174,7 +174,7 @@ class BrainPOPLegacyBaseIE(BrainPOPBaseIE): } def _real_extract(self, url): - slug, display_id = self._match_valid_url(url).group('slug', 'id') + display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) topic_data = self._search_json( r'var\s+content\s*=\s*', webpage, 'content data', diff --git a/yt_dlp/extractor/cnn.py b/yt_dlp/extractor/cnn.py index 8148762c54..a601e3eb53 100644 --- a/yt_dlp/extractor/cnn.py +++ b/yt_dlp/extractor/cnn.py @@ -272,6 +272,7 @@ class CNNIndonesiaIE(InfoExtractor): return merge_dicts(json_ld_data, { '_type': 'url_transparent', 'url': embed_url, + 'id': video_id, 'upload_date': upload_date, 'tags': try_call(lambda: self._html_search_meta('keywords', webpage).split(', ')), }) diff --git a/yt_dlp/extractor/fifa.py b/yt_dlp/extractor/fifa.py index ae837f6a02..e08b114023 100644 --- a/yt_dlp/extractor/fifa.py +++ b/yt_dlp/extractor/fifa.py @@ -7,7 +7,7 @@ from ..utils import ( class FifaIE(InfoExtractor): - _VALID_URL = r'https?://www\.fifa\.com/fifaplus/(?P\w{2})/watch/([^#?]+/)?(?P\w+)' + _VALID_URL = r'https?://www\.fifa\.com/fifaplus/\w{2}/watch/([^#?]+/)?(?P\w+)' _TESTS = [{ 'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y', 'info_dict': { @@ -51,7 +51,7 @@ class FifaIE(InfoExtractor): }] def _real_extract(self, url): - video_id, locale = self._match_valid_url(url).group('id', 'locale') + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) preconnect_link = self._search_regex( diff --git a/yt_dlp/extractor/nowness.py b/yt_dlp/extractor/nowness.py index c001a82e9f..8d568aef7c 100644 --- a/yt_dlp/extractor/nowness.py +++ b/yt_dlp/extractor/nowness.py @@ -129,7 +129,7 @@ class NownessSeriesIE(NownessBaseIE): } def _real_extract(self, url): - display_id, series = self._api_request(url, 'series/getBySlug/%s') + _, series = self._api_request(url, 'series/getBySlug/%s') entries = [self._extract_url_result(post) for post in series['posts']] series_title = None series_description = None diff --git a/yt_dlp/extractor/radiofrance.py b/yt_dlp/extractor/radiofrance.py index 9d90439841..fe3fc17419 100644 --- a/yt_dlp/extractor/radiofrance.py +++ b/yt_dlp/extractor/radiofrance.py @@ -414,7 +414,7 @@ class RadioFranceProgramScheduleIE(RadioFranceBaseIE): _VALID_URL = rf'''(?x) {RadioFranceBaseIE._VALID_URL_BASE} /(?P{RadioFranceBaseIE._STATIONS_RE}) - /grille-programmes(?:\?date=(?P[\d-]+))? + /grille-programmes ''' _TESTS = [{ @@ -463,7 +463,7 @@ class RadioFranceProgramScheduleIE(RadioFranceBaseIE): })) def _real_extract(self, url): - station, date = self._match_valid_url(url).group('station', 'date') + station = self._match_valid_url(url).group('station') webpage = self._download_webpage(url, station) grid_data = self._extract_data_from_webpage(webpage, station, 'grid') upload_date = strftime_or_none(grid_data.get('date'), '%Y%m%d') diff --git a/yt_dlp/extractor/rcti.py b/yt_dlp/extractor/rcti.py index 61b73a550c..c8e57e2aba 100644 --- a/yt_dlp/extractor/rcti.py +++ b/yt_dlp/extractor/rcti.py @@ -321,7 +321,7 @@ class RCTIPlusSeriesIE(RCTIPlusBaseIE): f'Only {video_type} will be downloaded. ' f'To download everything from the series, remove "/{video_type}" from the URL') - series_meta, meta_paths = self._call_api( + series_meta, _ = self._call_api( f'https://api.rctiplus.com/api/v1/program/{series_id}/detail', display_id, 'Downloading series metadata') metadata = { 'age_limit': try_get(series_meta, lambda x: self._AGE_RATINGS[x['age_restriction'][0]['code']]), diff --git a/yt_dlp/extractor/substack.py b/yt_dlp/extractor/substack.py index f0fa00ea57..efda234fd3 100644 --- a/yt_dlp/extractor/substack.py +++ b/yt_dlp/extractor/substack.py @@ -12,7 +12,7 @@ from ..utils.traversal import traverse_obj class SubstackIE(InfoExtractor): - _VALID_URL = r'https?://(?P[\w-]+)\.substack\.com/p/(?P[\w-]+)' + _VALID_URL = r'https?://[\w-]+\.substack\.com/p/(?P[\w-]+)' _TESTS = [{ 'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r', 'md5': 'f27e4fc6252001d48d479f45e65cdfd5', @@ -116,7 +116,7 @@ class SubstackIE(InfoExtractor): return formats, subtitles def _real_extract(self, url): - display_id, username = self._match_valid_url(url).group('id', 'username') + display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) webpage_info = self._parse_json(self._search_json( diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py index 9adb3086a8..b5b8804cc6 100644 --- a/yt_dlp/extractor/vk.py +++ b/yt_dlp/extractor/vk.py @@ -723,7 +723,7 @@ class VKWallPostIE(VKBaseIE): def _unmask_url(self, mask_url, vk_id): if 'audio_api_unavailable' in mask_url: extra = mask_url.split('?extra=')[1].split('#') - func, base = self._decode(extra[1]).split(chr(11)) + _, base = self._decode(extra[1]).split(chr(11)) mask_url = list(self._decode(extra[0])) url_len = len(mask_url) indexes = [None] * url_len diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index afb1226cfa..0527368e77 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -2760,7 +2760,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if max_depth == 1 and parent: return - max_comments, max_parents, max_replies, max_replies_per_thread, *_ = ( + _max_comments, max_parents, max_replies, max_replies_per_thread, *_ = ( int_or_none(p, default=sys.maxsize) for p in self._configuration_arg('max_comments') + [''] * 4) continuation = self._extract_continuation(root_continuation_data) diff --git a/yt_dlp/extractor/zingmp3.py b/yt_dlp/extractor/zingmp3.py index 1685edb92f..c786c2f593 100644 --- a/yt_dlp/extractor/zingmp3.py +++ b/yt_dlp/extractor/zingmp3.py @@ -476,7 +476,7 @@ class ZingMp3UserIE(ZingMp3BaseIE): class ZingMp3HubIE(ZingMp3BaseIE): IE_NAME = 'zingmp3:hub' - _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?Phub)/(?P[^/]+)/(?P[^\.]+)' + _VALID_URL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?Phub)/[^/?#]+/(?P[^./?#]+)' _TESTS = [{ 'url': 'https://zingmp3.vn/hub/Nhac-Moi/IWZ9Z0CA.html', 'info_dict': { @@ -496,7 +496,7 @@ class ZingMp3HubIE(ZingMp3BaseIE): }] def _real_extract(self, url): - song_id, regions, url_type = self._match_valid_url(url).group('id', 'regions', 'type') + song_id, url_type = self._match_valid_url(url).group('id', 'type') hub_detail = self._call_api(url_type, {'id': song_id}) entries = self._parse_items(traverse_obj(hub_detail, ( 'sections', lambda _, v: v['sectionId'] == 'hub', 'items', ...))) diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py index ef9c8bafab..661a2c3b51 100644 --- a/yt_dlp/networking/_helper.py +++ b/yt_dlp/networking/_helper.py @@ -200,7 +200,7 @@ def wrap_request_errors(func): def _socket_connect(ip_addr, timeout, source_address): - af, socktype, proto, canonname, sa = ip_addr + af, socktype, proto, _canonname, sa = ip_addr sock = socket.socket(af, socktype, proto) try: if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: @@ -215,7 +215,7 @@ def _socket_connect(ip_addr, timeout, source_address): def create_socks_proxy_socket(dest_addr, proxy_args, proxy_ip_addr, timeout, source_address): - af, socktype, proto, canonname, sa = proxy_ip_addr + af, socktype, proto, _canonname, sa = proxy_ip_addr sock = sockssocket(af, socktype, proto) try: connect_proxy_args = proxy_args.copy() diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index 6f6d85a7fd..6942e7f298 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -4770,7 +4770,7 @@ def jwt_encode(payload_data, key, *, alg='HS256', headers=None): # can be extended in future to verify the signature and parse header and return the algorithm used if it's not HS256 def jwt_decode_hs256(jwt): - header_b64, payload_b64, signature_b64 = jwt.split('.') + _header_b64, payload_b64, _signature_b64 = jwt.split('.') # add trailing ='s that may have been stripped, superfluous ='s are ignored return json.loads(base64.urlsafe_b64decode(f'{payload_b64}===')) From 83b8409366d0f9554eaeae56394b244dab64a2cb Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:35:55 -0500 Subject: [PATCH 033/175] [ci] Test with Python 3.14 (#13468) Authored by: bashonly --- .github/workflows/core.yml | 6 +++++- .github/workflows/download.yml | 2 +- .github/workflows/signature-tests.yml | 2 +- pyproject.toml | 4 +++- setup.cfg | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 1f9814dba6..93442529f5 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -37,17 +37,21 @@ jobs: matrix: os: [ubuntu-latest] # CPython 3.9 is in quick-test - python-version: ['3.10', '3.11', '3.12', '3.13', pypy-3.11] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14-dev', pypy-3.11] include: # atleast one of each CPython/PyPy tests must be in windows - os: windows-latest python-version: '3.9' - os: windows-latest python-version: '3.10' + - os: windows-latest + python-version: '3.11' - os: windows-latest python-version: '3.12' - os: windows-latest python-version: '3.13' + - os: windows-latest + python-version: '3.14-dev' - os: windows-latest python-version: pypy-3.11 steps: diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index c417124162..8dbfee6f88 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -28,7 +28,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - python-version: ['3.10', '3.11', '3.12', '3.13', pypy-3.11] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14-dev', pypy-3.11] include: # atleast one of each CPython/PyPy tests must be in windows - os: windows-latest diff --git a/.github/workflows/signature-tests.yml b/.github/workflows/signature-tests.yml index 4304e61dee..ae2221d28a 100644 --- a/.github/workflows/signature-tests.yml +++ b/.github/workflows/signature-tests.yml @@ -25,7 +25,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.11] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14-dev', pypy-3.11] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index d402314bd3..9b4ff20ba1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", @@ -173,7 +174,8 @@ python = [ "3.11", "3.12", "3.13", - "pypy3.10", + "3.14", + "pypy3.11", ] [tool.ruff] diff --git a/setup.cfg b/setup.cfg index a556eb29f5..d6541c8516 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ remove-unused-variables = true [tox:tox] skipsdist = true -envlist = py{39,310,311,312,313},pypy311 +envlist = py{39,310,311,312,313,314},pypy311 skip_missing_interpreters = true [testenv] # tox From f5cb721185e8725cf4eb4080e86aa9aa73ef25b3 Mon Sep 17 00:00:00 2001 From: sepro Date: Thu, 11 Sep 2025 21:32:35 +0200 Subject: [PATCH 034/175] [ie/loco] Fix extractor (#14256) Closes #14255 Authored by: seproDev --- yt_dlp/extractor/loco.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/loco.py b/yt_dlp/extractor/loco.py index 6c9a255678..c3d4434d90 100644 --- a/yt_dlp/extractor/loco.py +++ b/yt_dlp/extractor/loco.py @@ -37,7 +37,7 @@ class LocoIE(InfoExtractor): }, }, { 'url': 'https://loco.com/stream/c64916eb-10fb-46a9-9a19-8c4b7ed064e7', - 'md5': '45ebc8a47ee1c2240178757caf8881b5', + 'md5': '8b9bda03eba4d066928ae8d71f19befb', 'info_dict': { 'id': 'c64916eb-10fb-46a9-9a19-8c4b7ed064e7', 'ext': 'mp4', @@ -55,9 +55,9 @@ class LocoIE(InfoExtractor): 'tags': ['Gameplay'], 'series': 'GTA 5', 'timestamp': 1740612872, - 'modified_timestamp': 1740613037, + 'modified_timestamp': 1750948439, 'upload_date': '20250226', - 'modified_date': '20250226', + 'modified_date': '20250626', }, }, { # Requires video authorization @@ -123,8 +123,8 @@ class LocoIE(InfoExtractor): def _real_extract(self, url): video_type, video_id = self._match_valid_url(url).group('type', 'id') webpage = self._download_webpage(url, video_id) - stream = traverse_obj(self._search_nextjs_data(webpage, video_id), ( - 'props', 'pageProps', ('liveStreamData', 'stream', 'liveStream'), {dict}, any, {require('stream info')})) + stream = traverse_obj(self._search_nextjs_v13_data(webpage, video_id), ( + ..., (None, 'ssrData'), ('liveStreamData', 'stream', 'liveStream'), {dict}, any, {require('stream info')})) if access_token := self._get_access_token(video_id): self._request_webpage( From 7d9e48b22a780c2e8d2d2d68940d49fd2029ab70 Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Fri, 12 Sep 2025 05:42:01 +0900 Subject: [PATCH 035/175] [ie/tunein] Fix extractors (#13981) Authored by: doe1080 --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/tunein.py | 439 +++++++++++++++++++------------- 2 files changed, 266 insertions(+), 174 deletions(-) diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 4f74a91352..f08a1aaab4 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2149,6 +2149,7 @@ from .tubitv import ( ) from .tumblr import TumblrIE from .tunein import ( + TuneInEmbedIE, TuneInPodcastEpisodeIE, TuneInPodcastIE, TuneInShortenerIE, diff --git a/yt_dlp/extractor/tunein.py b/yt_dlp/extractor/tunein.py index 90fb04bf3d..b5743e8e82 100644 --- a/yt_dlp/extractor/tunein.py +++ b/yt_dlp/extractor/tunein.py @@ -1,244 +1,335 @@ +import functools import urllib.parse from .common import InfoExtractor from ..utils import ( OnDemandPagedList, - determine_ext, + UnsupportedError, + clean_html, + int_or_none, + join_nonempty, parse_iso8601, - traverse_obj, + update_url_query, + url_or_none, ) +from ..utils.traversal import traverse_obj class TuneInBaseIE(InfoExtractor): - _VALID_URL_BASE = r'https?://(?:www\.)?tunein\.com' - - def _extract_metadata(self, webpage, content_id): - return self._search_json(r'window.INITIAL_STATE=', webpage, 'hydration', content_id, fatal=False) + def _call_api(self, item_id, endpoint=None, note='Downloading JSON metadata', fatal=False, query=None): + return self._download_json( + join_nonempty('https://api.tunein.com/profiles', item_id, endpoint, delim='/'), + item_id, note=note, fatal=fatal, query=query) or {} def _extract_formats_and_subtitles(self, content_id): streams = self._download_json( - f'https://opml.radiotime.com/Tune.ashx?render=json&formats=mp3,aac,ogg,flash,hls&id={content_id}', - content_id)['body'] + 'https://opml.radiotime.com/Tune.ashx', content_id, query={ + 'formats': 'mp3,aac,ogg,flash,hls', + 'id': content_id, + 'render': 'json', + }) formats, subtitles = [], {} - for stream in streams: + for stream in traverse_obj(streams, ('body', lambda _, v: url_or_none(v['url']))): if stream.get('media_type') == 'hls': fmts, subs = self._extract_m3u8_formats_and_subtitles(stream['url'], content_id, fatal=False) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) - elif determine_ext(stream['url']) == 'pls': - playlist_content = self._download_webpage(stream['url'], content_id) - formats.append({ - 'url': self._search_regex(r'File1=(.*)', playlist_content, 'url', fatal=False), - 'abr': stream.get('bitrate'), - 'ext': stream.get('media_type'), - }) else: - formats.append({ - 'url': stream['url'], - 'abr': stream.get('bitrate'), - 'ext': stream.get('media_type'), - }) + formats.append(traverse_obj(stream, { + 'abr': ('bitrate', {int_or_none}), + 'ext': ('media_type', {str}), + 'url': ('url', {self._proto_relative_url}), + })) return formats, subtitles class TuneInStationIE(TuneInBaseIE): - _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'(?:/radio/[^?#]+-|/embed/player/)(?Ps\d+)' - _EMBED_REGEX = [r']+src=["\'](?P(?:https?://)?tunein\.com/embed/player/s\d+)'] - + IE_NAME = 'tunein:station' + _VALID_URL = r'https?://tunein\.com/radio/[^/?#]+(?Ps\d+)' _TESTS = [{ 'url': 'https://tunein.com/radio/Jazz24-885-s34682/', 'info_dict': { 'id': 's34682', - 'title': str, - 'description': 'md5:d6d0b89063fd68d529fa7058ee98619b', - 'thumbnail': r're:https?://cdn-profiles\.tunein\.com/.+', - 'location': 'Seattle-Tacoma, US', 'ext': 'mp3', + 'title': str, + 'alt_title': 'World Class Jazz', + 'channel_follower_count': int, + 'description': 'md5:d6d0b89063fd68d529fa7058ee98619b', + 'location': r're:Seattle-Tacoma, (?:US|WA)', 'live_status': 'is_live', + 'thumbnail': r're:https?://.+', }, - 'params': { - 'skip_download': True, - }, - }, { - 'url': 'https://tunein.com/embed/player/s6404/', - 'only_matching': True, + 'params': {'skip_download': 'Livestream'}, }, { 'url': 'https://tunein.com/radio/BBC-Radio-1-988-s24939/', 'info_dict': { 'id': 's24939', - 'title': str, - 'description': 'md5:ee2c56794844610d045f8caf5ff34d0c', - 'thumbnail': r're:https?://cdn-profiles\.tunein\.com/.+', - 'location': 'London, UK', 'ext': 'm4a', + 'title': str, + 'alt_title': 'The biggest new pop and all-day vibes', + 'channel_follower_count': int, + 'description': 'md5:ee2c56794844610d045f8caf5ff34d0c', + 'location': 'London, UK', 'live_status': 'is_live', + 'thumbnail': r're:https?://.+', }, - 'params': { - 'skip_download': True, + 'params': {'skip_download': 'Livestream'}, + }] + + def _real_extract(self, url): + station_id = self._match_id(url) + formats, subtitles = self._extract_formats_and_subtitles(station_id) + + return { + 'id': station_id, + 'formats': formats, + 'subtitles': subtitles, + **traverse_obj(self._call_api(station_id), ('Item', { + 'title': ('Title', {clean_html}), + 'alt_title': ('Subtitle', {clean_html}, filter), + 'channel_follower_count': ('Actions', 'Follow', 'FollowerCount', {int_or_none}), + 'description': ('Description', {clean_html}, filter), + 'is_live': ('Actions', 'Play', 'IsLive', {bool}), + 'location': ('Properties', 'Location', 'DisplayName', {str}), + 'thumbnail': ('Image', {url_or_none}), + })), + } + + +class TuneInPodcastIE(TuneInBaseIE): + IE_NAME = 'tunein:podcast:program' + _PAGE_SIZE = 20 + _VALID_URL = r'https?://tunein\.com/podcasts(?:/[^/?#]+){1,2}(?Pp\d+)' + _TESTS = [{ + 'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/', + 'info_dict': { + 'id': 'p1153019', + 'title': 'Lex Fridman Podcast', }, + 'playlist_mincount': 200, + }, { + 'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/', + 'info_dict': { + 'id': 'p14', + 'title': 'BBC News', + }, + 'playlist_mincount': 35, + }] + + @classmethod + def suitable(cls, url): + return False if TuneInPodcastEpisodeIE.suitable(url) else super().suitable(url) + + def _fetch_page(self, url, podcast_id, page=0): + items = self._call_api( + podcast_id, 'contents', f'Downloading page {page + 1}', query={ + 'filter': 't:free', + 'limit': self._PAGE_SIZE, + 'offset': page * self._PAGE_SIZE, + }, + )['Items'] + + for item in traverse_obj(items, (..., 'GuideId', {str}, filter)): + yield self.url_result(update_url_query(url, {'topicId': item[1:]})) + + def _real_extract(self, url): + podcast_id = self._match_id(url) + + return self.playlist_result(OnDemandPagedList( + functools.partial(self._fetch_page, url, podcast_id), self._PAGE_SIZE), + podcast_id, traverse_obj(self._call_api(podcast_id), ('Item', 'Title', {str}))) + + +class TuneInPodcastEpisodeIE(TuneInBaseIE): + IE_NAME = 'tunein:podcast' + _VALID_URL = r'https?://tunein\.com/podcasts(?:/[^/?#]+){1,2}(?Pp\d+)/?\?(?:[^#]+&)?(?i:topicid)=(?P\d+)' + _TESTS = [{ + 'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354', + 'info_dict': { + 'id': 't236404354', + 'ext': 'mp3', + 'title': '#351 – MrBeast: Future of YouTube, Twitter, TikTok, and Instagram', + 'alt_title': 'Technology Podcasts >', + 'cast': 'count:1', + 'description': 'md5:1029895354ef073ff00f20b82eb6eb71', + 'display_id': '236404354', + 'duration': 8330, + 'thumbnail': r're:https?://.+', + 'timestamp': 1673458571, + 'upload_date': '20230111', + 'series': 'Lex Fridman Podcast', + 'series_id': 'p1153019', + }, + }, { + 'url': 'https://tunein.com/podcasts/The-BOB--TOM-Show-Free-Podcast-p20069/?topicId=174556405', + 'info_dict': { + 'id': 't174556405', + 'ext': 'mp3', + 'title': 'B&T Extra: Ohhh Yeah, It\'s Sexy Time', + 'alt_title': 'Westwood One >', + 'cast': 'count:2', + 'description': 'md5:6828234f410ab88c85655495c5fcfa88', + 'display_id': '174556405', + 'duration': 1203, + 'series': 'The BOB & TOM Show Free Podcast', + 'series_id': 'p20069', + 'thumbnail': r're:https?://.+', + 'timestamp': 1661799600, + 'upload_date': '20220829', + }, + }] + + def _real_extract(self, url): + series_id, display_id = self._match_valid_url(url).group('series_id', 'id') + episode_id = f't{display_id}' + formats, subtitles = self._extract_formats_and_subtitles(episode_id) + + return { + 'id': episode_id, + 'display_id': display_id, + 'formats': formats, + 'series': traverse_obj(self._call_api(series_id), ('Item', 'Title', {clean_html})), + 'series_id': series_id, + 'subtitles': subtitles, + **traverse_obj(self._call_api(episode_id), ('Item', { + 'title': ('Title', {clean_html}), + 'alt_title': ('Subtitle', {clean_html}, filter), + 'cast': ( + 'Properties', 'ParentProgram', 'Hosts', {clean_html}, + {lambda x: x.split(';')}, ..., {str.strip}, filter, all, filter), + 'description': ('Description', {clean_html}, filter), + 'duration': ('Actions', 'Play', 'Duration', {int_or_none}), + 'thumbnail': ('Image', {url_or_none}), + 'timestamp': ('Actions', 'Play', 'PublishTime', {parse_iso8601}), + })), + } + + +class TuneInEmbedIE(TuneInBaseIE): + IE_NAME = 'tunein:embed' + _VALID_URL = r'https?://tunein\.com/embed/player/(?P[^/?#]+)' + _EMBED_REGEX = [r']+\bsrc=["\'](?P(?:https?:)?//tunein\.com/embed/player/[^/?#"\']+)'] + _TESTS = [{ + 'url': 'https://tunein.com/embed/player/s6404/', + 'info_dict': { + 'id': 's6404', + 'ext': 'mp3', + 'title': str, + 'alt_title': 'South Africa\'s News and Information Leader', + 'channel_follower_count': int, + 'live_status': 'is_live', + 'location': 'Johannesburg, South Africa', + 'thumbnail': r're:https?://.+', + }, + 'params': {'skip_download': 'Livestream'}, + }, { + 'url': 'https://tunein.com/embed/player/t236404354/', + 'info_dict': { + 'id': 't236404354', + 'ext': 'mp3', + 'title': '#351 – MrBeast: Future of YouTube, Twitter, TikTok, and Instagram', + 'alt_title': 'Technology Podcasts >', + 'cast': 'count:1', + 'description': 'md5:1029895354ef073ff00f20b82eb6eb71', + 'display_id': '236404354', + 'duration': 8330, + 'series': 'Lex Fridman Podcast', + 'series_id': 'p1153019', + 'thumbnail': r're:https?://.+', + 'timestamp': 1673458571, + 'upload_date': '20230111', + }, + }, { + 'url': 'https://tunein.com/embed/player/p191660/', + 'info_dict': { + 'id': 'p191660', + 'title': 'SBS Tamil', + }, + 'playlist_mincount': 195, }] _WEBPAGE_TESTS = [{ 'url': 'https://www.martiniinthemorning.com/', 'info_dict': { 'id': 's55412', 'ext': 'mp3', - 'title': 'TuneInStation video #s55412', + 'title': str, + 'alt_title': 'Now that\'s music!', + 'channel_follower_count': int, + 'description': 'md5:41588a3e2cf34b3eafc6c33522fa611a', + 'live_status': 'is_live', + 'location': 'US', + 'thumbnail': r're:https?://.+', }, - 'expected_warnings': ['unable to extract hydration', 'Extractor failed to obtain "title"'], + 'params': {'skip_download': 'Livestream'}, }] def _real_extract(self, url): - station_id = self._match_id(url) + embed_id = self._match_id(url) + kind = { + 'p': 'program', + 's': 'station', + 't': 'topic', + }.get(embed_id[:1]) - webpage = self._download_webpage(url, station_id) - metadata = self._extract_metadata(webpage, station_id) - - formats, subtitles = self._extract_formats_and_subtitles(station_id) - return { - 'id': station_id, - 'title': traverse_obj(metadata, ('profiles', station_id, 'title')), - 'description': traverse_obj(metadata, ('profiles', station_id, 'description')), - 'thumbnail': traverse_obj(metadata, ('profiles', station_id, 'image')), - 'timestamp': parse_iso8601( - traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'publishTime'))), - 'location': traverse_obj( - metadata, ('profiles', station_id, 'metadata', 'properties', 'location', 'displayName'), - ('profiles', station_id, 'properties', 'location', 'displayName')), - 'formats': formats, - 'subtitles': subtitles, - 'is_live': traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'isLive')), - } - - -class TuneInPodcastIE(TuneInBaseIE): - _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/(?:podcasts/[^?#]+-|embed/player/)(?Pp\d+)/?(?:#|$)' - _EMBED_REGEX = [r']+src=["\'](?P(?:https?://)?tunein\.com/embed/player/p\d+)'] - - _TESTS = [{ - 'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019', - 'info_dict': { - 'id': 'p1153019', - 'title': 'Lex Fridman Podcast', - 'description': 'md5:bedc4e5f1c94f7dec6e4317b5654b00d', - }, - 'playlist_mincount': 200, - }, { - 'url': 'https://tunein.com/embed/player/p191660/', - 'only_matching': True, - }, { - 'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/', - 'info_dict': { - 'id': 'p14', - 'title': 'BBC News', - 'description': 'md5:30b9622bcc4bd101d4acd6f38f284aed', - }, - 'playlist_mincount': 36, - }] - - _PAGE_SIZE = 30 - - def _real_extract(self, url): - podcast_id = self._match_id(url) - - webpage = self._download_webpage(url, podcast_id, fatal=False) - metadata = self._extract_metadata(webpage, podcast_id) - - def page_func(page_num): - api_response = self._download_json( - f'https://api.tunein.com/profiles/{podcast_id}/contents', podcast_id, - note=f'Downloading page {page_num + 1}', query={ - 'filter': 't:free', - 'offset': page_num * self._PAGE_SIZE, - 'limit': self._PAGE_SIZE, - }) - - return [ - self.url_result( - f'https://tunein.com/podcasts/{podcast_id}?topicId={episode["GuideId"][1:]}', - TuneInPodcastEpisodeIE, title=episode.get('Title')) - for episode in api_response['Items']] - - entries = OnDemandPagedList(page_func, self._PAGE_SIZE) - return self.playlist_result( - entries, playlist_id=podcast_id, title=traverse_obj(metadata, ('profiles', podcast_id, 'title')), - description=traverse_obj(metadata, ('profiles', podcast_id, 'description'))) - - -class TuneInPodcastEpisodeIE(TuneInBaseIE): - _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/podcasts/(?:[^?&]+-)?(?Pp\d+)/?\?topicId=(?P\w\d+)' - - _TESTS = [{ - 'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354', - 'info_dict': { - 'id': 't236404354', - 'title': '#351 – MrBeast: Future of YouTube, Twitter, TikTok, and Instagram', - 'description': 'md5:2784533b98f8ac45c0820b1e4a8d8bb2', - 'thumbnail': r're:https?://cdn-profiles\.tunein\.com/.+', - 'timestamp': 1673458571, - 'upload_date': '20230111', - 'series_id': 'p1153019', - 'series': 'Lex Fridman Podcast', - 'ext': 'mp3', - }, - }] - - def _real_extract(self, url): - podcast_id, episode_id = self._match_valid_url(url).group('podcast_id', 'id') - episode_id = f't{episode_id}' - - webpage = self._download_webpage(url, episode_id) - metadata = self._extract_metadata(webpage, episode_id) - - formats, subtitles = self._extract_formats_and_subtitles(episode_id) - return { - 'id': episode_id, - 'title': traverse_obj(metadata, ('profiles', episode_id, 'title')), - 'description': traverse_obj(metadata, ('profiles', episode_id, 'description')), - 'thumbnail': traverse_obj(metadata, ('profiles', episode_id, 'image')), - 'timestamp': parse_iso8601( - traverse_obj(metadata, ('profiles', episode_id, 'actions', 'play', 'publishTime'))), - 'series_id': podcast_id, - 'series': traverse_obj(metadata, ('profiles', podcast_id, 'title')), - 'formats': formats, - 'subtitles': subtitles, - } + return self.url_result( + f'https://tunein.com/{kind}/?{kind}id={embed_id[1:]}') class TuneInShortenerIE(InfoExtractor): - _WORKING = False IE_NAME = 'tunein:shortener' IE_DESC = False # Do not list - _VALID_URL = r'https?://tun\.in/(?P[A-Za-z0-9]+)' - + _VALID_URL = r'https?://tun\.in/(?P[^/?#]+)' _TESTS = [{ - # test redirection 'url': 'http://tun.in/ser7s', 'info_dict': { 'id': 's34682', 'title': str, - 'description': 'md5:d6d0b89063fd68d529fa7058ee98619b', - 'thumbnail': r're:https?://cdn-profiles\.tunein\.com/.+', - 'location': 'Seattle-Tacoma, US', 'ext': 'mp3', + 'alt_title': 'World Class Jazz', + 'channel_follower_count': int, + 'description': 'md5:d6d0b89063fd68d529fa7058ee98619b', + 'location': r're:Seattle-Tacoma, (?:US|WA)', 'live_status': 'is_live', + 'thumbnail': r're:https?://.+', }, - 'params': { - 'skip_download': True, # live stream + 'params': {'skip_download': 'Livestream'}, + }, { + 'url': 'http://tun.in/tqeeFw', + 'info_dict': { + 'id': 't236404354', + 'title': str, + 'ext': 'mp3', + 'alt_title': 'Technology Podcasts >', + 'cast': 'count:1', + 'description': 'md5:1029895354ef073ff00f20b82eb6eb71', + 'display_id': '236404354', + 'duration': 8330, + 'series': 'Lex Fridman Podcast', + 'series_id': 'p1153019', + 'thumbnail': r're:https?://.+', + 'timestamp': 1673458571, + 'upload_date': '20230111', }, + 'params': {'skip_download': 'Livestream'}, + }, { + 'url': 'http://tun.in/pei6i', + 'info_dict': { + 'id': 'p14', + 'title': 'BBC News', + }, + 'playlist_mincount': 35, }] def _real_extract(self, url): redirect_id = self._match_id(url) # The server doesn't support HEAD requests - urlh = self._request_webpage( - url, redirect_id, note='Downloading redirect page') - - url = urlh.url - url_parsed = urllib.parse.urlparse(url) - if url_parsed.port == 443: - url = url_parsed._replace(netloc=url_parsed.hostname).url - - self.to_screen(f'Following redirect: {url}') - return self.url_result(url) + urlh = self._request_webpage(url, redirect_id, 'Downloading redirect page') + # Need to strip port from URL + parsed = urllib.parse.urlparse(urlh.url) + new_url = parsed._replace(netloc=parsed.hostname).geturl() + # Prevent infinite loop in case redirect fails + if self.suitable(new_url): + raise UnsupportedError(new_url) + return self.url_result(new_url) From 8cb037c0b06c2815080f87d61ea2e95c412785fc Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Fri, 12 Sep 2025 05:59:54 +0900 Subject: [PATCH 036/175] [ie/smotrim] Rework extractors (#14200) Closes #9372, Closes #11804, Closes #13900 Authored by: doe1080, swayll Co-authored-by: Nikolay Fedorov <40500428+swayll@users.noreply.github.com> --- yt_dlp/extractor/_extractors.py | 9 +- yt_dlp/extractor/rutv.py | 191 -------------- yt_dlp/extractor/smotrim.py | 428 ++++++++++++++++++++++++++++---- yt_dlp/extractor/vesti.py | 119 --------- 4 files changed, 389 insertions(+), 358 deletions(-) delete mode 100644 yt_dlp/extractor/rutv.py delete mode 100644 yt_dlp/extractor/vesti.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index f08a1aaab4..651168143c 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1779,7 +1779,6 @@ from .rutube import ( RutubePlaylistIE, RutubeTagsIE, ) -from .rutv import RUTVIE from .ruutu import RuutuIE from .ruv import ( RuvIE, @@ -1877,7 +1876,12 @@ from .skynewsau import SkyNewsAUIE from .slideshare import SlideshareIE from .slideslive import SlidesLiveIE from .slutload import SlutloadIE -from .smotrim import SmotrimIE +from .smotrim import ( + SmotrimAudioIE, + SmotrimIE, + SmotrimLiveIE, + SmotrimPlaylistIE, +) from .snapchat import SnapchatSpotlightIE from .snotr import SnotrIE from .softwhiteunderbelly import SoftWhiteUnderbellyIE @@ -2284,7 +2288,6 @@ from .utreon import UtreonIE from .varzesh3 import Varzesh3IE from .vbox7 import Vbox7IE from .veo import VeoIE -from .vesti import VestiIE from .vevo import ( VevoIE, VevoPlaylistIE, diff --git a/yt_dlp/extractor/rutv.py b/yt_dlp/extractor/rutv.py deleted file mode 100644 index 11270a1f2c..0000000000 --- a/yt_dlp/extractor/rutv.py +++ /dev/null @@ -1,191 +0,0 @@ -import re - -from .common import InfoExtractor -from ..utils import ExtractorError, int_or_none, str_to_int - - -class RUTVIE(InfoExtractor): - IE_DESC = 'RUTV.RU' - _VALID_URL = r'''(?x) - https?:// - (?:test)?player\.(?:rutv\.ru|vgtrk\.com)/ - (?P - flash\d+v/container\.swf\?id=| - iframe/(?Pswf|video|live)/id/| - index/iframe/cast_id/ - ) - (?P\d+) - ''' - _EMBED_REGEX = [ - r']+?src=(["\'])(?Phttps?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', - r']+?property=(["\'])og:video\1[^>]+?content=(["\'])(?Phttps?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)', - ] - - _TESTS = [{ - 'url': 'http://player.rutv.ru/flash2v/container.swf?id=774471&sid=kultura&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972347/video_id/978186/brand_id/31724', - 'info_dict': { - 'id': '774471', - 'ext': 'mp4', - 'title': 'Монологи на все времена. Концерт', - 'description': 'md5:18d8b5e6a41fb1faa53819471852d5d5', - 'duration': 2906, - 'thumbnail': r're:https?://cdn-st2\.smotrim\.ru/.+\.jpg', - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'https://player.vgtrk.com/flash2v/container.swf?id=774016&sid=russiatv&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972098/video_id/977760/brand_id/57638', - 'info_dict': { - 'id': '774016', - 'ext': 'mp4', - 'title': 'Чужой в семье Сталина', - 'description': '', - 'duration': 2539, - }, - 'skip': 'Invalid URL', - }, { - 'url': 'http://player.rutv.ru/iframe/swf/id/766888/sid/hitech/?acc_video_id=4000', - 'info_dict': { - 'id': '766888', - 'ext': 'mp4', - 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', - 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', - 'duration': 279, - 'thumbnail': r're:https?://cdn-st2\.smotrim\.ru/.+\.jpg', - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'http://player.rutv.ru/iframe/video/id/771852/start_zoom/true/showZoomBtn/false/sid/russiatv/?acc_video_id=episode_id/970443/video_id/975648/brand_id/5169', - 'info_dict': { - 'id': '771852', - 'ext': 'mp4', - 'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет', - 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8', - 'duration': 3096, - 'thumbnail': r're:https?://cdn-st2\.smotrim\.ru/.+\.jpg', - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'http://player.rutv.ru/iframe/live/id/51499/showZoomBtn/false/isPlay/true/sid/sochi2014', - 'info_dict': { - 'id': '51499', - 'ext': 'flv', - 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', - 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', - }, - 'skip': 'Invalid URL', - }, { - 'url': 'http://player.rutv.ru/iframe/live/id/21/showZoomBtn/false/isPlay/true/', - 'info_dict': { - 'id': '21', - 'ext': 'mp4', - 'title': str, - 'is_live': True, - }, - 'skip': 'Invalid URL', - }, { - 'url': 'https://testplayer.vgtrk.com/iframe/live/id/19201/showZoomBtn/false/isPlay/true/', - 'only_matching': True, - }] - _WEBPAGE_TESTS = [{ - 'url': 'http://istoriya-teatra.ru/news/item/f00/s05/n0000545/index.shtml', - 'info_dict': { - 'id': '1952012', - 'ext': 'mp4', - 'title': 'Новости культуры. Эфир от 10.10.2019 (23:30). Театр Сатиры отмечает день рождения премьерой', - 'description': 'md5:fced27112ff01ff8fc4a452fc088bad6', - 'duration': 191, - 'thumbnail': r're:https?://cdn-st2\.smotrim\.ru/.+\.jpg', - }, - 'params': {'skip_download': 'm3u8'}, - }] - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - video_path = mobj.group('path') - - if re.match(r'flash\d+v', video_path): - video_type = 'video' - elif video_path.startswith('iframe'): - video_type = mobj.group('type') - if video_type == 'swf': - video_type = 'video' - elif video_path.startswith('index/iframe/cast_id'): - video_type = 'live' - - is_live = video_type == 'live' - - json_data = self._download_json( - 'http://player.vgtrk.com/iframe/data{}/id/{}'.format('live' if is_live else 'video', video_id), - video_id, 'Downloading JSON') - - if json_data['errors']: - raise ExtractorError('{} said: {}'.format(self.IE_NAME, json_data['errors']), expected=True) - - playlist = json_data['data']['playlist'] - medialist = playlist['medialist'] - media = medialist[0] - - if media['errors']: - raise ExtractorError('{} said: {}'.format(self.IE_NAME, media['errors']), expected=True) - - view_count = int_or_none(playlist.get('count_views')) - priority_transport = playlist['priority_transport'] - - thumbnail = media['picture'] - width = int_or_none(media['width']) - height = int_or_none(media['height']) - description = media['anons'] - title = media['title'] - duration = int_or_none(media.get('duration')) - - formats = [] - subtitles = {} - - for transport, links in media['sources'].items(): - for quality, url in links.items(): - preference = -1 if priority_transport == transport else -2 - if transport == 'rtmp': - mobj = re.search(r'^(?Prtmp://[^/]+/(?P.+))/(?P.+)$', url) - if not mobj: - continue - fmt = { - 'url': mobj.group('url'), - 'play_path': mobj.group('playpath'), - 'app': mobj.group('app'), - 'page_url': 'http://player.rutv.ru', - 'player_url': 'http://player.rutv.ru/flash3v/osmf.swf?i=22', - 'rtmp_live': True, - 'ext': 'flv', - 'vbr': str_to_int(quality), - } - elif transport == 'm3u8': - fmt, subs = self._extract_m3u8_formats_and_subtitles( - url, video_id, 'mp4', quality=preference, m3u8_id='hls') - formats.extend(fmt) - self._merge_subtitles(subs, target=subtitles) - continue - else: - fmt = { - 'url': url, - } - fmt.update({ - 'width': int_or_none(quality, default=height, invscale=width, scale=height), - 'height': int_or_none(quality, default=height), - 'format_id': f'{transport}-{quality}', - 'source_preference': preference, - }) - formats.append(fmt) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'view_count': view_count, - 'duration': duration, - 'formats': formats, - 'subtitles': subtitles, - 'is_live': is_live, - '_format_sort_fields': ('source', ), - } diff --git a/yt_dlp/extractor/smotrim.py b/yt_dlp/extractor/smotrim.py index d3f1b695b3..098d369daf 100644 --- a/yt_dlp/extractor/smotrim.py +++ b/yt_dlp/extractor/smotrim.py @@ -1,65 +1,403 @@ +import functools +import json +import re +import urllib.parse + from .common import InfoExtractor -from ..utils import ExtractorError +from ..utils import ( + OnDemandPagedList, + clean_html, + determine_ext, + extract_attributes, + int_or_none, + parse_iso8601, + str_or_none, + unescapeHTML, + url_or_none, + urljoin, +) +from ..utils.traversal import ( + find_element, + find_elements, + require, + traverse_obj, +) -class SmotrimIE(InfoExtractor): - _VALID_URL = r'https?://smotrim\.ru/(?Pbrand|video|article|live)/(?P[0-9]+)' - _TESTS = [{ # video +class SmotrimBaseIE(InfoExtractor): + _BASE_URL = 'https://smotrim.ru' + _GEO_BYPASS = False + _GEO_COUNTRIES = ['RU'] + + def _extract_from_smotrim_api(self, typ, item_id): + path = f'data{typ.replace("-", "")}/{"uid" if typ == "live" else "id"}' + data = self._download_json( + f'https://player.smotrim.ru/iframe/{path}/{item_id}/sid/smotrim', item_id) + media = traverse_obj(data, ('data', 'playlist', 'medialist', -1, {dict})) + if traverse_obj(media, ('locked', {bool})): + self.raise_login_required() + if error_msg := traverse_obj(media, ('errors', {clean_html})): + self.raise_geo_restricted(error_msg, countries=self._GEO_COUNTRIES) + + webpage_url = traverse_obj(data, ('data', 'template', 'share_url', {url_or_none})) + webpage = self._download_webpage(webpage_url, item_id) + common = { + 'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage, default=None), + **traverse_obj(media, { + 'id': ('id', {str_or_none}), + 'title': (('episodeTitle', 'title'), {clean_html}, filter, any), + 'channel_id': ('channelId', {str_or_none}), + 'description': ('anons', {clean_html}, filter), + 'season': ('season', {clean_html}, filter), + 'series': (('brand_title', 'brandTitle'), {clean_html}, filter, any), + 'series_id': ('brand_id', {str_or_none}), + }), + } + + if typ == 'audio': + bookmark = self._search_json( + r'class="bookmark"[^>]+value\s*=\s*"', webpage, + 'bookmark', item_id, default={}, transform_source=unescapeHTML) + + metadata = { + 'vcodec': 'none', + **common, + **traverse_obj(media, { + 'ext': ('audio_url', {determine_ext(default_ext='mp3')}), + 'duration': ('duration', {int_or_none}), + 'url': ('audio_url', {url_or_none}), + }), + **traverse_obj(bookmark, { + 'title': ('subtitle', {clean_html}), + 'timestamp': ('published', {parse_iso8601}), + }), + } + elif typ == 'audio-live': + metadata = { + 'ext': 'mp3', + 'url': traverse_obj(media, ('source', 'auto', {url_or_none})), + 'vcodec': 'none', + **common, + } + else: + formats, subtitles = [], {} + for m3u8_url in traverse_obj(media, ( + 'sources', 'm3u8', {dict.values}, ..., {url_or_none}, + )): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + m3u8_url, item_id, 'mp4', m3u8_id='hls', fatal=False) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + + metadata = { + 'formats': formats, + 'subtitles': subtitles, + **self._search_json_ld(webpage, item_id), + **common, + } + + return { + 'age_limit': traverse_obj(data, ('data', 'age_restrictions', {int_or_none})), + 'is_live': typ in ('audio-live', 'live'), + 'tags': traverse_obj(webpage, ( + {find_elements(cls='tags-list__link')}, ..., {clean_html}, filter, all, filter)), + 'webpage_url': webpage_url, + **metadata, + } + + +class SmotrimIE(SmotrimBaseIE): + IE_NAME = 'smotrim' + _VALID_URL = r'(?:https?:)?//(?:(?:player|www)\.)?smotrim\.ru(?:/iframe)?/video(?:/id)?/(?P\d+)' + _EMBED_REGEX = [fr']+\bsrc=["\'](?P{_VALID_URL})'] + _TESTS = [{ 'url': 'https://smotrim.ru/video/1539617', - 'md5': 'b1923a533c8cab09679789d720d0b1c5', 'info_dict': { 'id': '1539617', 'ext': 'mp4', - 'title': 'Полиглот. Китайский с нуля за 16 часов! Урок №16', - 'description': '', + 'title': 'Урок №16', + 'duration': 2631, + 'series': 'Полиглот. Китайский с нуля за 16 часов!', + 'series_id': '60562', + 'tags': 'mincount:6', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1466771100, + 'upload_date': '20160624', + 'view_count': int, }, - 'add_ie': ['RUTV'], - }, { # article (geo-restricted? plays fine from the US and JP) + }, { + 'url': 'https://player.smotrim.ru/iframe/video/id/2988590', + 'info_dict': { + 'id': '2988590', + 'ext': 'mp4', + 'title': 'Трейлер', + 'age_limit': 16, + 'description': 'md5:6af7e68ecf4ed7b8ff6720d20c4da47b', + 'duration': 30, + 'series': 'Мы в разводе', + 'series_id': '71624', + 'tags': 'mincount:5', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1750670040, + 'upload_date': '20250623', + 'view_count': int, + 'webpage_url': 'https://smotrim.ru/video/2988590', + }, + }] + _WEBPAGE_TESTS = [{ 'url': 'https://smotrim.ru/article/2813445', - 'md5': 'e0ac453952afbc6a2742e850b4dc8e77', 'info_dict': { 'id': '2431846', 'ext': 'mp4', - 'title': 'Новости культуры. Съёмки первой программы "Большие и маленькие"', - 'description': 'md5:94a4a22472da4252bf5587a4ee441b99', + 'title': 'Съёмки первой программы "Большие и маленькие"', + 'description': 'md5:446c9a5d334b995152a813946353f447', + 'duration': 240, + 'series': 'Новости культуры', + 'series_id': '19725', + 'tags': 'mincount:6', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1656054443, + 'upload_date': '20220624', + 'view_count': int, + 'webpage_url': 'https://smotrim.ru/video/2431846', }, - 'add_ie': ['RUTV'], - }, { # brand, redirect - 'url': 'https://smotrim.ru/brand/64356', - 'md5': '740472999ccff81d7f6df79cecd91c18', + }, { + 'url': 'https://www.vesti.ru/article/4642878', 'info_dict': { - 'id': '2354523', + 'id': '3007209', 'ext': 'mp4', - 'title': 'Большие и маленькие. Лучшее. 4-й выпуск', - 'description': 'md5:84089e834429008371ea41ea3507b989', + 'title': 'Иностранные мессенджеры используют не только мошенники, но и вербовщики', + 'description': 'md5:74ab625a0a89b87b2e0ed98d6391b182', + 'duration': 265, + 'series': 'Вести. Дежурная часть', + 'series_id': '5204', + 'tags': 'mincount:6', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1754756280, + 'upload_date': '20250809', + 'view_count': int, + 'webpage_url': 'https://smotrim.ru/video/3007209', }, - 'add_ie': ['RUTV'], - }, { # live - 'url': 'https://smotrim.ru/live/19201', - 'info_dict': { - 'id': '19201', - 'ext': 'mp4', - # this looks like a TV channel name - 'title': 'Россия Культура. Прямой эфир', - 'description': '', - }, - 'add_ie': ['RUTV'], }] def _real_extract(self, url): - video_id, typ = self._match_valid_url(url).group('id', 'type') - rutv_type = 'video' - if typ not in ('video', 'live'): - webpage = self._download_webpage(url, video_id, f'Resolving {typ} link') - # there are two cases matching regex: - # 1. "embedUrl" in JSON LD (/brand/) - # 2. "src" attribute from iframe (/article/) - video_id = self._search_regex( - r'"https://player.smotrim.ru/iframe/video/id/(?P\d+)/', - webpage, 'video_id', default=None) - if not video_id: - raise ExtractorError('There are no video in this page.', expected=True) - elif typ == 'live': - rutv_type = 'live' + video_id = self._match_id(url) - return self.url_result(f'https://player.vgtrk.com/iframe/{rutv_type}/id/{video_id}') + return self._extract_from_smotrim_api('video', video_id) + + +class SmotrimAudioIE(SmotrimBaseIE): + IE_NAME = 'smotrim:audio' + _VALID_URL = r'https?://(?:(?:player|www)\.)?smotrim\.ru(?:/iframe)?/audio(?:/id)?/(?P\d+)' + _TESTS = [{ + 'url': 'https://smotrim.ru/audio/2573986', + 'md5': 'e28d94c20da524e242b2d00caef41a8e', + 'info_dict': { + 'id': '2573986', + 'ext': 'mp3', + 'title': 'Радиоспектакль', + 'description': 'md5:4bcaaf7d532bc78f76e478fad944e388', + 'duration': 3072, + 'series': 'Морис Леблан. Арсен Люпен, джентльмен-грабитель', + 'series_id': '66461', + 'tags': 'mincount:7', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1624884358, + 'upload_date': '20210628', + }, + }, { + 'url': 'https://player.smotrim.ru/iframe/audio/id/2860468', + 'md5': '5a6bc1fa24c7142958be1ad9cfae58a8', + 'info_dict': { + 'id': '2860468', + 'ext': 'mp3', + 'title': 'Колобок и музыкальная игра "Терем-теремок"', + 'duration': 1501, + 'series': 'Веселый колобок', + 'series_id': '68880', + 'tags': 'mincount:4', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': 1755925800, + 'upload_date': '20250823', + 'webpage_url': 'https://smotrim.ru/audio/2860468', + }, + }] + + def _real_extract(self, url): + audio_id = self._match_id(url) + + return self._extract_from_smotrim_api('audio', audio_id) + + +class SmotrimLiveIE(SmotrimBaseIE): + IE_NAME = 'smotrim:live' + _VALID_URL = r'''(?x: + (?:https?:)?// + (?:(?:(?:test)?player|www)\.)? + (?: + smotrim\.ru| + vgtrk\.com + ) + (?:/iframe)?/ + (?P + channel| + (?:audio-)?live + ) + (?:/u?id)?/(?P[\da-f-]+) + )''' + _EMBED_REGEX = [fr']+\bsrc=["\'](?P{_VALID_URL})'] + _TESTS = [{ + 'url': 'https://smotrim.ru/channel/76', + 'info_dict': { + 'id': '1661', + 'ext': 'mp4', + 'title': str, + 'channel_id': '76', + 'description': 'Смотрим прямой эфир «Москва 24»', + 'display_id': '76', + 'live_status': 'is_live', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': int, + 'upload_date': str, + }, + 'params': {'skip_download': 'Livestream'}, + }, { + # Radio + 'url': 'https://smotrim.ru/channel/81', + 'info_dict': { + 'id': '81', + 'ext': 'mp3', + 'title': str, + 'channel_id': '81', + 'live_status': 'is_live', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + }, + 'params': {'skip_download': 'Livestream'}, + }, { + # Sometimes geo-restricted to Russia + 'url': 'https://player.smotrim.ru/iframe/live/uid/381308c7-a066-4c4f-9656-83e2e792a7b4', + 'info_dict': { + 'id': '19201', + 'ext': 'mp4', + 'title': str, + 'channel_id': '4', + 'description': 'Смотрим прямой эфир «Россия К»', + 'display_id': '381308c7-a066-4c4f-9656-83e2e792a7b4', + 'live_status': 'is_live', + 'thumbnail': r're:https?://cdn-st\d+\.smotrim\.ru/.+\.(?:jpg|png)', + 'timestamp': int, + 'upload_date': str, + 'webpage_url': 'https://smotrim.ru/channel/4', + }, + 'params': {'skip_download': 'Livestream'}, + }, { + 'url': 'https://smotrim.ru/live/19201', + 'only_matching': True, + }, { + 'url': 'https://player.smotrim.ru/iframe/audio-live/id/81', + 'only_matching': True, + }, { + 'url': 'https://testplayer.vgtrk.com/iframe/live/id/19201', + 'only_matching': True, + }] + + def _real_extract(self, url): + typ, display_id = self._match_valid_url(url).group('type', 'id') + + if typ == 'live' and re.fullmatch(r'[0-9]+', display_id): + url = self._request_webpage(url, display_id).url + typ = self._match_valid_url(url).group('type') + + if typ == 'channel': + webpage = self._download_webpage(url, display_id) + src_url = traverse_obj(webpage, (( + ({find_element(cls='main-player__frame', html=True)}, {extract_attributes}, 'src'), + ({find_element(cls='audio-play-button', html=True)}, + {extract_attributes}, 'value', {urllib.parse.unquote}, {json.loads}, 'source'), + ), any, {self._proto_relative_url}, {url_or_none}, {require('src URL')})) + typ, video_id = self._match_valid_url(src_url).group('type', 'id') + else: + video_id = display_id + + return { + 'display_id': display_id, + **self._extract_from_smotrim_api(typ, video_id), + } + + +class SmotrimPlaylistIE(SmotrimBaseIE): + IE_NAME = 'smotrim:playlist' + _PAGE_SIZE = 15 + _VALID_URL = r'https?://smotrim\.ru/(?Pbrand|podcast)/(?P\d+)/?(?P[\w-]+)?' + _TESTS = [{ + # Video + 'url': 'https://smotrim.ru/brand/64356', + 'info_dict': { + 'id': '64356', + 'title': 'Большие и маленькие', + }, + 'playlist_mincount': 55, + }, { + # Video, season + 'url': 'https://smotrim.ru/brand/65293/3-sezon', + 'info_dict': { + 'id': '65293', + 'title': 'Спасская', + 'season': '3 сезон', + }, + 'playlist_count': 16, + }, { + # Audio + 'url': 'https://smotrim.ru/brand/68880', + 'info_dict': { + 'id': '68880', + 'title': 'Веселый колобок', + }, + 'playlist_mincount': 156, + }, { + # Podcast + 'url': 'https://smotrim.ru/podcast/8021', + 'info_dict': { + 'id': '8021', + 'title': 'Сила звука', + }, + 'playlist_mincount': 27, + }] + + def _fetch_page(self, endpoint, key, playlist_id, page): + page += 1 + items = self._download_json( + f'{self._BASE_URL}/api/{endpoint}', playlist_id, + f'Downloading page {page}', query={ + key: playlist_id, + 'limit': self._PAGE_SIZE, + 'page': page, + }, + ) + + for link in traverse_obj(items, ('contents', -1, 'list', ..., 'link', {str})): + yield self.url_result(urljoin(self._BASE_URL, link)) + + def _real_extract(self, url): + playlist_type, playlist_id, season = self._match_valid_url(url).group('type', 'id', 'season') + key = 'rubricId' if playlist_type == 'podcast' else 'brandId' + webpage = self._download_webpage(url, playlist_id) + playlist_title = self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None) + + if season: + return self.playlist_from_matches(traverse_obj(webpage, ( + {find_elements(tag='a', attr='href', value=r'/video/\d+', html=True, regex=True)}, + ..., {extract_attributes}, 'href', {str}, + )), playlist_id, playlist_title, season=traverse_obj(webpage, ( + {find_element(cls='seasons__item seasons__item--selected')}, {clean_html}, + )), ie=SmotrimIE, getter=urljoin(self._BASE_URL)) + + if traverse_obj(webpage, ( + {find_element(cls='brand-main-item__videos')}, {clean_html}, filter, + )): + endpoint = 'videos' + else: + endpoint = 'audios' + + return self.playlist_result(OnDemandPagedList( + functools.partial(self._fetch_page, endpoint, key, playlist_id), self._PAGE_SIZE), playlist_id, playlist_title) diff --git a/yt_dlp/extractor/vesti.py b/yt_dlp/extractor/vesti.py deleted file mode 100644 index 844041a61a..0000000000 --- a/yt_dlp/extractor/vesti.py +++ /dev/null @@ -1,119 +0,0 @@ -import re - -from .common import InfoExtractor -from .rutv import RUTVIE -from ..utils import ExtractorError - - -class VestiIE(InfoExtractor): - _WORKING = False - IE_DESC = 'Вести.Ru' - _VALID_URL = r'https?://(?:.+?\.)?vesti\.ru/(?P.+)' - - _TESTS = [ - { - 'url': 'http://www.vesti.ru/videos?vid=575582&cid=1', - 'info_dict': { - 'id': '765035', - 'ext': 'mp4', - 'title': 'Вести.net: биткоины в России не являются законными', - 'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b', - 'duration': 302, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/doc.html?id=1349233', - 'info_dict': { - 'id': '773865', - 'ext': 'mp4', - 'title': 'Участники митинга штурмуют Донецкую областную администрацию', - 'description': 'md5:1a160e98b3195379b4c849f2f4958009', - 'duration': 210, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/only_video.html?vid=576180', - 'info_dict': { - 'id': '766048', - 'ext': 'mp4', - 'title': 'США заморозило, Британию затопило', - 'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1', - 'duration': 87, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://hitech.vesti.ru/news/view/id/4000', - 'info_dict': { - 'id': '766888', - 'ext': 'mp4', - 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', - 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', - 'duration': 279, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403', - 'info_dict': { - 'id': '766403', - 'ext': 'mp4', - 'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы', - 'description': 'md5:55805dfd35763a890ff50fa9e35e31b3', - 'duration': 271, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Blocked outside Russia', - }, - { - 'url': 'http://sochi2014.vesti.ru/live/play/live_id/301', - 'info_dict': { - 'id': '51499', - 'ext': 'flv', - 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', - 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', - }, - 'params': { - # rtmp download - 'skip_download': True, - }, - 'skip': 'Translation has finished', - }, - ] - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - - page = self._download_webpage(url, video_id, 'Downloading page') - - mobj = re.search( - r']+?property="og:video"[^>]+?content="http://www\.vesti\.ru/i/flvplayer_videoHost\.swf\?vid=(?P\d+)', - page) - if mobj: - video_id = mobj.group('id') - page = self._download_webpage(f'http://www.vesti.ru/only_video.html?vid={video_id}', video_id, - 'Downloading video page') - - rutv_url = RUTVIE._extract_url(page) - if rutv_url: - return self.url_result(rutv_url, 'RUTV') - - raise ExtractorError('No video found', expected=True) From 17bfaa53edf5c52fce73cf0cef4592f929c2462d Mon Sep 17 00:00:00 2001 From: doe1080 <98906116+doe1080@users.noreply.github.com> Date: Fri, 12 Sep 2025 07:51:31 +0900 Subject: [PATCH 037/175] [ie/onsen] Add extractor (#10971) Closes #10902 Authored by: doe1080 --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/onsen.py | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 yt_dlp/extractor/onsen.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 651168143c..b3dd52b504 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1433,6 +1433,7 @@ from .onet import ( OnetPlIE, ) from .onionstudios import OnionStudiosIE +from .onsen import OnsenIE from .opencast import ( OpencastIE, OpencastPlaylistIE, diff --git a/yt_dlp/extractor/onsen.py b/yt_dlp/extractor/onsen.py new file mode 100644 index 0000000000..6424d7b154 --- /dev/null +++ b/yt_dlp/extractor/onsen.py @@ -0,0 +1,151 @@ +import base64 +import json + +from .common import InfoExtractor +from ..networking.exceptions import HTTPError +from ..utils import ( + ExtractorError, + clean_html, + int_or_none, + parse_qs, + str_or_none, + strftime_or_none, + update_url, + update_url_query, + url_or_none, +) +from ..utils.traversal import traverse_obj + + +class OnsenIE(InfoExtractor): + IE_NAME = 'onsen' + IE_DESC = 'インターネットラジオステーション＜音泉＞' + + _BASE_URL = 'https://www.onsen.ag' + _HEADERS = {'Referer': f'{_BASE_URL}/'} + _NETRC_MACHINE = 'onsen' + _VALID_URL = r'https?://(?:(?:share|www)\.)onsen\.ag/program/(?P[^/?#]+)' + _TESTS = [{ + 'url': 'https://share.onsen.ag/program/onsenking?p=90&c=MTA0NjI', + 'info_dict': { + 'id': '10462', + 'ext': 'm4a', + 'title': '第SP回', + 'cast': 'count:3', + 'description': 'md5:de62c80a41c4c8d84da53a1ee681ad18', + 'display_id': 'MTA0NjI=', + 'media_type': 'sound', + 'section_start': 0, + 'series': '音泉キング「下野紘」のラジオ きみはもちろん、＜音泉＞ファミリーだよね？', + 'series_id': 'onsenking', + 'tags': 'count:2', + 'thumbnail': r're:https?://d3bzklg4lms4gh\.cloudfront\.net/program_info/image/default/production/.+', + 'upload_date': '20220627', + 'webpage_url': 'https://www.onsen.ag/program/onsenking?c=MTA0NjI=', + }, + }, { + 'url': 'https://share.onsen.ag/program/girls-band-cry-radio?p=370&c=MTgwMDE', + 'info_dict': { + 'id': '18001', + 'ext': 'mp4', + 'title': '第4回', + 'cast': 'count:5', + 'description': 'md5:bbca8a389d99c90cbbce8f383c85fedd', + 'display_id': 'MTgwMDE=', + 'media_type': 'movie', + 'section_start': 0, + 'series': 'TVアニメ『ガールズバンドクライ』WEBラジオ「ガールズバンドクライ～ラジオにも全部ぶち込め。～」', + 'series_id': 'girls-band-cry-radio', + 'tags': 'count:3', + 'thumbnail': r're:https?://d3bzklg4lms4gh\.cloudfront\.net/program_info/image/default/production/.+', + 'upload_date': '20240425', + 'webpage_url': 'https://www.onsen.ag/program/girls-band-cry-radio?c=MTgwMDE=', + }, + 'skip': 'Only available for premium supporters', + }, { + 'url': 'https://www.onsen.ag/program/uma', + 'info_dict': { + 'id': 'uma', + 'title': 'UMA YELL RADIO', + }, + 'playlist_mincount': 35, + }] + + @staticmethod + def _get_encoded_id(program): + return base64.urlsafe_b64encode(str(program['id']).encode()).decode() + + def _perform_login(self, username, password): + sign_in = self._download_json( + f'{self._BASE_URL}/web_api/signin', None, 'Logging in', headers={ + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, data=json.dumps({ + 'session': { + 'email': username, + 'password': password, + }, + }).encode(), expected_status=401) + + if sign_in.get('error'): + raise ExtractorError('Invalid username or password', expected=True) + + def _real_extract(self, url): + program_id = self._match_id(url) + try: + programs = self._download_json( + f'{self._BASE_URL}/web_api/programs/{program_id}', program_id) + except ExtractorError as e: + if isinstance(e.cause, HTTPError) and e.cause.status == 404: + raise ExtractorError('Invalid URL', expected=True) + raise + + query = {k: v[-1] for k, v in parse_qs(url).items() if v} + if 'c' not in query: + entries = [ + self.url_result(update_url_query(url, {'c': self._get_encoded_id(program)}), OnsenIE) + for program in traverse_obj(programs, ('contents', lambda _, v: v['id'])) + ] + + return self.playlist_result( + entries, program_id, traverse_obj(programs, ('program_info', 'title', {clean_html}))) + + raw_id = base64.urlsafe_b64decode(f'{query["c"]}===').decode() + p_keys = ('contents', lambda _, v: v['id'] == int(raw_id)) + + program = traverse_obj(programs, (*p_keys, any)) + if not program: + raise ExtractorError( + 'This program is no longer available', expected=True) + m3u8_url = traverse_obj(program, ('streaming_url', {url_or_none})) + if not m3u8_url: + self.raise_login_required( + 'This program is only available for premium supporters') + + display_id = self._get_encoded_id(program) + date_str = self._search_regex( + rf'{program_id}0?(\d{{6}})', m3u8_url, 'date string', default=None) + + return { + 'display_id': display_id, + 'formats': self._extract_m3u8_formats(m3u8_url, raw_id, headers=self._HEADERS), + 'http_headers': self._HEADERS, + 'section_start': int_or_none(query.get('t', 0)), + 'upload_date': strftime_or_none(f'20{date_str}'), + 'webpage_url': f'{self._BASE_URL}/program/{program_id}?c={display_id}', + **traverse_obj(program, { + 'id': ('id', {int}, {str_or_none}), + 'title': ('title', {clean_html}), + 'media_type': ('media_type', {str}), + 'thumbnail': ('poster_image_url', {url_or_none}, {update_url(query=None)}), + }), + **traverse_obj(programs, { + 'cast': (('performers', (*p_keys, 'guests')), ..., 'name', {str}, filter), + 'series_id': ('directory_name', {str}), + }), + **traverse_obj(programs, ('program_info', { + 'description': ('description', {clean_html}, filter), + 'series': ('title', {clean_html}), + 'tags': ('hashtag_list', ..., {str}, filter), + })), + } From 65e90aea29cf3bfc9d1ae3e009fbf9a8db3a23c9 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 12 Sep 2025 03:15:41 -0500 Subject: [PATCH 038/175] [cleanup] Remove broken extractors (#14305) Closes #1466, Closes #2005, Closes #4897, Closes #5118, Closes #8489, Closes #13072 Authored by: bashonly --- yt_dlp/extractor/_extractors.py | 15 -- yt_dlp/extractor/cbsnews.py | 10 +- yt_dlp/extractor/crackle.py | 243 ------------------------------ yt_dlp/extractor/cwtv.py | 180 ---------------------- yt_dlp/extractor/paramountplus.py | 201 ------------------------ yt_dlp/extractor/sixplay.py | 119 --------------- yt_dlp/extractor/spotify.py | 167 -------------------- yt_dlp/extractor/unsupported.py | 53 ++++++- yt_dlp/extractor/xanimu.py | 52 ------- 9 files changed, 47 insertions(+), 993 deletions(-) delete mode 100644 yt_dlp/extractor/crackle.py delete mode 100644 yt_dlp/extractor/cwtv.py delete mode 100644 yt_dlp/extractor/paramountplus.py delete mode 100644 yt_dlp/extractor/sixplay.py delete mode 100644 yt_dlp/extractor/spotify.py delete mode 100644 yt_dlp/extractor/xanimu.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index b3dd52b504..9d3d353683 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -424,7 +424,6 @@ from .cpac import ( CPACPlaylistIE, ) from .cracked import CrackedIE -from .crackle import CrackleIE from .craftsy import CraftsyIE from .crooksandliars import CrooksAndLiarsIE from .crowdbunker import ( @@ -444,10 +443,6 @@ from .curiositystream import ( CuriosityStreamIE, CuriosityStreamSeriesIE, ) -from .cwtv import ( - CWTVIE, - CWTVMovieIE, -) from .cybrary import ( CybraryCourseIE, CybraryIE, @@ -1467,10 +1462,6 @@ from .panopto import ( PanoptoListIE, PanoptoPlaylistIE, ) -from .paramountplus import ( - ParamountPlusIE, - ParamountPlusSeriesIE, -) from .parler import ParlerIE from .parlview import ParlviewIE from .parti import ( @@ -1849,7 +1840,6 @@ from .simplecast import ( SimplecastPodcastIE, ) from .sina import SinaIE -from .sixplay import SixPlayIE from .skeb import SkebIE from .sky import ( SkyNewsIE, @@ -1930,10 +1920,6 @@ from .spiegel import SpiegelIE from .sport5 import Sport5IE from .sportbox import SportBoxIE from .sportdeutschland import SportDeutschlandIE -from .spotify import ( - SpotifyIE, - SpotifyShowIE, -) from .spreaker import ( SpreakerIE, SpreakerShowIE, @@ -2477,7 +2463,6 @@ from .wykop import ( WykopPostCommentIE, WykopPostIE, ) -from .xanimu import XanimuIE from .xboxclips import XboxClipsIE from .xhamster import ( XHamsterEmbedIE, diff --git a/yt_dlp/extractor/cbsnews.py b/yt_dlp/extractor/cbsnews.py index b01c0efd5d..59457813f5 100644 --- a/yt_dlp/extractor/cbsnews.py +++ b/yt_dlp/extractor/cbsnews.py @@ -5,8 +5,6 @@ import zlib from .anvato import AnvatoIE from .common import InfoExtractor -from .paramountplus import ParamountPlusIE -from ..networking import HEADRequest from ..utils import ( ExtractorError, UserNotLive, @@ -132,13 +130,7 @@ class CBSNewsEmbedIE(CBSNewsBaseIE): video_id = item['mpxRefId'] video_url = self._get_video_url(item) if not video_url: - # Old embeds redirect user to ParamountPlus but most links are 404 - pplus_url = f'https://www.paramountplus.com/shows/video/{video_id}' - try: - self._request_webpage(HEADRequest(pplus_url), video_id) - return self.url_result(pplus_url, ParamountPlusIE) - except ExtractorError: - self.raise_no_formats('This video is no longer available', True, video_id) + raise ExtractorError('This video is no longer available', expected=True) return self._extract_video(item, video_url, video_id) diff --git a/yt_dlp/extractor/crackle.py b/yt_dlp/extractor/crackle.py deleted file mode 100644 index c4ceba9408..0000000000 --- a/yt_dlp/extractor/crackle.py +++ /dev/null @@ -1,243 +0,0 @@ -import hashlib -import hmac -import re -import time - -from .common import InfoExtractor -from ..networking.exceptions import HTTPError -from ..utils import ( - ExtractorError, - determine_ext, - float_or_none, - int_or_none, - orderedSet, - parse_age_limit, - parse_duration, - url_or_none, -) - - -class CrackleIE(InfoExtractor): - _VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?(?:sony)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P\d+)' - _TESTS = [{ - # Crackle is available in the United States and territories - 'url': 'https://www.crackle.com/thanksgiving/2510064', - 'info_dict': { - 'id': '2510064', - 'ext': 'mp4', - 'title': 'Touch Football', - 'description': 'md5:cfbb513cf5de41e8b56d7ab756cff4df', - 'duration': 1398, - 'view_count': int, - 'average_rating': 0, - 'age_limit': 17, - 'genre': 'Comedy', - 'creator': 'Daniel Powell', - 'artist': 'Chris Elliott, Amy Sedaris', - 'release_year': 2016, - 'series': 'Thanksgiving', - 'episode': 'Touch Football', - 'season_number': 1, - 'episode_number': 1, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'expected_warnings': [ - 'Trying with a list of known countries', - ], - }, { - 'url': 'https://www.sonycrackle.com/thanksgiving/2510064', - 'only_matching': True, - }] - - _MEDIA_FILE_SLOTS = { - '360p.mp4': { - 'width': 640, - 'height': 360, - }, - '480p.mp4': { - 'width': 768, - 'height': 432, - }, - '480p_1mbps.mp4': { - 'width': 852, - 'height': 480, - }, - } - - def _download_json(self, url, *args, **kwargs): - # Authorization generation algorithm is reverse engineered from: - # https://www.sonycrackle.com/static/js/main.ea93451f.chunk.js - timestamp = time.strftime('%Y%m%d%H%M', time.gmtime()) - h = hmac.new(b'IGSLUQCBDFHEOIFM', '|'.join([url, timestamp]).encode(), hashlib.sha1).hexdigest().upper() - headers = { - 'Accept': 'application/json', - 'Authorization': '|'.join([h, timestamp, '117', '1']), - } - return InfoExtractor._download_json(self, url, *args, headers=headers, **kwargs) - - def _real_extract(self, url): - video_id = self._match_id(url) - - geo_bypass_country = self.get_param('geo_bypass_country', None) - countries = orderedSet((geo_bypass_country, 'US', 'AU', 'CA', 'AS', 'FM', 'GU', 'MP', 'PR', 'PW', 'MH', 'VI', '')) - num_countries, num = len(countries) - 1, 0 - - media = {} - for num, country in enumerate(countries): - if num == 1: # start hard-coded list - self.report_warning('%s. Trying with a list of known countries' % ( - f'Unable to obtain video formats from {geo_bypass_country} API' if geo_bypass_country - else 'No country code was given using --geo-bypass-country')) - elif num == num_countries: # end of list - geo_info = self._download_json( - 'https://web-api-us.crackle.com/Service.svc/geo/country', - video_id, fatal=False, note='Downloading geo-location information from crackle API', - errnote='Unable to fetch geo-location information from crackle') or {} - country = geo_info.get('CountryCode') - if country is None: - continue - self.to_screen(f'{self.IE_NAME} identified country as {country}') - if country in countries: - self.to_screen(f'Downloading from {country} API was already attempted. Skipping...') - continue - - if country is None: - continue - try: - media = self._download_json( - f'https://web-api-us.crackle.com/Service.svc/details/media/{video_id}/{country}?disableProtocols=true', - video_id, note=f'Downloading media JSON from {country} API', - errnote='Unable to download media JSON') - except ExtractorError as e: - # 401 means geo restriction, trying next country - if isinstance(e.cause, HTTPError) and e.cause.status == 401: - continue - raise - - status = media.get('status') - if status.get('messageCode') != '0': - raise ExtractorError( - '{} said: {} {} - {}'.format( - self.IE_NAME, status.get('messageCodeDescription'), status.get('messageCode'), status.get('message')), - expected=True) - - # Found video formats - if isinstance(media.get('MediaURLs'), list): - break - - ignore_no_formats = self.get_param('ignore_no_formats_error') - - if not media or (not media.get('MediaURLs') and not ignore_no_formats): - raise ExtractorError( - 'Unable to access the crackle API. Try passing your country code ' - 'to --geo-bypass-country. If it still does not work and the ' - 'video is available in your country') - title = media['Title'] - - formats, subtitles = [], {} - has_drm = False - for e in media.get('MediaURLs') or []: - if e.get('UseDRM'): - has_drm = True - format_url = url_or_none(e.get('DRMPath')) - else: - format_url = url_or_none(e.get('Path')) - if not format_url: - continue - ext = determine_ext(format_url) - if ext == 'm3u8': - fmts, subs = self._extract_m3u8_formats_and_subtitles( - format_url, video_id, 'mp4', entry_protocol='m3u8_native', - m3u8_id='hls', fatal=False) - formats.extend(fmts) - subtitles = self._merge_subtitles(subtitles, subs) - elif ext == 'mpd': - fmts, subs = self._extract_mpd_formats_and_subtitles( - format_url, video_id, mpd_id='dash', fatal=False) - formats.extend(fmts) - subtitles = self._merge_subtitles(subtitles, subs) - elif format_url.endswith('.ism/Manifest'): - fmts, subs = self._extract_ism_formats_and_subtitles( - format_url, video_id, ism_id='mss', fatal=False) - formats.extend(fmts) - subtitles = self._merge_subtitles(subtitles, subs) - else: - mfs_path = e.get('Type') - mfs_info = self._MEDIA_FILE_SLOTS.get(mfs_path) - if not mfs_info: - continue - formats.append({ - 'url': format_url, - 'format_id': 'http-' + mfs_path.split('.')[0], - 'width': mfs_info['width'], - 'height': mfs_info['height'], - }) - if not formats and has_drm: - self.report_drm(video_id) - - description = media.get('Description') - duration = int_or_none(media.get( - 'DurationInSeconds')) or parse_duration(media.get('Duration')) - view_count = int_or_none(media.get('CountViews')) - average_rating = float_or_none(media.get('UserRating')) - age_limit = parse_age_limit(media.get('Rating')) - genre = media.get('Genre') - release_year = int_or_none(media.get('ReleaseYear')) - creator = media.get('Directors') - artist = media.get('Cast') - - if media.get('MediaTypeDisplayValue') == 'Full Episode': - series = media.get('ShowName') - episode = title - season_number = int_or_none(media.get('Season')) - episode_number = int_or_none(media.get('Episode')) - else: - series = episode = season_number = episode_number = None - - cc_files = media.get('ClosedCaptionFiles') - if isinstance(cc_files, list): - for cc_file in cc_files: - if not isinstance(cc_file, dict): - continue - cc_url = url_or_none(cc_file.get('Path')) - if not cc_url: - continue - lang = cc_file.get('Locale') or 'en' - subtitles.setdefault(lang, []).append({'url': cc_url}) - - thumbnails = [] - images = media.get('Images') - if isinstance(images, list): - for image_key, image_url in images.items(): - mobj = re.search(r'Img_(\d+)[xX](\d+)', image_key) - if not mobj: - continue - thumbnails.append({ - 'url': image_url, - 'width': int(mobj.group(1)), - 'height': int(mobj.group(2)), - }) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'duration': duration, - 'view_count': view_count, - 'average_rating': average_rating, - 'age_limit': age_limit, - 'genre': genre, - 'creator': creator, - 'artist': artist, - 'release_year': release_year, - 'series': series, - 'episode': episode, - 'season_number': season_number, - 'episode_number': episode_number, - 'thumbnails': thumbnails, - 'subtitles': subtitles, - 'formats': formats, - } diff --git a/yt_dlp/extractor/cwtv.py b/yt_dlp/extractor/cwtv.py deleted file mode 100644 index cdb29fcee7..0000000000 --- a/yt_dlp/extractor/cwtv.py +++ /dev/null @@ -1,180 +0,0 @@ -import re - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - int_or_none, - parse_age_limit, - parse_iso8601, - parse_qs, - smuggle_url, - str_or_none, - update_url_query, -) -from ..utils.traversal import traverse_obj - - -class CWTVIE(InfoExtractor): - IE_NAME = 'cwtv' - _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch|guid)=(?P[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})' - _TESTS = [{ - 'url': 'https://www.cwtv.com/shows/continuum/a-stitch-in-time/?play=9149a1e1-4cb2-46d7-81b2-47d35bbd332b', - 'info_dict': { - 'id': '9149a1e1-4cb2-46d7-81b2-47d35bbd332b', - 'ext': 'mp4', - 'title': 'A Stitch in Time', - 'description': r're:(?s)City Protective Services officer Kiera Cameron is transported from 2077.+', - 'thumbnail': r're:https?://.+\.jpe?g', - 'duration': 2632, - 'timestamp': 1736928000, - 'uploader': 'CWTV', - 'chapters': 'count:5', - 'series': 'Continuum', - 'season_number': 1, - 'episode_number': 1, - 'age_limit': 14, - 'upload_date': '20250115', - 'season': 'Season 1', - 'episode': 'Episode 1', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63', - 'info_dict': { - 'id': '6b15e985-9345-4f60-baf8-56e96be57c63', - 'ext': 'mp4', - 'title': 'Legends of Yesterday', - 'description': r're:(?s)Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote.+', - 'duration': 2665, - 'series': 'Arrow', - 'season_number': 4, - 'season': '4', - 'episode_number': 8, - 'upload_date': '20151203', - 'timestamp': 1449122100, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'redirect to http://cwtv.com/shows/arrow/', - }, { - 'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088', - 'info_dict': { - 'id': '24282b12-ead2-42f2-95ad-26770c2c6088', - 'ext': 'mp4', - 'title': 'Jeff Davis 4', - 'description': 'Jeff Davis is back to make you laugh.', - 'duration': 1263, - 'series': 'Whose Line Is It Anyway?', - 'season_number': 11, - 'episode_number': 20, - 'upload_date': '20151006', - 'timestamp': 1444107300, - 'age_limit': 14, - 'uploader': 'CWTV', - 'thumbnail': r're:https?://.+\.jpe?g', - 'chapters': 'count:4', - 'episode': 'Episode 20', - 'season': 'Season 11', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, { - 'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6', - 'only_matching': True, - }, { - 'url': 'http://cwtvpr.com/the-cw/video?watch=9eee3f60-ef4e-440b-b3b2-49428ac9c54e', - 'only_matching': True, - }, { - 'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63', - 'only_matching': True, - }, { - 'url': 'http://www.cwtv.com/movies/play/?guid=0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - data = self._download_json( - f'https://images.cwtv.com/feed/app-2/video-meta/apiversion_22/device_android/guid_{video_id}', video_id) - if traverse_obj(data, 'result') != 'ok': - raise ExtractorError(traverse_obj(data, (('error_msg', 'msg'), {str}, any)), expected=True) - video_data = data['video'] - title = video_data['title'] - mpx_url = update_url_query( - video_data.get('mpx_url') or f'https://link.theplatform.com/s/cwtv/media/guid/2703454149/{video_id}', - {'formats': 'M3U+none'}) - - season = str_or_none(video_data.get('season')) - episode = str_or_none(video_data.get('episode')) - if episode and season: - episode = episode[len(season):] - - return { - '_type': 'url_transparent', - 'id': video_id, - 'title': title, - 'url': smuggle_url(mpx_url, {'force_smil_url': True}), - 'description': video_data.get('description_long'), - 'duration': int_or_none(video_data.get('duration_secs')), - 'series': video_data.get('series_name'), - 'season_number': int_or_none(season), - 'episode_number': int_or_none(episode), - 'timestamp': parse_iso8601(video_data.get('start_time')), - 'age_limit': parse_age_limit(video_data.get('rating')), - 'ie_key': 'ThePlatform', - 'thumbnail': video_data.get('large_thumbnail'), - } - - -class CWTVMovieIE(InfoExtractor): - IE_NAME = 'cwtv:movie' - _VALID_URL = r'https?://(?:www\.)?cwtv\.com/shows/(?P[\w-]+)/?\?(?:[^#]+&)?viewContext=Movies' - _TESTS = [{ - 'url': 'https://www.cwtv.com/shows/the-crush/?viewContext=Movies+Swimlane', - 'info_dict': { - 'id': '0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c', - 'ext': 'mp4', - 'title': 'The Crush', - 'upload_date': '20241112', - 'description': 'md5:1549acd90dff4a8273acd7284458363e', - 'chapters': 'count:9', - 'timestamp': 1731398400, - 'age_limit': 16, - 'duration': 5337, - 'series': 'The Crush', - 'season': 'Season 1', - 'uploader': 'CWTV', - 'season_number': 1, - 'episode': 'Episode 1', - 'episode_number': 1, - 'thumbnail': r're:https?://.+\.jpe?g', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }] - _UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}' - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - app_url = ( - self._html_search_meta('al:ios:url', webpage, default=None) - or self._html_search_meta('al:android:url', webpage, default=None)) - video_id = ( - traverse_obj(parse_qs(app_url), ('video_id', 0, {lambda x: re.fullmatch(self._UUID_RE, x)}, 0)) - or self._search_regex([ - rf'CWTV\.Site\.curPlayingGUID\s*=\s*["\']({self._UUID_RE})', - rf'CWTV\.Site\.viewInAppURL\s*=\s*["\']/shows/[\w-]+/watch-in-app/\?play=({self._UUID_RE})', - ], webpage, 'video ID')) - - return self.url_result( - f'https://www.cwtv.com/shows/{display_id}/{display_id}/?play={video_id}', CWTVIE, video_id) diff --git a/yt_dlp/extractor/paramountplus.py b/yt_dlp/extractor/paramountplus.py deleted file mode 100644 index 317f53b2bc..0000000000 --- a/yt_dlp/extractor/paramountplus.py +++ /dev/null @@ -1,201 +0,0 @@ -import itertools - -from .cbs import CBSBaseIE -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - int_or_none, - url_or_none, -) - - -class ParamountPlusIE(CBSBaseIE): - _VALID_URL = r'''(?x) - (?: - paramountplus:| - https?://(?:www\.)?(?: - paramountplus\.com/(?:shows|movies)/(?:video|[^/]+/video|[^/]+)/ - )(?P[\w-]+))''' - - # All tests are blocked outside US - _TESTS = [{ - 'url': 'https://www.paramountplus.com/shows/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/', - 'info_dict': { - 'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k', - 'ext': 'mp4', - 'title': 'CatDog - Climb Every CatDog/The Canine Mutiny', - 'description': 'md5:7ac835000645a69933df226940e3c859', - 'duration': 1426, - 'timestamp': 920264400, - 'upload_date': '19990301', - 'uploader': 'CBSI-NEW', - 'episode_number': 5, - 'thumbnail': r're:https?://.+\.jpg$', - 'season': 'Season 2', - 'chapters': 'count:3', - 'episode': 'Episode 5', - 'season_number': 2, - 'series': 'CatDog', - }, - 'params': { - 'skip_download': 'm3u8', - }, - }, { - 'url': 'https://www.paramountplus.com/shows/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/', - 'info_dict': { - 'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd', - 'ext': 'mp4', - 'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)', - 'description': 'md5:f4adcea3e8b106192022e121f1565bae', - 'duration': 2506, - 'timestamp': 1627063200, - 'upload_date': '20210723', - 'uploader': 'CBSI-NEW', - 'episode_number': 81, - 'thumbnail': r're:https?://.+\.jpg$', - 'season': 'Season 2', - 'chapters': 'count:4', - 'episode': 'Episode 81', - 'season_number': 2, - 'series': 'Tooning Out The News', - }, - 'params': { - 'skip_download': 'm3u8', - }, - }, { - 'url': 'https://www.paramountplus.com/movies/video/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC/', - 'info_dict': { - 'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', - 'ext': 'mp4', - 'title': 'Daddy\'s Home', - 'upload_date': '20151225', - 'description': 'md5:9a6300c504d5e12000e8707f20c54745', - 'uploader': 'CBSI-NEW', - 'timestamp': 1451030400, - 'thumbnail': r're:https?://.+\.jpg$', - 'chapters': 'count:0', - 'duration': 5761, - 'series': 'Paramount+ Movies', - }, - 'params': { - 'skip_download': 'm3u8', - }, - 'skip': 'DRM', - }, { - 'url': 'https://www.paramountplus.com/movies/video/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc/', - 'info_dict': { - 'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', - 'ext': 'mp4', - 'uploader': 'CBSI-NEW', - 'description': 'md5:bc7b6fea84ba631ef77a9bda9f2ff911', - 'timestamp': 1577865600, - 'title': 'Sonic the Hedgehog', - 'upload_date': '20200101', - 'thumbnail': r're:https?://.+\.jpg$', - 'chapters': 'count:0', - 'duration': 5932, - 'series': 'Paramount+ Movies', - }, - 'params': { - 'skip_download': 'm3u8', - }, - 'skip': 'DRM', - }, { - 'url': 'https://www.paramountplus.com/shows/the-real-world/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/the-real-world-reunion/', - 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/shows/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/', - 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/movies/video/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/', - 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/movies/paw-patrol-the-movie/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/', - 'only_matching': True, - }] - - def _extract_video_info(self, content_id, mpx_acc=2198311517): - items_data = self._download_json( - f'https://www.paramountplus.com/apps-api/v2.0/androidtv/video/cid/{content_id}.json', - content_id, query={ - 'locale': 'en-us', - 'at': 'ABCXgPuoStiPipsK0OHVXIVh68zNys+G4f7nW9R6qH68GDOcneW6Kg89cJXGfiQCsj0=', - }, headers=self.geo_verification_headers()) - - asset_types = { - item.get('assetType'): { - 'format': 'SMIL', - 'formats': 'M3U+none,MPEG4', # '+none' specifies ProtectionScheme (no DRM) - } for item in items_data['itemList'] - } - item = items_data['itemList'][-1] - - info, error = {}, None - metadata = { - 'title': item.get('title'), - 'series': item.get('seriesTitle'), - 'season_number': int_or_none(item.get('seasonNum')), - 'episode_number': int_or_none(item.get('episodeNum')), - 'duration': int_or_none(item.get('duration')), - 'thumbnail': url_or_none(item.get('thumbnail')), - } - try: - info = self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info=metadata) - except ExtractorError as e: - error = e - - # Check for DRM formats to give appropriate error - if not info.get('formats'): - for query in asset_types.values(): - query['formats'] = 'MPEG-DASH,M3U,MPEG4' # allows DRM formats - - try: - drm_info = self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info=metadata) - except ExtractorError: - if error: - raise error from None - raise - if drm_info['formats']: - self.report_drm(content_id) - elif error: - raise error - - return info - - -class ParamountPlusSeriesIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P[a-zA-Z0-9-_]+)/?(?:[#?]|$)' - _TESTS = [{ - 'url': 'https://www.paramountplus.com/shows/drake-josh', - 'playlist_mincount': 50, - 'info_dict': { - 'id': 'drake-josh', - }, - }, { - 'url': 'https://www.paramountplus.com/shows/hawaii_five_0/', - 'playlist_mincount': 240, - 'info_dict': { - 'id': 'hawaii_five_0', - }, - }, { - 'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/', - 'playlist_mincount': 248, - 'info_dict': { - 'id': 'spongebob-squarepants', - }, - }] - - def _entries(self, show_name): - for page in itertools.count(): - show_json = self._download_json( - f'https://www.paramountplus.com/shows/{show_name}/xhr/episodes/page/{page}/size/50/xs/0/season/0', show_name) - if not show_json.get('success'): - return - for episode in show_json['result']['data']: - yield self.url_result( - 'https://www.paramountplus.com{}'.format(episode['url']), - ie=ParamountPlusIE.ie_key(), video_id=episode['content_id']) - - def _real_extract(self, url): - show_name = self._match_id(url) - return self.playlist_result(self._entries(show_name), playlist_id=show_name) diff --git a/yt_dlp/extractor/sixplay.py b/yt_dlp/extractor/sixplay.py deleted file mode 100644 index 6037a35116..0000000000 --- a/yt_dlp/extractor/sixplay.py +++ /dev/null @@ -1,119 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - determine_ext, - int_or_none, - parse_qs, - qualities, - try_get, -) - - -class SixPlayIE(InfoExtractor): - IE_NAME = '6play' - _VALID_URL = r'(?:6play:|https?://(?:www\.)?(?P6play\.fr|rtlplay\.be|play\.rtl\.hr|rtlmost\.hu)/.+?-c_)(?P[0-9]+)' - _TESTS = [{ - 'url': 'https://www.6play.fr/minute-par-minute-p_9533/le-but-qui-a-marque-lhistoire-du-football-francais-c_12041051', - 'md5': '31fcd112637baa0c2ab92c4fcd8baf27', - 'info_dict': { - 'id': '12041051', - 'ext': 'mp4', - 'title': 'Le but qui a marqué l\'histoire du football français !', - 'description': 'md5:b59e7e841d646ef1eb42a7868eb6a851', - }, - }, { - 'url': 'https://www.rtlplay.be/rtl-info-13h-p_8551/les-titres-du-rtlinfo-13h-c_12045869', - 'only_matching': True, - }, { - 'url': 'https://play.rtl.hr/pj-masks-p_9455/epizoda-34-sezona-1-catboyevo-cudo-na-dva-kotaca-c_11984989', - 'only_matching': True, - }, { - 'url': 'https://www.rtlmost.hu/megtorve-p_14167/megtorve-6-resz-c_12397787', - 'only_matching': True, - }] - - def _real_extract(self, url): - domain, video_id = self._match_valid_url(url).groups() - service, consumer_name = { - '6play.fr': ('6play', 'm6web'), - 'rtlplay.be': ('rtlbe_rtl_play', 'rtlbe'), - 'play.rtl.hr': ('rtlhr_rtl_play', 'rtlhr'), - 'rtlmost.hu': ('rtlhu_rtl_most', 'rtlhu'), - }.get(domain, ('6play', 'm6web')) - - data = self._download_json( - f'https://pc.middleware.6play.fr/6play/v2/platforms/m6group_web/services/{service}/videos/clip_{video_id}', - video_id, headers={ - 'x-customer-name': consumer_name, - }, query={ - 'csa': 5, - 'with': 'clips', - }) - - clip_data = data['clips'][0] - title = clip_data['title'] - - urls = [] - quality_key = qualities(['lq', 'sd', 'hq', 'hd']) - formats = [] - subtitles = {} - assets = clip_data.get('assets') or [] - for asset in assets: - asset_url = asset.get('full_physical_path') - protocol = asset.get('protocol') - if not asset_url or ((protocol == 'primetime' or asset.get('type') == 'usp_hlsfp_h264') and not ('_drmnp.ism/' in asset_url or '_unpnp.ism/' in asset_url)) or asset_url in urls: - continue - urls.append(asset_url) - container = asset.get('video_container') - ext = determine_ext(asset_url) - if protocol == 'http_subtitle' or ext == 'vtt': - subtitles.setdefault('fr', []).append({'url': asset_url}) - continue - if container == 'm3u8' or ext == 'm3u8': - if protocol == 'usp': - if parse_qs(asset_url).get('token', [None])[0]: - urlh = self._request_webpage( - asset_url, video_id, fatal=False, - headers=self.geo_verification_headers()) - if not urlh: - continue - asset_url = urlh.url - asset_url = asset_url.replace('_drmnp.ism/', '_unpnp.ism/') - for i in range(3, 0, -1): - asset_url = asset_url.replace('_sd1/', f'_sd{i}/') - m3u8_formats = self._extract_m3u8_formats( - asset_url, video_id, 'mp4', 'm3u8_native', - m3u8_id='hls', fatal=False) - formats.extend(m3u8_formats) - formats.extend(self._extract_mpd_formats( - asset_url.replace('.m3u8', '.mpd'), - video_id, mpd_id='dash', fatal=False)) - if m3u8_formats: - break - else: - formats.extend(self._extract_m3u8_formats( - asset_url, video_id, 'mp4', 'm3u8_native', - m3u8_id='hls', fatal=False)) - elif container == 'mp4' or ext == 'mp4': - quality = asset.get('video_quality') - formats.append({ - 'url': asset_url, - 'format_id': quality, - 'quality': quality_key(quality), - 'ext': ext, - }) - - def get(getter): - for src in (data, clip_data): - v = try_get(src, getter, str) - if v: - return v - - return { - 'id': video_id, - 'title': title, - 'description': get(lambda x: x['description']), - 'duration': int_or_none(clip_data.get('duration')), - 'series': get(lambda x: x['program']['title']), - 'formats': formats, - 'subtitles': subtitles, - } diff --git a/yt_dlp/extractor/spotify.py b/yt_dlp/extractor/spotify.py deleted file mode 100644 index de67a61148..0000000000 --- a/yt_dlp/extractor/spotify.py +++ /dev/null @@ -1,167 +0,0 @@ -import functools -import json -import re - -from .common import InfoExtractor -from ..utils import ( - OnDemandPagedList, - clean_podcast_url, - float_or_none, - int_or_none, - strip_or_none, - traverse_obj, - try_get, - unified_strdate, -) - - -class SpotifyBaseIE(InfoExtractor): - _WORKING = False - _ACCESS_TOKEN = None - _OPERATION_HASHES = { - 'Episode': '8276d4423d709ae9b68ec1b74cc047ba0f7479059a37820be730f125189ac2bf', - 'MinimalShow': '13ee079672fad3f858ea45a55eb109553b4fb0969ed793185b2e34cbb6ee7cc0', - 'ShowEpisodes': 'e0e5ce27bd7748d2c59b4d44ba245a8992a05be75d6fabc3b20753fc8857444d', - } - _VALID_URL_TEMPL = r'https?://open\.spotify\.com/(?:embed-podcast/|embed/|)%s/(?P[^/?&#]+)' - _EMBED_REGEX = [r']+src="(?Phttps?://open\.spotify.com/embed/[^"]+)"'] - - def _real_initialize(self): - self._ACCESS_TOKEN = self._download_json( - 'https://open.spotify.com/get_access_token', None)['accessToken'] - - def _call_api(self, operation, video_id, variables, **kwargs): - return self._download_json( - 'https://api-partner.spotify.com/pathfinder/v1/query', video_id, query={ - 'operationName': 'query' + operation, - 'variables': json.dumps(variables), - 'extensions': json.dumps({ - 'persistedQuery': { - 'sha256Hash': self._OPERATION_HASHES[operation], - }, - }), - }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN}, - **kwargs)['data'] - - def _extract_episode(self, episode, series): - episode_id = episode['id'] - title = episode['name'].strip() - - formats = [] - audio_preview = episode.get('audioPreview') or {} - audio_preview_url = audio_preview.get('url') - if audio_preview_url: - f = { - 'url': audio_preview_url.replace('://p.scdn.co/mp3-preview/', '://anon-podcast.scdn.co/'), - 'vcodec': 'none', - } - audio_preview_format = audio_preview.get('format') - if audio_preview_format: - f['format_id'] = audio_preview_format - mobj = re.match(r'([0-9A-Z]{3})_(?:[A-Z]+_)?(\d+)', audio_preview_format) - if mobj: - f.update({ - 'abr': int(mobj.group(2)), - 'ext': mobj.group(1).lower(), - }) - formats.append(f) - - for item in (try_get(episode, lambda x: x['audio']['items']) or []): - item_url = item.get('url') - if not (item_url and item.get('externallyHosted')): - continue - formats.append({ - 'url': clean_podcast_url(item_url), - 'vcodec': 'none', - }) - - thumbnails = [] - for source in (try_get(episode, lambda x: x['coverArt']['sources']) or []): - source_url = source.get('url') - if not source_url: - continue - thumbnails.append({ - 'url': source_url, - 'width': int_or_none(source.get('width')), - 'height': int_or_none(source.get('height')), - }) - - return { - 'id': episode_id, - 'title': title, - 'formats': formats, - 'thumbnails': thumbnails, - 'description': strip_or_none(episode.get('description')), - 'duration': float_or_none(try_get( - episode, lambda x: x['duration']['totalMilliseconds']), 1000), - 'release_date': unified_strdate(try_get( - episode, lambda x: x['releaseDate']['isoString'])), - 'series': series, - } - - -class SpotifyIE(SpotifyBaseIE): - IE_NAME = 'spotify' - IE_DESC = 'Spotify episodes' - _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'episode' - _TESTS = [{ - 'url': 'https://open.spotify.com/episode/4Z7GAJ50bgctf6uclHlWKo', - 'md5': '74010a1e3fa4d9e1ab3aa7ad14e42d3b', - 'info_dict': { - 'id': '4Z7GAJ50bgctf6uclHlWKo', - 'ext': 'mp3', - 'title': 'From the archive: Why time management is ruining our lives', - 'description': 'md5:b120d9c4ff4135b42aa9b6d9cde86935', - 'duration': 2083.605, - 'release_date': '20201217', - 'series': "The Guardian's Audio Long Reads", - }, - }, { - 'url': 'https://open.spotify.com/embed/episode/4TvCsKKs2thXmarHigWvXE?si=7eatS8AbQb6RxqO2raIuWA', - 'only_matching': True, - }] - - def _real_extract(self, url): - episode_id = self._match_id(url) - episode = self._call_api('Episode', episode_id, { - 'uri': 'spotify:episode:' + episode_id, - })['episode'] - return self._extract_episode( - episode, try_get(episode, lambda x: x['podcast']['name'])) - - -class SpotifyShowIE(SpotifyBaseIE): - IE_NAME = 'spotify:show' - IE_DESC = 'Spotify shows' - _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'show' - _TEST = { - 'url': 'https://open.spotify.com/show/4PM9Ke6l66IRNpottHKV9M', - 'info_dict': { - 'id': '4PM9Ke6l66IRNpottHKV9M', - 'title': 'The Story from the Guardian', - 'description': 'The Story podcast is dedicated to our finest audio documentaries, investigations and long form stories', - }, - 'playlist_mincount': 36, - } - _PER_PAGE = 100 - - def _fetch_page(self, show_id, page=0): - return self._call_api('ShowEpisodes', show_id, { - 'limit': 100, - 'offset': page * self._PER_PAGE, - 'uri': f'spotify:show:{show_id}', - }, note=f'Downloading page {page + 1} JSON metadata')['podcast'] - - def _real_extract(self, url): - show_id = self._match_id(url) - first_page = self._fetch_page(show_id) - - def _entries(page): - podcast = self._fetch_page(show_id, page) if page else first_page - yield from map( - functools.partial(self._extract_episode, series=podcast.get('name')), - traverse_obj(podcast, ('episodes', 'items', ..., 'episode'))) - - return self.playlist_result( - OnDemandPagedList(_entries, self._PER_PAGE), - show_id, first_page.get('name'), first_page.get('description')) diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py index 05ae4dd18a..4857156913 100644 --- a/yt_dlp/extractor/unsupported.py +++ b/yt_dlp/extractor/unsupported.py @@ -30,13 +30,13 @@ class KnownDRMIE(UnsupportedInfoExtractor): r'play\.hbomax\.com', r'channel(?:4|5)\.com', r'peacocktv\.com', - r'(?:[\w\.]+\.)?disneyplus\.com', - r'open\.spotify\.com/(?:track|playlist|album|artist)', + r'(?:[\w.]+\.)?disneyplus\.com', + r'open\.spotify\.com', r'tvnz\.co\.nz', r'oneplus\.ch', r'artstation\.com/learning/courses', r'philo\.com', - r'(?:[\w\.]+\.)?mech-plus\.com', + r'(?:[\w.]+\.)?mech-plus\.com', r'aha\.video', r'mubi\.com', r'vootkids\.com', @@ -57,6 +57,14 @@ class KnownDRMIE(UnsupportedInfoExtractor): r'ctv\.ca', r'noovo\.ca', r'tsn\.ca', + r'paramountplus\.com', + r'(?:m\.)?(?:sony)?crackle\.com', + r'cw(?:tv(?:pr)?|seed)\.com', + r'6play\.fr', + r'rtlplay\.be', + r'play\.rtl\.hr', + r'rtlmost\.hu', + r'plus\.rtl\.de(?!/podcast/)', ) _TESTS = [{ @@ -78,10 +86,7 @@ class KnownDRMIE(UnsupportedInfoExtractor): 'url': r'https://www.disneyplus.com', 'only_matching': True, }, { - 'url': 'https://open.spotify.com/artist/', - 'only_matching': True, - }, { - 'url': 'https://open.spotify.com/track/', + 'url': 'https://open.spotify.com', 'only_matching': True, }, { # https://github.com/yt-dlp/yt-dlp/issues/4122 @@ -184,6 +189,39 @@ class KnownDRMIE(UnsupportedInfoExtractor): }, { 'url': 'https://www.tsn.ca/video/relaxed-oilers-look-to-put-emotional-game-2-loss-in-the-rearview%7E3148747', 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com', + 'only_matching': True, + }, { + 'url': 'https://www.crackle.com', + 'only_matching': True, + }, { + 'url': 'https://m.sonycrackle.com', + 'only_matching': True, + }, { + 'url': 'https://www.cwtv.com', + 'only_matching': True, + }, { + 'url': 'https://www.cwseed.com', + 'only_matching': True, + }, { + 'url': 'https://cwtvpr.com', + 'only_matching': True, + }, { + 'url': 'https://www.6play.fr', + 'only_matching': True, + }, { + 'url': 'https://www.rtlplay.be', + 'only_matching': True, + }, { + 'url': 'https://play.rtl.hr', + 'only_matching': True, + }, { + 'url': 'https://www.rtlmost.hu', + 'only_matching': True, + }, { + 'url': 'https://plus.rtl.de/video-tv/', + 'only_matching': True, }] def _real_extract(self, url): @@ -222,6 +260,7 @@ class KnownPiracyIE(UnsupportedInfoExtractor): r'91porn\.com', r'einthusan\.(?:tv|com|ca)', r'yourupload\.com', + r'xanimu\.com', ) _TESTS = [{ diff --git a/yt_dlp/extractor/xanimu.py b/yt_dlp/extractor/xanimu.py deleted file mode 100644 index b489358779..0000000000 --- a/yt_dlp/extractor/xanimu.py +++ /dev/null @@ -1,52 +0,0 @@ -import re - -from .common import InfoExtractor -from ..utils import int_or_none - - -class XanimuIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?xanimu\.com/(?P[^/]+)/?' - _TESTS = [{ - 'url': 'https://xanimu.com/51944-the-princess-the-frog-hentai/', - 'md5': '899b88091d753d92dad4cb63bbf357a7', - 'info_dict': { - 'id': '51944-the-princess-the-frog-hentai', - 'ext': 'mp4', - 'title': 'The Princess + The Frog Hentai', - 'thumbnail': 'https://xanimu.com/storage/2020/09/the-princess-and-the-frog-hentai.jpg', - 'description': r're:^Enjoy The Princess \+ The Frog Hentai', - 'duration': 207.0, - 'age_limit': 18, - }, - }, { - 'url': 'https://xanimu.com/huge-expansion/', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - formats = [] - for format_id in ['videoHigh', 'videoLow']: - format_url = self._search_json( - rf'var\s+{re.escape(format_id)}\s*=', webpage, format_id, - video_id, default=None, contains_pattern=r'[\'"]([^\'"]+)[\'"]') - if format_url: - formats.append({ - 'url': format_url, - 'format_id': format_id, - 'quality': -2 if format_id.endswith('Low') else None, - }) - - return { - 'id': video_id, - 'formats': formats, - 'title': self._search_regex(r'[\'"]headline[\'"]:\s*[\'"]([^"]+)[\'"]', webpage, - 'title', default=None) or self._html_extract_title(webpage), - 'thumbnail': self._html_search_meta('thumbnailUrl', webpage, default=None), - 'description': self._html_search_meta('description', webpage, default=None), - 'duration': int_or_none(self._search_regex(r'duration:\s*[\'"]([^\'"]+?)[\'"]', - webpage, 'duration', fatal=False)), - 'age_limit': 18, - } From eb4b3a5fc7765a6cd0370ca44ccee0d7d5111dd7 Mon Sep 17 00:00:00 2001 From: sepro Date: Sat, 13 Sep 2025 21:57:54 +0200 Subject: [PATCH 039/175] [cleanup] Remove setup.cfg (#14314) Authored by: seproDev --- setup.cfg | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d6541c8516..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,39 +0,0 @@ -[flake8] -exclude = build,venv,.tox,.git,.pytest_cache -ignore = E402,E501,E731,E741,W503 -max_line_length = 120 -per_file_ignores = - devscripts/lazy_load_template.py: F401 - - -[autoflake] -ignore-init-module-imports = true -ignore-pass-after-docstring = true -remove-all-unused-imports = true -remove-duplicate-keys = true -remove-unused-variables = true - - -[tox:tox] -skipsdist = true -envlist = py{39,310,311,312,313,314},pypy311 -skip_missing_interpreters = true - -[testenv] # tox -deps = - pytest -commands = pytest {posargs:"-m not download"} -passenv = HOME # For test_compat_expanduser -setenv = - # PYTHONWARNINGS = error # Catches PIP's warnings too - - -[isort] -py_version = 39 -multi_line_output = VERTICAL_HANGING_INDENT -line_length = 80 -reverse_relative = true -ensure_newline_before_comments = true -include_trailing_comma = true -known_first_party = - test From e2d37bcc8e84be9ce0f67fc24cb830c13963d10f Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:20:28 -0500 Subject: [PATCH 040/175] [build] Refactor Linux build jobs (#14275) Authored by: bashonly --- .github/workflows/build.yml | 207 ++++++++++++++++------------------ bundle/docker/compose.yml | 29 ++++- bundle/docker/linux/build.sh | 9 +- bundle/docker/linux/verify.sh | 23 ++-- 4 files changed, 141 insertions(+), 127 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e382f9007..5371bf140e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,6 +84,8 @@ jobs: origin: ${{ steps.process_inputs.outputs.origin }} timestamp: ${{ steps.process_inputs.outputs.timestamp }} version: ${{ steps.process_inputs.outputs.version }} + linux_matrix: ${{ steps.linux_matrix.outputs.matrix }} + steps: - name: Process inputs id: process_inputs @@ -118,6 +120,69 @@ jobs: with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write('\n'.join(f'{key}={value}' for key, value in outputs.items())) + - name: Build Linux matrix + id: linux_matrix + env: + INPUTS: ${{ toJSON(inputs) }} + PYTHON_VERSION: '3.13' + UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 + shell: python + run: | + import json + import os + EXE_MAP = { + 'linux': [{ + 'os': 'linux', + 'arch': 'x86_64', + 'runner': 'ubuntu-24.04', + }, { + 'os': 'linux', + 'arch': 'aarch64', + 'runner': 'ubuntu-24.04-arm', + }], + 'linux_armv7l': [{ + 'os': 'linux', + 'arch': 'armv7l', + 'runner': 'ubuntu-24.04-arm', + 'qemu_platform': 'linux/arm/v7', + 'onefile': False, + 'cache_requirements': True, + 'update_to': 'yt-dlp/yt-dlp@2023.03.04', + }], + 'musllinux': [{ + 'os': 'musllinux', + 'arch': 'x86_64', + 'runner': 'ubuntu-24.04', + }, { + 'os': 'musllinux', + 'arch': 'aarch64', + 'runner': 'ubuntu-24.04-arm', + }], + } + INPUTS = json.loads(os.environ['INPUTS']) + matrix = [exe for key, group in EXE_MAP.items() for exe in group if INPUTS.get(key)] + if not matrix: + # If we send an empty matrix when no linux inputs are given, the entire workflow fails + matrix = [EXE_MAP['linux'][0]] + for exe in matrix: + exe['exe'] = '_'.join(filter(None, ( + 'yt-dlp', + exe['os'], + exe['arch'] != 'x86_64' and exe['arch'], + ))) + exe.setdefault('qemu_platform', None) + exe.setdefault('onefile', True) + exe.setdefault('onedir', True) + exe.setdefault('cache_requirements', False) + exe.setdefault('python_version', os.environ['PYTHON_VERSION']) + exe.setdefault('update_to', os.environ['UPDATE_TO']) + if not any(INPUTS.get(key) for key in EXE_MAP): + print('skipping linux job') + else: + print(json.dumps(matrix, indent=2)) + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'matrix={json.dumps(matrix)}') + unix: needs: process if: inputs.unix @@ -127,24 +192,30 @@ jobs: ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 + steps: - uses: actions/checkout@v4 with: fetch-depth: 0 # Needed for changelog + - uses: actions/setup-python@v6 with: python-version: "3.10" + - name: Install Requirements run: | sudo apt -y install zip pandoc man sed + - name: Prepare run: | python devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" python devscripts/update_changelog.py -vv python devscripts/make_lazy_extractors.py + - name: Build Unix platform-independent binary run: | make all tar + - name: Verify --update-to if: vars.UPDATE_TO_VERIFICATION run: | @@ -154,6 +225,7 @@ jobs: ./yt-dlp_downgraded -v --update-to "${UPDATE_TO}" downgraded_version="$(./yt-dlp_downgraded --version)" [[ "${version}" != "${downgraded_version}" ]] + - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -164,154 +236,71 @@ jobs: compression-level: 0 linux: + name: ${{ matrix.os }} (${{ matrix.arch }}) + if: inputs.linux || inputs.linux_armv7l || inputs.musllinux needs: process - if: inputs.linux runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - include: - - exe: yt-dlp_linux - platform: x86_64 - runner: ubuntu-24.04 - - exe: yt-dlp_linux_aarch64 - platform: aarch64 - runner: ubuntu-24.04-arm + include: ${{ fromJSON(needs.process.outputs.linux_matrix) }} env: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} VERSION: ${{ needs.process.outputs.version }} EXE_NAME: ${{ matrix.exe }} - UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 - steps: - - uses: actions/checkout@v4 - - name: Build executable - env: - SERVICE: linux_${{ matrix.platform }} - run: | - mkdir -p ./dist - pushd bundle/docker - docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - popd - sudo chown "${USER}:docker" "./dist/${EXE_NAME}" - - name: Verify executable in container - if: vars.UPDATE_TO_VERIFICATION - env: - SERVICE: linux_${{ matrix.platform }}_verify - run: | - cd bundle/docker - docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - - name: Verify --update-to - if: vars.UPDATE_TO_VERIFICATION - run: | - chmod +x "./dist/${EXE_NAME}" - mkdir -p ~/testing - cp "./dist/${EXE_NAME}" ~/testing/"${EXE_NAME}_downgraded" - version="$("./dist/${EXE_NAME}" --version)" - ~/testing/"${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}" - downgraded_version="$(~/testing/"${EXE_NAME}_downgraded" --version)" - [[ "${version}" != "${downgraded_version}" ]] - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: build-bin-${{ github.job }}_${{ matrix.platform }} - path: | - dist/${{ matrix.exe }}* - compression-level: 0 + PYTHON_VERSION: ${{ matrix.python_version }} + UPDATE_TO: ${{ (vars.UPDATE_TO_VERIFICATION && matrix.update_to) || '' }} + SKIP_ONEDIR_BUILD: ${{ (!matrix.onedir && '1') || '' }} + SKIP_ONEFILE_BUILD: ${{ (!matrix.onefile && '1') || '' }} - linux_armv7l: - needs: process - if: inputs.linux_armv7l - permissions: - contents: read - runs-on: ubuntu-24.04-arm - env: - CHANNEL: ${{ inputs.channel }} - ORIGIN: ${{ needs.process.outputs.origin }} - VERSION: ${{ needs.process.outputs.version }} - EXE_NAME: yt-dlp_linux_armv7l steps: - uses: actions/checkout@v4 + - name: Cache requirements + if: matrix.cache_requirements id: cache-venv uses: actions/cache@v4 env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 with: path: | - ~/yt-dlp-build-venv - key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} + venv + key: cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} restore-keys: | - cache-reqs-${{ github.job }}-${{ github.ref }}- - cache-reqs-${{ github.job }}- + cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}- + cache-reqs-${{ matrix.os }}_${{ matrix.arch }}- + - name: Set up QEMU + if: matrix.qemu_platform uses: docker/setup-qemu-action@v3 with: - platforms: linux/arm/v7 - - name: Build executable - env: - SERVICE: linux_armv7l - run: | - mkdir -p ./dist - mkdir -p ~/yt-dlp-build-venv - cd bundle/docker - docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - - name: Verify executable in container - if: vars.UPDATE_TO_VERIFICATION - env: - SERVICE: linux_armv7l_verify - run: | - cd bundle/docker - docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: build-bin-${{ github.job }} - path: | - dist/yt-dlp_linux_armv7l.zip - compression-level: 0 + platforms: ${{ matrix.qemu_platform }} - musllinux: - needs: process - if: inputs.musllinux - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - exe: yt-dlp_musllinux - platform: x86_64 - runner: ubuntu-24.04 - - exe: yt-dlp_musllinux_aarch64 - platform: aarch64 - runner: ubuntu-24.04-arm - env: - CHANNEL: ${{ inputs.channel }} - ORIGIN: ${{ needs.process.outputs.origin }} - VERSION: ${{ needs.process.outputs.version }} - EXE_NAME: ${{ matrix.exe }} - steps: - - uses: actions/checkout@v4 - name: Build executable env: - SERVICE: musllinux_${{ matrix.platform }} + SERVICE: ${{ matrix.os }}_${{ matrix.arch }} run: | + mkdir -p ./venv mkdir -p ./dist pushd bundle/docker docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" popd - sudo chown "${USER}:docker" "./dist/${EXE_NAME}" + if [[ -z "${SKIP_ONEFILE_BUILD}" ]]; then + sudo chown "${USER}:docker" "./dist/${EXE_NAME}" + fi + - name: Verify executable in container - if: vars.UPDATE_TO_VERIFICATION env: - SERVICE: musllinux_${{ matrix.platform }}_verify + SERVICE: ${{ matrix.os }}_${{ matrix.arch }}_verify run: | cd bundle/docker docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-bin-${{ github.job }}_${{ matrix.platform }} + name: build-bin-${{ matrix.os }}_${{ matrix.arch }} path: | dist/${{ matrix.exe }}* compression-level: 0 @@ -542,8 +531,6 @@ jobs: - process - unix - linux - - linux_armv7l - - musllinux - macos - windows runs-on: ubuntu-latest diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml index 77062f594a..19a011d7a2 100644 --- a/bundle/docker/compose.yml +++ b/bundle/docker/compose.yml @@ -13,6 +13,9 @@ services: CHANNEL: ${CHANNEL:?} ORIGIN: ${ORIGIN:?} VERSION: + PYTHON_VERSION: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../..:/yt-dlp @@ -27,6 +30,8 @@ services: environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../../dist:/build @@ -43,6 +48,9 @@ services: CHANNEL: ${CHANNEL:?} ORIGIN: ${ORIGIN:?} VERSION: + PYTHON_VERSION: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../..:/yt-dlp @@ -57,6 +65,8 @@ services: environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../../dist:/build @@ -73,10 +83,12 @@ services: CHANNEL: ${CHANNEL:?} ORIGIN: ${ORIGIN:?} VERSION: - SKIP_ONEFILE_BUILD: "1" + PYTHON_VERSION: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../..:/yt-dlp - - ~/yt-dlp-build-venv:/yt-dlp-build-venv + - ../../venv:/yt-dlp-build-venv linux_armv7l_verify: build: @@ -89,7 +101,8 @@ services: environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: - TEST_ONEDIR_BUILD: "1" + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../../dist:/build @@ -106,6 +119,9 @@ services: CHANNEL: ${CHANNEL:?} ORIGIN: ${ORIGIN:?} VERSION: + PYTHON_VERSION: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../..:/yt-dlp @@ -120,6 +136,8 @@ services: environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../../dist:/build @@ -136,6 +154,9 @@ services: CHANNEL: ${CHANNEL:?} ORIGIN: ${ORIGIN:?} VERSION: + PYTHON_VERSION: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: EXCLUDE_CURL_CFFI: "1" volumes: - ../..:/yt-dlp @@ -151,5 +172,7 @@ services: environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: + SKIP_ONEDIR_BUILD: + SKIP_ONEFILE_BUILD: volumes: - ../../dist:/build diff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.sh index 1ce330a5b0..b278ebf60f 100755 --- a/bundle/docker/linux/build.sh +++ b/bundle/docker/linux/build.sh @@ -1,16 +1,17 @@ #!/bin/bash set -exuo pipefail -if [[ -z "${USE_PYTHON_VERSION:-}" ]]; then - USE_PYTHON_VERSION="3.13" +if [[ -z "${PYTHON_VERSION:-}" ]]; then + PYTHON_VERSION="3.13" + echo "Defaulting to using Python ${PYTHON_VERSION}" fi function runpy { - "/opt/shared-cpython-${USE_PYTHON_VERSION}/bin/python${USE_PYTHON_VERSION}" "$@" + "/opt/shared-cpython-${PYTHON_VERSION}/bin/python${PYTHON_VERSION}" "$@" } function venvpy { - "python${USE_PYTHON_VERSION}" "$@" + "python${PYTHON_VERSION}" "$@" } INCLUDES=( diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.sh index 062a576f9d..ecb071a8c5 100755 --- a/bundle/docker/linux/verify.sh +++ b/bundle/docker/linux/verify.sh @@ -1,7 +1,11 @@ #!/bin/sh set -eu -if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then +if [ -n "${SKIP_ONEFILE_BUILD:-}" ]; then + if [ -n "${SKIP_ONEDIR_BUILD:-}" ]; then + echo "All executable builds were skipped" + exit 1 + fi echo "Extracting zip to verify onedir build" if command -v python3 >/dev/null 2>&1; then python3 -m zipfile -e "/build/${EXE_NAME}.zip" ./ @@ -22,21 +26,20 @@ if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then fi unzip "/build/${EXE_NAME}.zip" -d ./ fi -else - echo "Verifying onefile build" - cp "/build/${EXE_NAME}" ./ -fi - -chmod +x "./${EXE_NAME}" - -if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then + chmod +x "./${EXE_NAME}" "./${EXE_NAME}" -v || true "./${EXE_NAME}" --version exit 0 fi +echo "Verifying onefile build" +cp "/build/${EXE_NAME}" ./ +chmod +x "./${EXE_NAME}" + if [ -z "${UPDATE_TO:-}" ]; then - UPDATE_TO="yt-dlp/yt-dlp@2025.09.05" + "./${EXE_NAME}" -v || true + "./${EXE_NAME}" --version + exit 0 fi cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded" From 8ab262c66bd3e1d8874fb2d070068ba1f0d48f16 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:30:06 -0500 Subject: [PATCH 041/175] [cleanup] Remove references to setup.cfg (#14315) Fix eb4b3a5fc7765a6cd0370ca44ccee0d7d5111dd7 Authored by: bashonly --- Makefile | 4 ++-- pyproject.toml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 273cb3cc0b..404250c815 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ tar: yt-dlp.tar.gz # intended use: when building a source distribution, # make pypi-files && python3 -m build -sn . pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \ - completions yt-dlp.1 pyproject.toml setup.cfg devscripts/* test/* + completions yt-dlp.1 pyproject.toml devscripts/* test/* .PHONY: all clean clean-all clean-test clean-dist clean-cache \ completions completion-bash completion-fish completion-zsh \ @@ -159,7 +159,7 @@ yt-dlp.tar.gz: all README.md supportedsites.md Changelog.md LICENSE \ CONTRIBUTING.md Collaborators.md CONTRIBUTORS AUTHORS \ Makefile yt-dlp.1 README.txt completions .gitignore \ - setup.cfg yt-dlp yt_dlp pyproject.toml devscripts test + yt-dlp yt_dlp pyproject.toml devscripts test AUTHORS: Changelog.md @if [ -d '.git' ] && command -v git > /dev/null ; then \ diff --git a/pyproject.toml b/pyproject.toml index 9b4ff20ba1..3438140525 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,6 @@ include = [ "/LICENSE", # included as license "/pyproject.toml", # included by default "/README.md", # included as readme - "/setup.cfg", "/supportedsites.md", ] artifacts = [ From ae3923b6b23bc62115be55510d6b5842f7a46b5f Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:55:35 -0500 Subject: [PATCH 042/175] [ci] Improve workflow checks (#14316) Authored by: bashonly --- .github/workflows/test-workflows.yml | 8 +++++++- bundle/docker/linux/build.sh | 1 + devscripts/setup_variables_tests.py | 9 +-------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 36ff86e48d..c07e01d7af 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -3,12 +3,14 @@ on: push: paths: - .github/workflows/* + - bundle/docker/linux/*.sh - devscripts/setup_variables.py - devscripts/setup_variables_tests.py - devscripts/utils.py pull_request: paths: - .github/workflows/* + - bundle/docker/linux/*.sh - devscripts/setup_variables.py - devscripts/setup_variables_tests.py - devscripts/utils.py @@ -32,6 +34,7 @@ jobs: env: ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }} run: | + python -m devscripts.install_deps -o --include test sudo apt -y install shellcheck python -m pip install -U pyflakes curl -LO "${ACTIONLINT_REPO}/releases/download/v${ACTIONLINT_VERSION}/${ACTIONLINT_TARBALL}" @@ -41,6 +44,9 @@ jobs: - name: Run actionlint run: | ./actionlint -color + - name: Check Docker shell scripts + run: | + shellcheck bundle/docker/linux/*.sh - name: Test GHA devscripts run: | - python -m devscripts.setup_variables_tests + pytest -Werror --tb=short devscripts/setup_variables_tests.py diff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.sh index b278ebf60f..71adaad058 100755 --- a/bundle/docker/linux/build.sh +++ b/bundle/docker/linux/build.sh @@ -24,6 +24,7 @@ if [[ -z "${EXCLUDE_CURL_CFFI:-}" ]]; then fi runpy -m venv /yt-dlp-build-venv +# shellcheck disable=SC1091 source /yt-dlp-build-venv/bin/activate # Inside the venv we use venvpy instead of runpy venvpy -m ensurepip --upgrade --default-pip diff --git a/devscripts/setup_variables_tests.py b/devscripts/setup_variables_tests.py index 8cb52daa1f..42abba9d1f 100644 --- a/devscripts/setup_variables_tests.py +++ b/devscripts/setup_variables_tests.py @@ -1,4 +1,3 @@ -# Allow direct execution import os import sys @@ -55,7 +54,7 @@ def _test(github_repository, note, repo_vars, repo_secrets, inputs, expected=Non assert result == exp, f'unexpected result: {github_repository} {note}' -def main(): +def test_setup_variables(): DEFAULT_VERSION_WITH_REVISION = dt.datetime.now(tz=dt.timezone.utc).strftime('%Y.%m.%d.%H%M%S') DEFAULT_VERSION = calculate_version() BASE_REPO_VARS = { @@ -323,9 +322,3 @@ def main(): 'pypi_project': None, 'pypi_suffix': None, }, ignore_revision=True) - - print('all tests passed') - - -if __name__ == '__main__': - main() From f3829463c728a5b5e62b3fc157e71c99b26edac7 Mon Sep 17 00:00:00 2001 From: sepro Date: Sun, 14 Sep 2025 00:32:25 +0200 Subject: [PATCH 043/175] [utils] `random_user_agent`: Bump versions (#14317) Authored by: seproDev --- yt_dlp/utils/networking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py index 467312ce75..44f877badb 100644 --- a/yt_dlp/utils/networking.py +++ b/yt_dlp/utils/networking.py @@ -17,7 +17,7 @@ from .traversal import traverse_obj def random_user_agent(): USER_AGENT_TMPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{} Safari/537.36' # Target versions released within the last ~6 months - CHROME_MAJOR_VERSION_RANGE = (132, 138) + CHROME_MAJOR_VERSION_RANGE = (134, 140) return USER_AGENT_TMPL.format(f'{random.randint(*CHROME_MAJOR_VERSION_RANGE)}.0.0.0') From df4b4e8ccf3385be6d2ad65465a0704c223dfdfb Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:50:34 -0500 Subject: [PATCH 044/175] [build] Use PyInstaller 6.16 for Windows (#14318) Authored by: bashonly --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5371bf140e..fda92cea88 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -420,23 +420,23 @@ jobs: runner: windows-2025 python_version: '3.10' platform_tag: win_amd64 - pyi_version: '6.15.0' - pyi_tag: '2025.09.08.215938' - pyi_hash: f70e327d849b29562caf01c30db60e6e8b2facb68124612bb53b04f96ffe6852 + pyi_version: '6.16.0' + pyi_tag: '2025.09.13.221251' + pyi_hash: b6496c7630c3afe66900cfa824e8234a8c2e2c81704bd7facd79586abc76c0e5 - arch: 'x86' runner: windows-2025 python_version: '3.10' platform_tag: win32 - pyi_version: '6.15.0' - pyi_tag: '2025.09.08.215938' - pyi_hash: b9af6b49a3556d478935de2632cb7dbef41a0c226f7a8ce36efc3ca2aeab3d51 + pyi_version: '6.16.0' + pyi_tag: '2025.09.13.221251' + pyi_hash: 2d881843580efdc54f3523507fc6d9c5b6051ee49c743a6d9b7003ac5758c226 - arch: 'arm64' runner: windows-11-arm python_version: '3.13' # arm64 only has Python >= 3.11 available platform_tag: win_arm64 - pyi_version: '6.15.0' - pyi_tag: '2025.09.08.215938' - pyi_hash: 5dac9f802085432dd3135708e835ef4c08570c308d07c3ef8154495b76bf2a83 + pyi_version: '6.16.0' + pyi_tag: '2025.09.13.221251' + pyi_hash: 4250c9085e34a95c898f3ee2f764914fc36ec59f0d97c28e6a75fcf21f7b144f env: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} From b81e9272dce5844e8fba371cb4b4fd95ad3ed819 Mon Sep 17 00:00:00 2001 From: sepro Date: Tue, 16 Sep 2025 19:43:00 +0200 Subject: [PATCH 045/175] [ie/vk] Support vksport URLs (#14341) Closes #14175 Authored by: seproDev --- yt_dlp/extractor/vk.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py index b5b8804cc6..87670d5059 100644 --- a/yt_dlp/extractor/vk.py +++ b/yt_dlp/extractor/vk.py @@ -96,12 +96,12 @@ class VKIE(VKBaseIE): https?:// (?: (?: - (?:(?:m|new)\.)?vk(?:(?:video)?\.ru|\.com)/video_| + (?:(?:m|new|vksport)\.)?vk(?:(?:video)?\.ru|\.com)/video_| (?:www\.)?daxab\.com/ ) ext\.php\?(?P.*?\boid=(?P-?\d+).*?\bid=(?P\d+).*)| (?: - (?:(?:m|new)\.)?vk(?:(?:video)?\.ru|\.com)/(?:.+?\?.*?z=)?(?:video|clip)| + (?:(?:m|new|vksport)\.)?vk(?:(?:video)?\.ru|\.com)/(?:.+?\?.*?z=)?(?:video|clip)| (?:www\.)?daxab\.com/embed/ ) (?P-?\d+_\d+)(?:.*\blist=(?P([\da-f]+)|(ln-[\da-zA-Z]+)))? @@ -359,6 +359,10 @@ class VKIE(VKBaseIE): 'url': 'https://vk.ru/video-220754053_456242564', 'only_matching': True, }, + { + 'url': 'https://vksport.vkvideo.ru/video-124096712_456240773', + 'only_matching': True, + }, ] def _real_extract(self, url): From 677997d84eaec0037397f7d935386daa3025b004 Mon Sep 17 00:00:00 2001 From: thegymguy Date: Tue, 16 Sep 2025 19:19:08 -0300 Subject: [PATCH 046/175] [ie/xhamster] Fix extractor (#14345) Fix a1c98226a4e869a34cc764a9dcf7a4558516308e Closes #14145 Authored by: arand, thegymguy Co-authored-by: arand <183498+arand@users.noreply.github.com> --- yt_dlp/extractor/xhamster.py | 45 +++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py index 27efd43129..4f9da4c2d8 100644 --- a/yt_dlp/extractor/xhamster.py +++ b/yt_dlp/extractor/xhamster.py @@ -1,4 +1,5 @@ import base64 +import codecs import itertools import re @@ -11,13 +12,13 @@ from ..utils import ( extract_attributes, float_or_none, int_or_none, + join_nonempty, parse_duration, str_or_none, try_call, try_get, unified_strdate, url_or_none, - urljoin, ) @@ -142,6 +143,27 @@ class XHamsterIE(InfoExtractor): 'only_matching': True, }] + _XOR_KEY = b'xh7999' + + def _decipher_format_url(self, format_url, format_id): + cipher_type, _, ciphertext = try_call( + lambda: base64.b64decode(format_url).decode().partition('_')) or [None] * 3 + + if not cipher_type or not ciphertext: + self.report_warning(f'Skipping format "{format_id}": failed to decipher URL') + return None + + if cipher_type == 'xor': + return bytes( + a ^ b for a, b in + zip(ciphertext.encode(), itertools.cycle(self._XOR_KEY))).decode() + + if cipher_type == 'rot13': + return codecs.decode(ciphertext, cipher_type) + + self.report_warning(f'Skipping format "{format_id}": unsupported cipher type "{cipher_type}"') + return None + def _real_extract(self, url): mobj = self._match_valid_url(url) video_id = mobj.group('id') or mobj.group('id_2') @@ -212,7 +234,7 @@ class XHamsterIE(InfoExtractor): hls_url = hls_sources.get(hls_format_key) if not hls_url: continue - hls_url = urljoin(url, hls_url) + hls_url = self._decipher_format_url(hls_url, f'hls-{hls_format_key}') if not hls_url or hls_url in format_urls: continue format_urls.add(hls_url) @@ -221,7 +243,7 @@ class XHamsterIE(InfoExtractor): m3u8_id='hls', fatal=False)) standard_sources = xplayer_sources.get('standard') if isinstance(standard_sources, dict): - for format_id, formats_list in standard_sources.items(): + for identifier, formats_list in standard_sources.items(): if not isinstance(formats_list, list): continue for standard_format in formats_list: @@ -231,12 +253,11 @@ class XHamsterIE(InfoExtractor): standard_url = standard_format.get(standard_format_key) if not standard_url: continue - decoded = try_call(lambda: base64.b64decode(standard_url)) - if decoded and decoded[:4] == b'xor_': - standard_url = bytes( - a ^ b for a, b in - zip(decoded[4:], itertools.cycle(b'xh7999'))).decode() - standard_url = urljoin(url, standard_url) + quality = (str_or_none(standard_format.get('quality')) + or str_or_none(standard_format.get('label')) + or '') + format_id = join_nonempty(identifier, quality) + standard_url = self._decipher_format_url(standard_url, format_id) if not standard_url or standard_url in format_urls: continue format_urls.add(standard_url) @@ -246,11 +267,9 @@ class XHamsterIE(InfoExtractor): standard_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)) continue - quality = (str_or_none(standard_format.get('quality')) - or str_or_none(standard_format.get('label')) - or '') + formats.append({ - 'format_id': f'{format_id}-{quality}', + 'format_id': format_id, 'url': standard_url, 'ext': ext, 'height': get_height(quality), From 820c6e244571557fcfc127d4b3680e2d07c04dca Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:03:09 -0500 Subject: [PATCH 047/175] [ie/mitele] Remove extractor (#14348) Closes #13535 Authored by: bashonly --- yt_dlp/extractor/_extractors.py | 1 - yt_dlp/extractor/mitele.py | 102 -------------------------------- yt_dlp/extractor/unsupported.py | 4 ++ 3 files changed, 4 insertions(+), 103 deletions(-) delete mode 100644 yt_dlp/extractor/mitele.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 9d3d353683..e6b34e9e64 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1136,7 +1136,6 @@ from .mit import ( OCWMITIE, TechTVMITIE, ) -from .mitele import MiTeleIE from .mixch import ( MixchArchiveIE, MixchIE, diff --git a/yt_dlp/extractor/mitele.py b/yt_dlp/extractor/mitele.py deleted file mode 100644 index 76fef337a2..0000000000 --- a/yt_dlp/extractor/mitele.py +++ /dev/null @@ -1,102 +0,0 @@ -from .telecinco import TelecincoBaseIE -from ..utils import ( - int_or_none, - parse_iso8601, -) - - -class MiTeleIE(TelecincoBaseIE): - IE_DESC = 'mitele.es' - _VALID_URL = r'https?://(?:www\.)?mitele\.es/(?:[^/]+/)+(?P[^/]+)/player' - _TESTS = [{ - 'url': 'http://www.mitele.es/programas-tv/diario-de/57b0dfb9c715da65618b4afa/player', - 'info_dict': { - 'id': 'FhYW1iNTE6J6H7NkQRIEzfne6t2quqPg', - 'ext': 'mp4', - 'title': 'Diario de La redacción Programa 144', - 'description': 'md5:07c35a7b11abb05876a6a79185b58d27', - 'series': 'Diario de', - 'season': 'Season 14', - 'season_number': 14, - 'episode': 'Tor, la web invisible', - 'episode_number': 3, - 'thumbnail': r're:(?i)^https?://.*\.jpg$', - 'duration': 2913, - 'age_limit': 16, - 'timestamp': 1471209401, - 'upload_date': '20160814', - }, - 'skip': 'HTTP Error 404 Not Found', - }, { - # no explicit title - 'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/57b0de3dc915da14058b4876/player', - 'info_dict': { - 'id': 'oyNG1iNTE6TAPP-JmCjbwfwJqqMMX3Vq', - 'ext': 'mp4', - 'title': 'Cuarto Milenio Temporada 6 Programa 226', - 'description': 'md5:5ff132013f0cd968ffbf1f5f3538a65f', - 'series': 'Cuarto Milenio', - 'season': 'Season 6', - 'season_number': 6, - 'episode': 'Episode 24', - 'episode_number': 24, - 'thumbnail': r're:(?i)^https?://.*\.jpg$', - 'duration': 7313, - 'age_limit': 12, - 'timestamp': 1471209021, - 'upload_date': '20160814', - }, - 'params': { - 'skip_download': True, - }, - 'skip': 'HTTP Error 404 Not Found', - }, { - 'url': 'https://www.mitele.es/programas-tv/horizonte/temporada-5/programa-171-40_013480051/player/', - 'info_dict': { - 'id': '7adbe22e-cd41-4787-afa4-36f3da7c2c6f', - 'ext': 'mp4', - 'title': 'Horizonte Temporada 5 Programa 171', - 'description': 'md5:97f1fb712c5ac27e5693a8b3c5c0c6e3', - 'episode': 'Las Zonas de Bajas Emisiones, a debate', - 'episode_number': 171, - 'season': 'Season 5', - 'season_number': 5, - 'series': 'Horizonte', - 'duration': 7012, - 'upload_date': '20240927', - 'timestamp': 1727416450, - 'thumbnail': 'https://album.mediaset.es/eimg/2024/09/27/horizonte-171_9f02.jpg', - 'age_limit': 12, - }, - 'params': {'geo_bypass_country': 'ES'}, - }, { - 'url': 'http://www.mitele.es/series-online/la-que-se-avecina/57aac5c1c915da951a8b45ed/player', - 'only_matching': True, - }, { - 'url': 'https://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144-40_1006364575251/player/', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - pre_player = self._search_json( - r'window\.\$REACTBASE_STATE\.prePlayer_mtweb\s*=', - webpage, 'Pre Player', display_id)['prePlayer'] - title = pre_player['title'] - video_info = self._parse_content(pre_player['video'], url) - content = pre_player.get('content') or {} - info = content.get('info') or {} - - video_info.update({ - 'title': title, - 'description': info.get('synopsis'), - 'series': content.get('title'), - 'season_number': int_or_none(info.get('season_number')), - 'episode': content.get('subtitle'), - 'episode_number': int_or_none(info.get('episode_number')), - 'duration': int_or_none(info.get('duration')), - 'age_limit': int_or_none(info.get('rating')), - 'timestamp': parse_iso8601(pre_player.get('publishedTime')), - }) - return video_info diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py index 4857156913..333c7e9a2b 100644 --- a/yt_dlp/extractor/unsupported.py +++ b/yt_dlp/extractor/unsupported.py @@ -65,6 +65,7 @@ class KnownDRMIE(UnsupportedInfoExtractor): r'play\.rtl\.hr', r'rtlmost\.hu', r'plus\.rtl\.de(?!/podcast/)', + r'mediasetinfinity\.es', ) _TESTS = [{ @@ -222,6 +223,9 @@ class KnownDRMIE(UnsupportedInfoExtractor): }, { 'url': 'https://plus.rtl.de/video-tv/', 'only_matching': True, + }, { + 'url': 'https://www.mediasetinfinity.es/', + 'only_matching': True, }] def _real_extract(self, url): From e123a48f1155703d8709a4221a42bd45c0a2b3ce Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 16 Sep 2025 23:58:42 -0500 Subject: [PATCH 048/175] [ie/telecinco] Support browser impersonation (#14351) Closes #14349 Authored by: bashonly --- yt_dlp/extractor/telecinco.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/telecinco.py b/yt_dlp/extractor/telecinco.py index a34f2afd4a..bdcae3b774 100644 --- a/yt_dlp/extractor/telecinco.py +++ b/yt_dlp/extractor/telecinco.py @@ -46,7 +46,7 @@ class TelecincoBaseIE(InfoExtractor): error_code = traverse_obj( self._webpage_read_content(error.cause.response, caronte['cerbero'], video_id, fatal=False), ({json.loads}, 'code', {int})) - if error_code in (4038, 40313): + if error_code in (4036, 4038, 40313): self.raise_geo_restricted(countries=['ES']) raise @@ -140,7 +140,7 @@ class TelecincoIE(TelecincoBaseIE): def _real_extract(self, url): display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) + webpage = self._download_webpage(url, display_id, impersonate=True) article = self._search_json( r'window\.\$REACTBASE_STATE\.article(?:_multisite)?\s*=', webpage, 'article', display_id)['article'] From b2c01d0498653e0239c7226c5a7fcb614dd4dbc8 Mon Sep 17 00:00:00 2001 From: sepro Date: Fri, 19 Sep 2025 22:22:47 +0200 Subject: [PATCH 049/175] [ie/applepodcast] Fix extractor (#14372) Closes #14368 Authored by: seproDev --- yt_dlp/extractor/applepodcasts.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/applepodcasts.py b/yt_dlp/extractor/applepodcasts.py index b99d24e0eb..91a7028367 100644 --- a/yt_dlp/extractor/applepodcasts.py +++ b/yt_dlp/extractor/applepodcasts.py @@ -1,5 +1,6 @@ from .common import InfoExtractor from ..utils import ( + clean_html, clean_podcast_url, int_or_none, parse_iso8601, @@ -17,7 +18,7 @@ class ApplePodcastsIE(InfoExtractor): 'ext': 'mp3', 'title': 'Ferreck Dawn - To The Break of Dawn 117', 'episode': 'Ferreck Dawn - To The Break of Dawn 117', - 'description': 'md5:1fc571102f79dbd0a77bfd71ffda23bc', + 'description': 'md5:8c4f5c2c30af17ed6a98b0b9daf15b76', 'upload_date': '20240812', 'timestamp': 1723449600, 'duration': 3596, @@ -58,7 +59,7 @@ class ApplePodcastsIE(InfoExtractor): r'', fatal=False) - - clip = traverse_obj(hydration_data, ('clips', ...), get_all=False) - if not clip: - raise ExtractorError( - 'Could not find video information.', video_id=video_id) - - title = clip['contentTitle'] - - source_width = int_or_none(clip.get('sourceWidth')) - source_height = int_or_none(clip.get('sourceHeight')) - - aspect_ratio = source_width / source_height if source_width and source_height else 16 / 9 - - def add_item(container, item_url, height, id_key='format_id', item_id=None): - item_id = item_id or '%dp' % height - if item_id not in item_url: - return - container.append({ - 'url': item_url, - id_key: item_id, - 'width': round(aspect_ratio * height), - 'height': height, - }) + content_data = self._download_json( + f'https://medal.tv/api/content/{video_id}', video_id, + headers={'Accept': 'application/json'}) formats = [] - thumbnails = [] - for k, v in clip.items(): - if not (v and isinstance(v, str)): - continue - mobj = re.match(r'(contentUrl|thumbnail)(?:(\d+)p)?$', k) - if not mobj: - continue - prefix = mobj.group(1) - height = int_or_none(mobj.group(2)) - if prefix == 'contentUrl': - add_item( - formats, v, height or source_height, - item_id=None if height else 'source') - elif prefix == 'thumbnail': - add_item(thumbnails, v, height, 'id') - - error = clip.get('error') - if not formats and error: - if error == 404: - self.raise_no_formats( - 'That clip does not exist.', - expected=True, video_id=video_id) - else: - self.raise_no_formats( - f'An unknown error occurred ({error}).', - video_id=video_id) - - # Necessary because the id of the author is not known in advance. - # Won't raise an issue if no profile can be found as this is optional. - author = traverse_obj(hydration_data, ('profiles', ...), get_all=False) or {} - author_id = str_or_none(author.get('userId')) - author_url = format_field(author_id, None, 'https://medal.tv/users/%s') + if m3u8_url := url_or_none(content_data.get('contentUrlHls')): + formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls')) + if http_url := url_or_none(content_data.get('contentUrl')): + formats.append({ + 'url': http_url, + 'format_id': 'http-source', + 'ext': 'mp4', + 'quality': 1, + }) + formats = [fmt for fmt in formats if 'video/privacy-protected-guest' not in fmt['url']] + if not formats: + # Fallback, does not require auth + self.report_warning('Video formats are not available through API, falling back to social video URL') + urlh = self._request_webpage( + f'https://medal.tv/api/content/{video_id}/socialVideoUrl', video_id, + note='Checking social video URL') + formats.append({ + 'url': urlh.url, + 'format_id': 'social-video', + 'ext': 'mp4', + 'quality': -1, + }) return { 'id': video_id, - 'title': title, 'formats': formats, - 'thumbnails': thumbnails, - 'description': clip.get('contentDescription'), - 'uploader': author.get('displayName'), - 'timestamp': float_or_none(clip.get('created'), 1000), - 'uploader_id': author_id, - 'uploader_url': author_url, - 'duration': int_or_none(clip.get('videoLengthSeconds')), - 'view_count': int_or_none(clip.get('views')), - 'like_count': int_or_none(clip.get('likes')), - 'comment_count': int_or_none(clip.get('comments')), + **traverse_obj(content_data, { + 'title': ('contentTitle', {str}), + 'description': ('contentDescription', {str}), + 'timestamp': ('created', {int_or_none(scale=1000)}), + 'duration': ('videoLengthSeconds', {int_or_none}), + 'view_count': ('views', {int_or_none}), + 'like_count': ('likes', {int_or_none}), + 'comment_count': ('comments', {int_or_none}), + 'uploader': ('poster', 'displayName', {str}), + 'uploader_id': ('poster', 'userId', {str}), + 'uploader_url': ('poster', 'userId', {str}, filter, {lambda x: x and f'https://medal.tv/users/{x}'}), + 'tags': ('tags', ..., {str}), + 'thumbnail': ('thumbnailUrl', {url_or_none}), + }), } From 0eed3fe530d6ff4b668494c5b1d4d6fc1ade96f7 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:23:00 -0600 Subject: [PATCH 156/175] [pp/ffmpeg] Fix uncaught error if bad --ffmpeg-location is given (#15104) Revert 9f77e04c76e36e1cbbf49bc9eb385fa6ef804b67 Closes #12829 Authored by: bashonly --- yt_dlp/downloader/external.py | 2 ++ yt_dlp/postprocessor/ffmpeg.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 82bc5106a5..14879b358d 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -457,6 +457,8 @@ class FFmpegFD(ExternalFD): @classmethod def available(cls, path=None): + # TODO: Fix path for ffmpeg + # Fixme: This may be wrong when --ffmpeg-location is used return FFmpegPostProcessor().available def on_process_started(self, proc, stdin): diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 5c9c0390d9..a850bdd102 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -192,7 +192,10 @@ class FFmpegPostProcessor(PostProcessor): @property def available(self): - return bool(self._ffmpeg_location.get()) or self.basename is not None + # If we return that ffmpeg is available, then the basename property *must* be run + # (as doing so has side effects), and its value can never be None + # See: https://github.com/yt-dlp/yt-dlp/issues/12829 + return self.basename is not None @property def executable(self): From 2c9f0c3456057aff0631d9ea6d3eda70ffd8aabe Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Wed, 19 Nov 2025 12:17:29 -0600 Subject: [PATCH 157/175] [ie/sproutvideo] Fix extractor (#15113) Closes #15112 Authored by: bashonly --- yt_dlp/extractor/sproutvideo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/sproutvideo.py b/yt_dlp/extractor/sproutvideo.py index ff9dc7dee2..80fc6d2141 100644 --- a/yt_dlp/extractor/sproutvideo.py +++ b/yt_dlp/extractor/sproutvideo.py @@ -101,8 +101,8 @@ class SproutVideoIE(InfoExtractor): webpage = self._download_webpage( url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'})) data = self._search_json( - r'(?:var|const|let)\s+(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage, 'player info', - video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];', + r'(?:window\.|(?:var|const|let)\s+)(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage, + 'player info', video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];', transform_source=lambda x: base64.b64decode(x).decode()) # SproutVideo may send player info for 'SMPTE Color Monitor Test' [a791d7b71b12ecc52e] From c2e7e9cdb2261adde01048d161914b156a3bad51 Mon Sep 17 00:00:00 2001 From: sepro Date: Thu, 20 Nov 2025 16:22:45 +0100 Subject: [PATCH 158/175] [ie/URPlay] Fix extractor (#15120) Closes #13028 Authored by: seproDev --- yt_dlp/extractor/urplay.py | 48 ++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/yt_dlp/extractor/urplay.py b/yt_dlp/extractor/urplay.py index a0ac2a0bc6..ad48e350ef 100644 --- a/yt_dlp/extractor/urplay.py +++ b/yt_dlp/extractor/urplay.py @@ -7,15 +7,15 @@ from ..utils import ( parse_age_limit, try_get, unified_timestamp, + url_or_none, ) -from ..utils.traversal import traverse_obj +from ..utils.traversal import require, traverse_obj class URPlayIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P[0-9]+)' _TESTS = [{ 'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand', - 'md5': '5ba36643c77cc3d34ffeadad89937d1e', 'info_dict': { 'id': '203704', 'ext': 'mp4', @@ -31,6 +31,7 @@ class URPlayIE(InfoExtractor): 'episode': 'Om vetenskap, kritiskt tänkande och motstånd', 'age_limit': 15, }, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv', 'info_dict': { @@ -49,6 +50,7 @@ class URPlayIE(InfoExtractor): 'tags': 'count:7', 'episode': 'Mitt barn skadar sig själv', }, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde', 'info_dict': { @@ -68,6 +70,27 @@ class URPlayIE(InfoExtractor): 'episode': 'Sovkudde', 'season': 'Säsong 1', }, + 'params': {'skip_download': 'm3u8'}, + }, { + # Only accessible through new media api + 'url': 'https://urplay.se/program/242932-vulkanernas-krafter-fran-kraftfull-till-forgorande', + 'info_dict': { + 'id': '242932', + 'ext': 'mp4', + 'title': 'Vulkanernas krafter : Från kraftfull till förgörande', + 'description': 'md5:742bb87048e7d5a7f209d28f9bb70ab1', + 'age_limit': 15, + 'duration': 2613, + 'thumbnail': 'https://assets.ur.se/id/242932/images/1_hd.jpg', + 'categories': ['Vetenskap & teknik'], + 'tags': ['Geofysik', 'Naturvetenskap', 'Vulkaner', 'Vulkanutbrott'], + 'series': 'Vulkanernas krafter', + 'episode': 'Från kraftfull till förgörande', + 'episode_number': 2, + 'timestamp': 1763514000, + 'upload_date': '20251119', + }, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden', 'only_matching': True, @@ -88,21 +111,12 @@ class URPlayIE(InfoExtractor): webpage, 'urplayer data'), video_id)['accessibleEpisodes'] urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id)) episode = urplayer_data['title'] - - host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect'] - formats = [] - urplayer_streams = urplayer_data.get('streamingInfo', {}) - - for k, v in urplayer_streams.get('raw', {}).items(): - if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)): - continue - file_http = v.get('location') - if file_http: - formats.extend(self._extract_wowza_formats( - f'http://{host}/{file_http}playlist.m3u8', - video_id, skip_protocols=['f4m', 'rtmp', 'rtsp'])) - - subtitles = {} + sources = self._download_json( + f'https://media-api.urplay.se/config-streaming/v1/urplay/sources/{video_id}', video_id, + note='Downloading streaming information') + hls_url = traverse_obj(sources, ('sources', 'hls', {url_or_none}, {require('HLS URL')})) + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + hls_url, video_id, 'mp4', m3u8_id='hls') def parse_lang_code(code): "3-character language code or None (utils candidate)" From 20f83f208eae863250b35e2761adad88e91d85a1 Mon Sep 17 00:00:00 2001 From: "Michael D." Date: Thu, 20 Nov 2025 19:56:25 +0100 Subject: [PATCH 159/175] [ie/netapp] Add extractors (#15122) Closes #14902 Authored by: darkstar --- yt_dlp/extractor/_extractors.py | 4 ++ yt_dlp/extractor/netapp.py | 79 +++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 yt_dlp/extractor/netapp.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index fc77804692..5f82ad54e9 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1284,6 +1284,10 @@ from .nest import ( NestClipIE, NestIE, ) +from .netapp import ( + NetAppCollectionIE, + NetAppVideoIE, +) from .neteasemusic import ( NetEaseMusicAlbumIE, NetEaseMusicDjRadioIE, diff --git a/yt_dlp/extractor/netapp.py b/yt_dlp/extractor/netapp.py new file mode 100644 index 0000000000..a665472094 --- /dev/null +++ b/yt_dlp/extractor/netapp.py @@ -0,0 +1,79 @@ +from .brightcove import BrightcoveNewIE +from .common import InfoExtractor +from ..utils import parse_iso8601 +from ..utils.traversal import require, traverse_obj + + +class NetAppBaseIE(InfoExtractor): + _BC_URL = 'https://players.brightcove.net/6255154784001/default_default/index.html?videoId={}' + + @staticmethod + def _parse_metadata(item): + return traverse_obj(item, { + 'title': ('name', {str}), + 'description': ('description', {str}), + 'timestamp': ('createdAt', {parse_iso8601}), + }) + + +class NetAppVideoIE(NetAppBaseIE): + _VALID_URL = r'https?://media\.netapp\.com/video-detail/(?P[0-9a-f-]+)' + + _TESTS = [{ + 'url': 'https://media.netapp.com/video-detail/da25fc01-82ad-5284-95bc-26920200a222/seamless-storage-for-modern-kubernetes-deployments', + 'info_dict': { + 'id': '1843620950167202073', + 'ext': 'mp4', + 'title': 'Seamless storage for modern Kubernetes deployments', + 'description': 'md5:1ee39e315243fe71fb90af2796037248', + 'uploader_id': '6255154784001', + 'duration': 2159.41, + 'thumbnail': r're:https://house-fastly-signed-us-east-1-prod\.brightcovecdn\.com/image/.*\.jpg', + 'tags': 'count:15', + 'timestamp': 1758213949, + 'upload_date': '20250918', + }, + }, { + 'url': 'https://media.netapp.com/video-detail/45593e5d-cf1c-5996-978c-c9081906e69f/unleash-ai-innovation-with-your-data-with-the-netapp-platform', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_uuid = self._match_id(url) + metadata = self._download_json( + f'https://api.media.netapp.com/client/detail/{video_uuid}', video_uuid) + + brightcove_video_id = traverse_obj(metadata, ( + 'sections', lambda _, v: v['type'] == 'Player', 'video', {str}, any, {require('brightcove video id')})) + + video_item = traverse_obj(metadata, ('sections', lambda _, v: v['type'] == 'VideoDetail', any)) + + return self.url_result( + self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id, + url_transparent=True, **self._parse_metadata(video_item)) + + +class NetAppCollectionIE(NetAppBaseIE): + _VALID_URL = r'https?://media\.netapp\.com/collection/(?P[0-9a-f-]+)' + _TESTS = [{ + 'url': 'https://media.netapp.com/collection/9820e190-f2a6-47ac-9c0a-98e5e64234a4', + 'info_dict': { + 'title': 'Featured sessions', + 'id': '9820e190-f2a6-47ac-9c0a-98e5e64234a4', + }, + 'playlist_count': 4, + }] + + def _entries(self, metadata): + for item in traverse_obj(metadata, ('items', lambda _, v: v['brightcoveVideoId'])): + brightcove_video_id = item['brightcoveVideoId'] + yield self.url_result( + self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id, + url_transparent=True, **self._parse_metadata(item)) + + def _real_extract(self, url): + collection_uuid = self._match_id(url) + metadata = self._download_json( + f'https://api.media.netapp.com/client/collection/{collection_uuid}', collection_uuid) + + return self.playlist_result(self._entries(metadata), collection_uuid, playlist_title=metadata.get('name')) From 6842620d56e4c4e6affb90c2f8dff8a36dee852c Mon Sep 17 00:00:00 2001 From: Elioo <79273475+beliote@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:01:07 +0100 Subject: [PATCH 160/175] [ie/Digiteka] Rework extractor (#14903) Closes #12454 Authored by: beliote --- yt_dlp/extractor/digiteka.py | 95 +++++++++++++++--------------------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/yt_dlp/extractor/digiteka.py b/yt_dlp/extractor/digiteka.py index e56ec63e86..1bbec62165 100644 --- a/yt_dlp/extractor/digiteka.py +++ b/yt_dlp/extractor/digiteka.py @@ -1,5 +1,6 @@ from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import int_or_none, url_or_none +from ..utils.traversal import traverse_obj class DigitekaIE(InfoExtractor): @@ -25,74 +26,56 @@ class DigitekaIE(InfoExtractor): )/(?P[\d+a-z]+)''' _EMBED_REGEX = [r'<(?:iframe|script)[^>]+src=["\'](?P(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)'] _TESTS = [{ - # news - 'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r', - 'md5': '276a0e49de58c7e85d32b057837952a2', + 'url': 'https://www.ultimedia.com/default/index/videogeneric/id/3x5x55k', 'info_dict': { - 'id': 's8uk0r', + 'id': '3x5x55k', 'ext': 'mp4', - 'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées', + 'title': 'Il est passionné de DS', 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 74, - 'upload_date': '20150317', - 'timestamp': 1426604939, - 'uploader_id': '3fszv', + 'duration': 89, + 'upload_date': '20251012', + 'timestamp': 1760285363, + 'uploader_id': '3pz33', }, - }, { - # music - 'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8', - 'md5': '2ea3513813cf230605c7e2ffe7eca61c', - 'info_dict': { - 'id': 'xvpfp8', - 'ext': 'mp4', - 'title': 'Two - C\'est La Vie (clip)', - 'thumbnail': r're:^https?://.*\.jpg', - 'duration': 233, - 'upload_date': '20150224', - 'timestamp': 1424760500, - 'uploader_id': '3rfzk', - }, - }, { - 'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes', - 'only_matching': True, + 'params': {'skip_download': True}, }] + _IFRAME_MD_ID = '01836272' # One static ID working for Ultimedia iframes def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - video_type = mobj.group('embed_type') or mobj.group('site_type') - if video_type == 'music': - video_type = 'musique' + video_id = self._match_id(url) - deliver_info = self._download_json( - f'http://www.ultimedia.com/deliver/video?video={video_id}&topic={video_type}', - video_id) - - yt_id = deliver_info.get('yt_id') - if yt_id: - return self.url_result(yt_id, 'Youtube') - - jwconf = deliver_info['jwconf'] + video_info = self._download_json( + f'https://www.ultimedia.com/player/getConf/{self._IFRAME_MD_ID}/1/{video_id}', video_id, + note='Downloading player configuration')['video'] formats = [] - for source in jwconf['playlist'][0]['sources']: - formats.append({ - 'url': source['file'], - 'format_id': source.get('label'), - }) + subtitles = {} - title = deliver_info['title'] - thumbnail = jwconf.get('image') - duration = int_or_none(deliver_info.get('duration')) - timestamp = int_or_none(deliver_info.get('release_time')) - uploader_id = deliver_info.get('owner_id') + if hls_url := traverse_obj(video_info, ('media_sources', 'hls', 'hls_auto', {url_or_none})): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + + for format_id, mp4_url in traverse_obj(video_info, ('media_sources', 'mp4', {dict.items}, ...)): + if not mp4_url: + continue + formats.append({ + 'url': mp4_url, + 'format_id': format_id, + 'height': int_or_none(format_id.partition('_')[2]), + 'ext': 'mp4', + }) return { 'id': video_id, - 'title': title, - 'thumbnail': thumbnail, - 'duration': duration, - 'timestamp': timestamp, - 'uploader_id': uploader_id, 'formats': formats, + 'subtitles': subtitles, + **traverse_obj(video_info, { + 'title': ('title', {str}), + 'thumbnail': ('image', {url_or_none}), + 'duration': ('duration', {int_or_none}), + 'timestamp': ('creationDate', {int_or_none}), + 'uploader_id': ('ownerId', {str}), + }), } From 3cb5e4db54d44fe82d4eee94ae2f37cbce2e7dfc Mon Sep 17 00:00:00 2001 From: putridambassador121 <100704281+putridambassador121@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:07:07 -0300 Subject: [PATCH 161/175] [ie/AGalega] Add extractor (#15105) Closes #14758 Authored by: putridambassador121 --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/agalega.py | 91 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 yt_dlp/extractor/agalega.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 5f82ad54e9..5a71096c96 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -75,6 +75,7 @@ from .afreecatv import ( AfreecaTVLiveIE, AfreecaTVUserIE, ) +from .agalega import AGalegaIE from .agora import ( TokFMAuditionIE, TokFMPodcastIE, diff --git a/yt_dlp/extractor/agalega.py b/yt_dlp/extractor/agalega.py new file mode 100644 index 0000000000..c02d4ae3a4 --- /dev/null +++ b/yt_dlp/extractor/agalega.py @@ -0,0 +1,91 @@ +import json +import time + +from .common import InfoExtractor +from ..utils import jwt_decode_hs256, url_or_none +from ..utils.traversal import traverse_obj + + +class AGalegaBaseIE(InfoExtractor): + _access_token = None + + @staticmethod + def _jwt_is_expired(token): + return jwt_decode_hs256(token)['exp'] - time.time() < 120 + + def _refresh_access_token(self, video_id): + AGalegaBaseIE._access_token = self._download_json( + 'https://www.agalega.gal/api/fetch-api/jwt/token', video_id, + note='Downloading access token', + data=json.dumps({ + 'username': None, + 'password': None, + 'client': 'crtvg', + 'checkExistsCookies': False, + }).encode())['access'] + + def _call_api(self, endpoint, display_id, note, fatal=True, query=None): + if not AGalegaBaseIE._access_token or self._jwt_is_expired(AGalegaBaseIE._access_token): + self._refresh_access_token(endpoint) + return self._download_json( + f'https://api-agalega.interactvty.com/api/2.0/contents/{endpoint}', display_id, + note=note, fatal=fatal, query=query, + headers={'Authorization': f'jwtok {AGalegaBaseIE._access_token}'}) + + +class AGalegaIE(AGalegaBaseIE): + IE_NAME = 'agalega:videos' + _VALID_URL = r'https?://(?:www\.)?agalega\.gal/videos/(?:detail/)?(?P[0-9]+)' + _TESTS = [{ + 'url': 'https://www.agalega.gal/videos/288664-lr-ninguencheconta', + 'md5': '04533a66c5f863d08dd9724b11d1c223', + 'info_dict': { + 'id': '288664', + 'title': 'Roberto e Ángel Martín atenden consultas dos espectadores', + 'description': 'O cómico ademais fai un repaso dalgúns momentos da súa traxectoria profesional', + 'thumbnail': 'https://crtvg-bucket.flumotion.cloud/content_cards/2ef32c3b9f6249d9868fd8f11d389d3d.png', + 'ext': 'mp4', + }, + }, { + 'url': 'https://www.agalega.gal/videos/detail/296152-pulso-activo-7', + 'md5': '26df7fdcf859f38ad92d837279d6b56d', + 'info_dict': { + 'id': '296152', + 'title': 'Pulso activo | 18-11-2025', + 'description': 'Anxo, Noemí, Silvia e Estrella comparten as sensacións da clase de Eddy.', + 'thumbnail': 'https://crtvg-bucket.flumotion.cloud/content_cards/a6bb7da6c8994b82bf961ac6cad1707b.png', + 'ext': 'mp4', + }, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + content_data = self._call_api( + f'content/{video_id}/', video_id, note='Downloading content data', fatal=False, + query={ + 'optional_fields': 'image,is_premium,short_description,has_subtitle', + }) + resource_data = self._call_api( + f'content_resources/{video_id}/', video_id, note='Downloading resource data', + query={ + 'optional_fields': 'media_url', + }) + + formats = [] + subtitles = {} + for m3u8_url in traverse_obj(resource_data, ('results', ..., 'media_url', {url_or_none})): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + m3u8_url, video_id, ext='mp4', m3u8_id='hls') + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + + return { + 'id': video_id, + 'formats': formats, + 'subtitles': subtitles, + **traverse_obj(content_data, { + 'title': ('name', {str}), + 'description': (('description', 'short_description'), {str}, any), + 'thumbnail': ('image', {url_or_none}), + }), + } From 0c696239ef418776ac6ba20284bd2f3976a011b4 Mon Sep 17 00:00:00 2001 From: Sojiroh Date: Fri, 21 Nov 2025 20:08:20 -0300 Subject: [PATCH 162/175] [ie/WistiaChannel] Fix extractor (#14218) Closes #14204 Authored by: Sojiroh --- yt_dlp/extractor/wistia.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/wistia.py b/yt_dlp/extractor/wistia.py index 53ffb5529a..d12e9e4c4a 100644 --- a/yt_dlp/extractor/wistia.py +++ b/yt_dlp/extractor/wistia.py @@ -339,11 +339,20 @@ class WistiaChannelIE(WistiaBaseIE): 'title': 'The Roof S2: The Modern CRO', 'thumbnail': r're:https?://embed(?:-ssl)?\.wistia\.com/.+\.(?:jpg|png)', 'duration': 86.487, - 'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season.\n', + 'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season. ', 'timestamp': 1619790290, 'upload_date': '20210430', }, 'params': {'noplaylist': True, 'skip_download': True}, + }, { + # Channel with episodes structure instead of videos + 'url': 'https://fast.wistia.net/embed/channel/sapab9p6qd', + 'info_dict': { + 'id': 'sapab9p6qd', + 'title': 'Credo: An RCIA Program', + 'description': '\n', + }, + 'playlist_mincount': 80, }] _WEBPAGE_TESTS = [{ 'url': 'https://www.profitwell.com/recur/boxed-out', @@ -399,8 +408,7 @@ class WistiaChannelIE(WistiaBaseIE): entries = [ self.url_result(f'wistia:{video["hashedId"]}', WistiaIE, title=video.get('name')) - for video in traverse_obj(series, ('sections', ..., 'videos', ...)) or [] - if video.get('hashedId') + for video in traverse_obj(series, ('sections', ..., ('videos', 'episodes'), lambda _, v: v['hashedId'])) ] return self.playlist_result( From 715af0c636b2b33fb3df1eb2ee37eac8262d43ac Mon Sep 17 00:00:00 2001 From: WhatAmISupposedToPutHere Date: Sun, 23 Nov 2025 01:49:36 +0100 Subject: [PATCH 163/175] [ie/youtube] Determine wait time from player response (#14646) Closes #14645 Authored by: WhatAmISupposedToPutHere, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- README.md | 1 - yt_dlp/extractor/youtube/_video.py | 40 ++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8189015c72..c98c69f418 100644 --- a/README.md +++ b/README.md @@ -1870,7 +1870,6 @@ The following extractors use this feature: * `po_token`: Proof of Origin (PO) Token(s) to use. Comma seperated list of PO Tokens in the format `CLIENT.CONTEXT+PO_TOKEN`, e.g. `youtube:po_token=web.gvs+XXX,web.player=XXX,web_safari.gvs+YYY`. Context can be any of `gvs` (Google Video Server URLs), `player` (Innertube player request) or `subs` (Subtitles) * `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) * `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) -* `playback_wait`: Duration (in seconds) to wait inbetween the extraction and download stages in order to ensure the formats are available. The default is `6` seconds * `jsc_trace`: Enable debug logging for JS Challenge fetching. Either `true` or `false` (default) #### youtube-ejs diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 57edad3c0f..600e0ccda6 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -76,7 +76,7 @@ STREAMING_DATA_FETCH_GVS_PO_TOKEN = '__yt_dlp_fetch_gvs_po_token' STREAMING_DATA_PLAYER_TOKEN_PROVIDED = '__yt_dlp_player_token_provided' STREAMING_DATA_INNERTUBE_CONTEXT = '__yt_dlp_innertube_context' STREAMING_DATA_IS_PREMIUM_SUBSCRIBER = '__yt_dlp_is_premium_subscriber' -STREAMING_DATA_FETCHED_TIMESTAMP = '__yt_dlp_fetched_timestamp' +STREAMING_DATA_AVAILABLE_AT_TIMESTAMP = '__yt_dlp_available_at_timestamp' PO_TOKEN_GUIDE_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide' @@ -3032,7 +3032,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): elif pr: # Save client details for introspection later innertube_context = traverse_obj(player_ytcfg or self._get_default_ytcfg(client), 'INNERTUBE_CONTEXT') - fetched_timestamp = int(time.time()) sd = pr.setdefault('streamingData', {}) sd[STREAMING_DATA_CLIENT_NAME] = client sd[STREAMING_DATA_FETCH_GVS_PO_TOKEN] = fetch_gvs_po_token_func @@ -3040,7 +3039,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): sd[STREAMING_DATA_INNERTUBE_CONTEXT] = innertube_context sd[STREAMING_DATA_FETCH_SUBS_PO_TOKEN] = fetch_subs_po_token_func sd[STREAMING_DATA_IS_PREMIUM_SUBSCRIBER] = is_premium_subscriber - sd[STREAMING_DATA_FETCHED_TIMESTAMP] = fetched_timestamp + sd[STREAMING_DATA_AVAILABLE_AT_TIMESTAMP] = self._get_available_at_timestamp(pr, video_id, client) for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})): f[STREAMING_DATA_CLIENT_NAME] = client f[STREAMING_DATA_FETCH_GVS_PO_TOKEN] = fetch_gvs_po_token_func @@ -3172,9 +3171,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # save pots per client to avoid fetching again gvs_pots = {} - # For handling potential pre-playback required waiting period - playback_wait = int_or_none(self._configuration_arg('playback_wait', [None])[0], default=6) - def get_language_code_and_preference(fmt_stream): audio_track = fmt_stream.get('audioTrack') or {} display_name = audio_track.get('displayName') or '' @@ -3199,7 +3195,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): is_premium_subscriber = streaming_data[STREAMING_DATA_IS_PREMIUM_SUBSCRIBER] player_token_provided = streaming_data[STREAMING_DATA_PLAYER_TOKEN_PROVIDED] client_name = streaming_data.get(STREAMING_DATA_CLIENT_NAME) - available_at = streaming_data[STREAMING_DATA_FETCHED_TIMESTAMP] + playback_wait + available_at = streaming_data[STREAMING_DATA_AVAILABLE_AT_TIMESTAMP] streaming_formats = traverse_obj(streaming_data, (('formats', 'adaptiveFormats'), ...)) def get_stream_id(fmt_stream): @@ -3653,6 +3649,36 @@ class YoutubeIE(YoutubeBaseInfoExtractor): })) return webpage + def _get_available_at_timestamp(self, player_response, video_id, client): + now = time.time() + wait_seconds = 0 + + for renderer in traverse_obj(player_response, ( + 'adSlots', lambda _, v: v['adSlotRenderer']['adSlotMetadata']['triggerEvent'] == 'SLOT_TRIGGER_EVENT_BEFORE_CONTENT', + 'adSlotRenderer', 'fulfillmentContent', 'fulfilledLayout', 'playerBytesAdLayoutRenderer', 'renderingContent', ( + None, + ('playerBytesSequentialLayoutRenderer', 'sequentialLayouts', ..., 'playerBytesAdLayoutRenderer', 'renderingContent'), + ), 'instreamVideoAdRenderer', {dict}, + )): + duration = traverse_obj(renderer, ('playerVars', {urllib.parse.parse_qs}, 'length_seconds', -1, {int_or_none})) + ad = 'an ad' if duration is None else f'a {duration}s ad' + + skip_time = traverse_obj(renderer, ('skipOffsetMilliseconds', {float_or_none(scale=1000)})) + if skip_time is not None: + # YT allows skipping this ad; use the wait-until-skip time instead of full ad duration + skip_time = skip_time if skip_time % 1 else int(skip_time) + ad += f' skippable after {skip_time}s' + duration = skip_time + + if duration is not None: + self.write_debug(f'{video_id}: Detected {ad} for {client}') + wait_seconds += duration + + if wait_seconds: + return math.ceil(now) + wait_seconds + + return int(now) + def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None): live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails')) is_live = get_first(video_details, 'isLive') From e564b4a8080cff48fa0c28f20272c05085ee6130 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Mon, 24 Nov 2025 01:56:43 +0100 Subject: [PATCH 164/175] Respect `PATHEXT` when locating JS runtime on Windows (#15117) Fixes #15043 Authored by: Grub4K --- yt_dlp/utils/_jsruntime.py | 54 +++++++++++++++++++++++++++++++++----- yt_dlp/utils/_utils.py | 20 +++++++++----- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/yt_dlp/utils/_jsruntime.py b/yt_dlp/utils/_jsruntime.py index bd8fd1f880..94db52bf19 100644 --- a/yt_dlp/utils/_jsruntime.py +++ b/yt_dlp/utils/_jsruntime.py @@ -1,21 +1,61 @@ from __future__ import annotations + import abc import dataclasses import functools import os.path +import sys from ._utils import _get_exe_version_output, detect_exe_version, int_or_none -# NOT public API -def runtime_version_tuple(v): +def _runtime_version_tuple(v): # NB: will return (0,) if `v` is an invalid version string return tuple(int_or_none(x, default=0) for x in v.split('.')) +_FALLBACK_PATHEXT = ('.COM', '.EXE', '.BAT', '.CMD') + + +def _find_exe(basename: str) -> str: + if os.name != 'nt': + return basename + + paths: list[str] = [] + + # binary dir + if getattr(sys, 'frozen', False): + paths.append(os.path.dirname(sys.executable)) + # cwd + paths.append(os.getcwd()) + # PATH items + if path := os.environ.get('PATH'): + paths.extend(filter(None, path.split(os.path.pathsep))) + + pathext = os.environ.get('PATHEXT') + if pathext is None: + exts = _FALLBACK_PATHEXT + else: + exts = tuple(ext for ext in pathext.split(os.pathsep) if ext) + + visited = [] + for path in map(os.path.realpath, paths): + normed = os.path.normcase(path) + if normed in visited: + continue + visited.append(normed) + + for ext in exts: + binary = os.path.join(path, f'{basename}{ext}') + if os.access(binary, os.F_OK | os.X_OK) and not os.path.isdir(binary): + return binary + + return basename + + def _determine_runtime_path(path, basename): if not path: - return basename + return _find_exe(basename) if os.path.isdir(path): return os.path.join(path, basename) return path @@ -52,7 +92,7 @@ class DenoJsRuntime(JsRuntime): if not out: return None version = detect_exe_version(out, r'^deno (\S+)', 'unknown') - vt = runtime_version_tuple(version) + vt = _runtime_version_tuple(version) return JsRuntimeInfo( name='deno', path=path, version=version, version_tuple=vt, supported=vt >= self.MIN_SUPPORTED_VERSION) @@ -67,7 +107,7 @@ class BunJsRuntime(JsRuntime): if not out: return None version = detect_exe_version(out, r'^(\S+)', 'unknown') - vt = runtime_version_tuple(version) + vt = _runtime_version_tuple(version) return JsRuntimeInfo( name='bun', path=path, version=version, version_tuple=vt, supported=vt >= self.MIN_SUPPORTED_VERSION) @@ -82,7 +122,7 @@ class NodeJsRuntime(JsRuntime): if not out: return None version = detect_exe_version(out, r'^v(\S+)', 'unknown') - vt = runtime_version_tuple(version) + vt = _runtime_version_tuple(version) return JsRuntimeInfo( name='node', path=path, version=version, version_tuple=vt, supported=vt >= self.MIN_SUPPORTED_VERSION) @@ -100,7 +140,7 @@ class QuickJsRuntime(JsRuntime): is_ng = 'QuickJS-ng' in out version = detect_exe_version(out, r'^QuickJS(?:-ng)?\s+version\s+(\S+)', 'unknown') - vt = runtime_version_tuple(version.replace('-', '.')) + vt = _runtime_version_tuple(version.replace('-', '.')) if is_ng: return JsRuntimeInfo( name='quickjs-ng', path=path, version=version, version_tuple=vt, diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index c6ae21f6c7..65cd2373ce 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -876,13 +876,19 @@ class Popen(subprocess.Popen): kwargs.setdefault('encoding', 'utf-8') kwargs.setdefault('errors', 'replace') - if shell and os.name == 'nt' and kwargs.get('executable') is None: - if not isinstance(args, str): - args = shell_quote(args, shell=True) - shell = False - # Set variable for `cmd.exe` newline escaping (see `utils.shell_quote`) - env['='] = '"^\n\n"' - args = f'{self.__comspec()} /Q /S /D /V:OFF /E:ON /C "{args}"' + if os.name == 'nt' and kwargs.get('executable') is None: + # Must apply shell escaping if we are trying to run a batch file + # These conditions should be very specific to limit impact + if not shell and isinstance(args, list) and args and args[0].lower().endswith(('.bat', '.cmd')): + shell = True + + if shell: + if not isinstance(args, str): + args = shell_quote(args, shell=True) + shell = False + # Set variable for `cmd.exe` newline escaping (see `utils.shell_quote`) + env['='] = '"^\n\n"' + args = f'{self.__comspec()} /Q /S /D /V:OFF /E:ON /C "{args}"' super().__init__(args, *remaining, env=env, shell=shell, **kwargs, startupinfo=self._startupinfo) From 12d411722a3d7a0382d1d230a904ecd4e20298b6 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 24 Nov 2025 11:27:43 +0000 Subject: [PATCH 165/175] [ie/nhk] Fix extractors (#14528) Closes #14223, Closes #14589 Authored by: garret1317 --- yt_dlp/extractor/nhk.py | 303 ++++++++++++++-------------------------- 1 file changed, 104 insertions(+), 199 deletions(-) diff --git a/yt_dlp/extractor/nhk.py b/yt_dlp/extractor/nhk.py index eef3ed820c..99186ad414 100644 --- a/yt_dlp/extractor/nhk.py +++ b/yt_dlp/extractor/nhk.py @@ -23,96 +23,38 @@ from ..utils import ( class NhkBaseIE(InfoExtractor): - _API_URL_TEMPLATE = 'https://nwapi.nhk.jp/nhkworld/%sod%slist/v7b/%s/%s/%s/all%s.json' + _API_URL_TEMPLATE = 'https://api.nhkworld.jp/showsapi/v1/{lang}/{content_format}_{page_type}/{m_id}{extra_page}' _BASE_URL_REGEX = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P[a-z]{2})/' def _call_api(self, m_id, lang, is_video, is_episode, is_clip): + content_format = 'video' if is_video else 'audio' + content_type = 'clips' if is_clip else 'episodes' + if not is_episode: + extra_page = f'/{content_format}_{content_type}' + page_type = 'programs' + else: + extra_page = '' + page_type = content_type + return self._download_json( - self._API_URL_TEMPLATE % ( - 'v' if is_video else 'r', - 'clip' if is_clip else 'esd', - 'episode' if is_episode else 'program', - m_id, lang, '/all' if is_video else ''), - m_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'] or [] - - def _get_api_info(self, refresh=True): - if not refresh: - return self.cache.load('nhk', 'api_info') - - self.cache.store('nhk', 'api_info', {}) - movie_player_js = self._download_webpage( - 'https://movie-a.nhk.or.jp/world/player/js/movie-player.js', None, - note='Downloading stream API information') - api_info = { - 'url': self._search_regex( - r'prod:[^;]+\bapiUrl:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API url'), - 'token': self._search_regex( - r'prod:[^;]+\btoken:\s*[\'"]([^\'"]+)[\'"]', movie_player_js, None, 'stream API token'), - } - self.cache.store('nhk', 'api_info', api_info) - return api_info - - def _extract_stream_info(self, vod_id): - for refresh in (False, True): - api_info = self._get_api_info(refresh) - if not api_info: - continue - - api_url = api_info.pop('url') - meta = traverse_obj( - self._download_json( - api_url, vod_id, 'Downloading stream url info', fatal=False, query={ - **api_info, - 'type': 'json', - 'optional_id': vod_id, - 'active_flg': 1, - }), ('meta', 0)) - stream_url = traverse_obj( - meta, ('movie_url', ('mb_auto', 'auto_sp', 'auto_pc'), {url_or_none}), get_all=False) - - if stream_url: - formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, vod_id) - return { - **traverse_obj(meta, { - 'duration': ('duration', {int_or_none}), - 'timestamp': ('publication_date', {unified_timestamp}), - 'release_timestamp': ('insert_date', {unified_timestamp}), - 'modified_timestamp': ('update_date', {unified_timestamp}), - }), - 'formats': formats, - 'subtitles': subtitles, - } - raise ExtractorError('Unable to extract stream url') + self._API_URL_TEMPLATE.format( + lang=lang, content_format=content_format, page_type=page_type, + m_id=m_id, extra_page=extra_page), + join_nonempty(m_id, lang)) def _extract_episode_info(self, url, episode=None): fetch_episode = episode is None lang, m_type, episode_id = NhkVodIE._match_valid_url(url).group('lang', 'type', 'id') is_video = m_type != 'audio' - if is_video: - episode_id = episode_id[:4] + '-' + episode_id[4:] - if fetch_episode: episode = self._call_api( - episode_id, lang, is_video, True, episode_id[:4] == '9999')[0] + episode_id, lang, is_video, is_episode=True, is_clip=episode_id[:4] == '9999') - def get_clean_field(key): - return clean_html(episode.get(key + '_clean') or episode.get(key)) + video_id = join_nonempty('id', 'lang', from_dict=episode) - title = get_clean_field('sub_title') - series = get_clean_field('title') - - thumbnails = [] - for s, w, h in [('', 640, 360), ('_l', 1280, 720)]: - img_path = episode.get('image' + s) - if not img_path: - continue - thumbnails.append({ - 'id': f'{h}p', - 'height': h, - 'width': w, - 'url': 'https://www3.nhk.or.jp' + img_path, - }) + title = episode.get('title') + series = traverse_obj(episode, (('video_program', 'audio_program'), any, 'title')) episode_name = title if series and title: @@ -125,37 +67,52 @@ class NhkBaseIE(InfoExtractor): episode_name = None info = { - 'id': episode_id + '-' + lang, + 'id': video_id, 'title': title, - 'description': get_clean_field('description'), - 'thumbnails': thumbnails, 'series': series, 'episode': episode_name, + **traverse_obj(episode, { + 'description': ('description', {str}), + 'release_timestamp': ('first_broadcasted_at', {unified_timestamp}), + 'categories': ('categories', ..., 'name', {str}), + 'tags': ('tags', ..., 'name', {str}), + 'thumbnails': ('images', lambda _, v: v['url'], { + 'url': ('url', {urljoin(url)}), + 'width': ('width', {int_or_none}), + 'height': ('height', {int_or_none}), + }), + 'webpage_url': ('url', {urljoin(url)}), + }), + 'extractor_key': NhkVodIE.ie_key(), + 'extractor': NhkVodIE.IE_NAME, } - if is_video: - vod_id = episode['vod_id'] - info.update({ - **self._extract_stream_info(vod_id), - 'id': vod_id, - }) - + # XXX: We are assuming that 'video' and 'audio' are mutually exclusive + stream_info = traverse_obj(episode, (('video', 'audio'), {dict}, any)) or {} + if not stream_info.get('url'): + self.raise_no_formats('Stream not found; it has most likely expired', expected=True) else: - if fetch_episode: + stream_url = stream_info['url'] + if is_video: + formats, subtitles = self._extract_m3u8_formats_and_subtitles(stream_url, video_id) + info.update({ + 'formats': formats, + 'subtitles': subtitles, + **traverse_obj(stream_info, ({ + 'duration': ('duration', {int_or_none}), + 'timestamp': ('published_at', {unified_timestamp}), + })), + }) + else: # From https://www3.nhk.or.jp/nhkworld/common/player/radio/inline/rod.html - audio_path = remove_end(episode['audio']['audio'], '.m4a') + audio_path = remove_end(stream_url, '.m4a') info['formats'] = self._extract_m3u8_formats( f'{urljoin("https://vod-stream.nhk.jp", audio_path)}/index.m3u8', episode_id, 'm4a', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) for f in info['formats']: f['language'] = lang - else: - info.update({ - '_type': 'url_transparent', - 'ie_key': NhkVodIE.ie_key(), - 'url': url, - }) + return info @@ -168,29 +125,29 @@ class NhkVodIE(NhkBaseIE): # Content available only for a limited period of time. Visit # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples. _TESTS = [{ - 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2049126/', + 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/2049165/', 'info_dict': { - 'id': 'nw_vod_v_en_2049_126_20230413233000_01_1681398302', + 'id': '2049165-en', 'ext': 'mp4', - 'title': 'Japan Railway Journal - The Tohoku Shinkansen: Full Speed Ahead', - 'description': 'md5:49f7c5b206e03868a2fdf0d0814b92f6', + 'title': 'Japan Railway Journal - Choshi Electric Railway: Fighting to Get Back on Track', + 'description': 'md5:ab57df2fca7f04245148c2e787bb203d', 'thumbnail': r're:https://.+/.+\.jpg', - 'episode': 'The Tohoku Shinkansen: Full Speed Ahead', + 'episode': 'Choshi Electric Railway: Fighting to Get Back on Track', 'series': 'Japan Railway Journal', - 'modified_timestamp': 1707217907, - 'timestamp': 1681428600, - 'release_timestamp': 1693883728, - 'duration': 1679, - 'upload_date': '20230413', - 'modified_date': '20240206', - 'release_date': '20230905', + 'duration': 1680, + 'categories': ['Biz & Tech'], + 'tags': ['Akita', 'Chiba', 'Trains', 'Transcript', 'All (Japan Navigator)'], + 'timestamp': 1759055880, + 'upload_date': '20250928', + 'release_timestamp': 1758810600, + 'release_date': '20250925', }, }, { # video clip 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/', 'md5': '153c3016dfd252ba09726588149cf0e7', 'info_dict': { - 'id': 'lpZXIwaDE6_Z-976CPsFdxyICyWUzlT5', + 'id': '9999011-en', 'ext': 'mp4', 'title': 'Dining with the Chef - Chef Saito\'s Family recipe: MENCHI-KATSU', 'description': 'md5:5aee4a9f9d81c26281862382103b0ea5', @@ -198,24 +155,23 @@ class NhkVodIE(NhkBaseIE): 'series': 'Dining with the Chef', 'episode': 'Chef Saito\'s Family recipe: MENCHI-KATSU', 'duration': 148, - 'upload_date': '20190816', - 'release_date': '20230902', - 'release_timestamp': 1693619292, - 'modified_timestamp': 1707217907, - 'modified_date': '20240206', - 'timestamp': 1565997540, + 'categories': ['Food'], + 'tags': ['Washoku'], + 'timestamp': 1548212400, + 'upload_date': '20190123', }, }, { # radio - 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/livinginjapan-20231001-1/', + 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/audio/livinginjapan-20240901-1/', 'info_dict': { - 'id': 'livinginjapan-20231001-1-en', + 'id': 'livinginjapan-20240901-1-en', 'ext': 'm4a', - 'title': 'Living in Japan - Tips for Travelers to Japan / Ramen Vending Machines', + 'title': 'Living in Japan - Weekend Hiking / Self-protection from crime', 'series': 'Living in Japan', - 'description': 'md5:0a0e2077d8f07a03071e990a6f51bfab', + 'description': 'md5:4d0e14ab73bdbfedb60a53b093954ed6', 'thumbnail': r're:https://.+/.+\.jpg', - 'episode': 'Tips for Travelers to Japan / Ramen Vending Machines', + 'episode': 'Weekend Hiking / Self-protection from crime', + 'categories': ['Interactive'], }, }, { 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2015173/', @@ -256,96 +212,51 @@ class NhkVodIE(NhkBaseIE): }, 'skip': 'expires 2023-10-15', }, { - # a one-off (single-episode series). title from the api is just '

' - 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/3004952/', + # a one-off (single-episode series). title from the api is just null + 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/3026036/', 'info_dict': { - 'id': 'nw_vod_v_en_3004_952_20230723091000_01_1690074552', + 'id': '3026036-en', 'ext': 'mp4', - 'title': 'Barakan Discovers - AMAMI OSHIMA: Isson\'s Treasure Isla', - 'description': 'md5:5db620c46a0698451cc59add8816b797', - 'thumbnail': r're:https://.+/.+\.jpg', - 'release_date': '20230905', - 'timestamp': 1690103400, - 'duration': 2939, - 'release_timestamp': 1693898699, - 'upload_date': '20230723', - 'modified_timestamp': 1707217907, - 'modified_date': '20240206', - 'episode': 'AMAMI OSHIMA: Isson\'s Treasure Isla', - 'series': 'Barakan Discovers', + 'title': 'STATELESS: The Japanese Left Behind in the Philippines', + 'description': 'md5:9a2fd51cdfa9f52baae28569e0053786', + 'duration': 2955, + 'thumbnail': 'https://www3.nhk.or.jp/nhkworld/en/shows/3026036/images/wide_l_QPtWpt4lzVhm3NzPAMIIF35MCg4CdNwcikPaTS5Q.jpg', + 'categories': ['Documentary', 'Culture & Lifestyle'], + 'tags': ['Transcript', 'Documentary 360', 'The Pursuit of PEACE'], + 'timestamp': 1758931800, + 'upload_date': '20250927', + 'release_timestamp': 1758931800, + 'release_date': '20250927', }, }, { # /ondemand/video/ url with alphabetical character in 5th position of id 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a07/', 'info_dict': { - 'id': 'nw_c_en_9999-a07', + 'id': '9999a07-en', 'ext': 'mp4', 'episode': 'Mini-Dramas on SDGs: Ep 1 Close the Gender Gap [Director\'s Cut]', 'series': 'Mini-Dramas on SDGs', - 'modified_date': '20240206', 'title': 'Mini-Dramas on SDGs - Mini-Dramas on SDGs: Ep 1 Close the Gender Gap [Director\'s Cut]', 'description': 'md5:3f9dcb4db22fceb675d90448a040d3f6', - 'timestamp': 1621962360, - 'duration': 189, - 'release_date': '20230903', - 'modified_timestamp': 1707217907, + 'timestamp': 1621911600, + 'duration': 190, 'upload_date': '20210525', 'thumbnail': r're:https://.+/.+\.jpg', - 'release_timestamp': 1693713487, + 'categories': ['Current Affairs', 'Entertainment'], }, }, { 'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999d17/', 'info_dict': { - 'id': 'nw_c_en_9999-d17', + 'id': '9999d17-en', 'ext': 'mp4', 'title': 'Flowers of snow blossom - The 72 Pentads of Yamato', 'description': 'Today’s focus: Snow', - 'release_timestamp': 1693792402, - 'release_date': '20230904', - 'upload_date': '20220128', - 'timestamp': 1643370960, 'thumbnail': r're:https://.+/.+\.jpg', 'duration': 136, - 'series': '', - 'modified_date': '20240206', - 'modified_timestamp': 1707217907, - }, - }, { - # new /shows/ url format - 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/2032307/', - 'info_dict': { - 'id': 'nw_vod_v_en_2032_307_20240321113000_01_1710990282', - 'ext': 'mp4', - 'title': 'Japanology Plus - 20th Anniversary Special Part 1', - 'description': 'md5:817d41fc8e54339ad2a916161ea24faf', - 'episode': '20th Anniversary Special Part 1', - 'series': 'Japanology Plus', - 'thumbnail': r're:https://.+/.+\.jpg', - 'duration': 1680, - 'timestamp': 1711020600, - 'upload_date': '20240321', - 'release_timestamp': 1711022683, - 'release_date': '20240321', - 'modified_timestamp': 1711031012, - 'modified_date': '20240321', - }, - }, { - 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/3020025/', - 'info_dict': { - 'id': 'nw_vod_v_en_3020_025_20230325144000_01_1679723944', - 'ext': 'mp4', - 'title': '100 Ideas to Save the World - Working Styles Evolve', - 'description': 'md5:9e6c7778eaaf4f7b4af83569649f84d9', - 'episode': 'Working Styles Evolve', - 'series': '100 Ideas to Save the World', - 'thumbnail': r're:https://.+/.+\.jpg', - 'duration': 899, - 'upload_date': '20230325', - 'timestamp': 1679755200, - 'release_date': '20230905', - 'release_timestamp': 1693880540, - 'modified_date': '20240206', - 'modified_timestamp': 1707217907, + 'categories': ['Culture & Lifestyle', 'Science & Nature'], + 'tags': ['Nara', 'Temples & Shrines', 'Winter', 'Snow'], + 'timestamp': 1643339040, + 'upload_date': '20220128', }, }, { # new /shows/audio/ url format @@ -373,6 +284,7 @@ class NhkVodProgramIE(NhkBaseIE): 'id': 'sumo', 'title': 'GRAND SUMO Highlights', 'description': 'md5:fc20d02dc6ce85e4b72e0273aa52fdbf', + 'series': 'GRAND SUMO Highlights', }, 'playlist_mincount': 1, }, { @@ -381,6 +293,7 @@ class NhkVodProgramIE(NhkBaseIE): 'id': 'japanrailway', 'title': 'Japan Railway Journal', 'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f', + 'series': 'Japan Railway Journal', }, 'playlist_mincount': 12, }, { @@ -390,6 +303,7 @@ class NhkVodProgramIE(NhkBaseIE): 'id': 'japanrailway', 'title': 'Japan Railway Journal', 'description': 'md5:ea39d93af7d05835baadf10d1aae0e3f', + 'series': 'Japan Railway Journal', }, 'playlist_mincount': 12, }, { @@ -399,17 +313,9 @@ class NhkVodProgramIE(NhkBaseIE): 'id': 'livinginjapan', 'title': 'Living in Japan', 'description': 'md5:665bb36ec2a12c5a7f598ee713fc2b54', + 'series': 'Living in Japan', }, - 'playlist_mincount': 12, - }, { - # /tv/ program url - 'url': 'https://www3.nhk.or.jp/nhkworld/en/tv/designtalksplus/', - 'info_dict': { - 'id': 'designtalksplus', - 'title': 'DESIGN TALKS plus', - 'description': 'md5:47b3b3a9f10d4ac7b33b53b70a7d2837', - }, - 'playlist_mincount': 20, + 'playlist_mincount': 11, }, { 'url': 'https://www3.nhk.or.jp/nhkworld/en/shows/10yearshayaomiyazaki/', 'only_matching': True, @@ -430,9 +336,8 @@ class NhkVodProgramIE(NhkBaseIE): program_id, lang, m_type != 'audio', False, episode_type == 'clip') def entries(): - for episode in episodes: - if episode_path := episode.get('url'): - yield self._extract_episode_info(urljoin(url, episode_path), episode) + for episode in traverse_obj(episodes, ('items', lambda _, v: v['url'])): + yield self._extract_episode_info(urljoin(url, episode['url']), episode) html = self._download_webpage(url, program_id) program_title = self._extract_meta_from_class_elements([ @@ -446,7 +351,7 @@ class NhkVodProgramIE(NhkBaseIE): 'tAudioProgramMain__info', # /shows/audio/programs/ 'p-program-description'], html) # /tv/ - return self.playlist_result(entries(), program_id, program_title, program_description) + return self.playlist_result(entries(), program_id, program_title, program_description, series=program_title) class NhkForSchoolBangumiIE(InfoExtractor): From 26c2545b87e2b22f134d1f567ed4d4b0b91c3253 Mon Sep 17 00:00:00 2001 From: sepro Date: Fri, 28 Nov 2025 23:14:03 +0100 Subject: [PATCH 166/175] [ie/S4C] Fix geo-restricted content (#15196) Closes #15190 Authored by: seproDev --- yt_dlp/extractor/s4c.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/yt_dlp/extractor/s4c.py b/yt_dlp/extractor/s4c.py index 6eb8b2b2c6..d35436d104 100644 --- a/yt_dlp/extractor/s4c.py +++ b/yt_dlp/extractor/s4c.py @@ -15,14 +15,15 @@ class S4CIE(InfoExtractor): 'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Y_Swn_2023S4C_099_ii.jpg', }, }, { - 'url': 'https://www.s4c.cymru/clic/programme/856636948', + # Geo restricted to the UK + 'url': 'https://www.s4c.cymru/clic/programme/886303048', 'info_dict': { - 'id': '856636948', + 'id': '886303048', 'ext': 'mp4', - 'title': 'Am Dro', + 'title': 'Pennod 1', + 'description': 'md5:7e3f364b70f61fcdaa8b4cb4a3eb3e7a', 'duration': 2880, - 'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe', - 'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Am_Dro_2022-23S4C_P6_4005.jpg', + 'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Stad_2025S4C_P1_210053.jpg', }, }] @@ -51,7 +52,7 @@ class S4CIE(InfoExtractor): 'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={ 'mode': 'od', 'application': 'clic', - 'region': 'WW', + 'region': 'UK' if player_config.get('application') == 's4chttpl' else 'WW', 'extra': 'false', 'thirdParty': 'false', 'filename': player_config['filename'], From 280165026886a1f1614ab527c34c66d71faa5d69 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 29 Nov 2025 15:18:49 -0600 Subject: [PATCH 167/175] [build] Bump PyInstaller minimum version requirement to 6.17.0 (#15199) Ref: https://github.com/pyinstaller/pyinstaller/issues/9149 Authored by: bashonly --- .github/workflows/build.yml | 18 +++++++++--------- pyproject.toml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7487f1c2f..8f5df4ce4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -422,23 +422,23 @@ jobs: runner: windows-2025 python_version: '3.10' platform_tag: win_amd64 - pyi_version: '6.16.0' - pyi_tag: '2025.09.13.221251' - pyi_hash: b6496c7630c3afe66900cfa824e8234a8c2e2c81704bd7facd79586abc76c0e5 + pyi_version: '6.17.0' + pyi_tag: '2025.11.29.054325' + pyi_hash: e28cc13e4ad0cc74330d832202806d0c1976e9165da6047309348ca663c0ed3d - arch: 'x86' runner: windows-2025 python_version: '3.10' platform_tag: win32 - pyi_version: '6.16.0' - pyi_tag: '2025.09.13.221251' - pyi_hash: 2d881843580efdc54f3523507fc6d9c5b6051ee49c743a6d9b7003ac5758c226 + pyi_version: '6.17.0' + pyi_tag: '2025.11.29.054325' + pyi_hash: c00f600c17de3bdd589f043f60ab64fc34fcba6dd902ad973af9c8afc74f80d1 - arch: 'arm64' runner: windows-11-arm python_version: '3.13' # arm64 only has Python >= 3.11 available platform_tag: win_arm64 - pyi_version: '6.16.0' - pyi_tag: '2025.09.13.221251' - pyi_hash: 4250c9085e34a95c898f3ee2f764914fc36ec59f0d97c28e6a75fcf21f7b144f + pyi_version: '6.17.0' + pyi_tag: '2025.11.29.054325' + pyi_hash: a2033b18b4f7bc6108b5fd76a92c6c1de0a12ec4fe98a23396a9f978cb4b7d7b env: CHANNEL: ${{ inputs.channel }} ORIGIN: ${{ needs.process.outputs.origin }} diff --git a/pyproject.toml b/pyproject.toml index d2c5745b95..d06e71d74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ build = [ "build", "hatchling>=1.27.0", "pip", - "setuptools>=71.0.2,<81", # See https://github.com/pyinstaller/pyinstaller/issues/9149 + "setuptools>=71.0.2", "wheel", ] dev = [ @@ -86,7 +86,7 @@ test = [ "pytest-rerunfailures~=14.0", ] pyinstaller = [ - "pyinstaller>=6.13.0", # Windows temp cleanup fixed in 6.13.0 + "pyinstaller>=6.17.0", # 6.17.0+ needed for compat with setuptools 81+ ] [project.urls] From 419776ecf57269efb13095386a19ddc75c1f11b2 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sun, 30 Nov 2025 17:35:05 -0600 Subject: [PATCH 168/175] [ie/youtube] Extract all automatic caption languages (#15156) Closes #14889, Closes #15150 Authored by: bashonly --- yt_dlp/extractor/youtube/_video.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 600e0ccda6..a792332046 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -4029,6 +4029,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor): STREAMING_DATA_CLIENT_NAME: client_name, }) + def set_audio_lang_from_orig_subs_lang(lang_code): + for f in formats: + if f.get('acodec') != 'none' and not f.get('language'): + f['language'] = lang_code + subtitles = {} skipped_subs_clients = set() @@ -4088,7 +4093,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): orig_lang = qs.get('lang', [None])[-1] lang_name = self._get_text(caption_track, 'name', max_runs=1) - if caption_track.get('kind') != 'asr': + is_manual_subs = caption_track.get('kind') != 'asr' + if is_manual_subs: if not lang_code: continue process_language( @@ -4099,16 +4105,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if not trans_code: continue orig_trans_code = trans_code - if caption_track.get('kind') != 'asr' and trans_code != 'und': + if is_manual_subs and trans_code != 'und': if not get_translated_subs: continue trans_code += f'-{lang_code}' trans_name += format_field(lang_name, None, ' from %s') if lang_code == f'a-{orig_trans_code}': # Set audio language based on original subtitles - for f in formats: - if f.get('acodec') != 'none' and not f.get('language'): - f['language'] = orig_trans_code + set_audio_lang_from_orig_subs_lang(orig_trans_code) # Add an "-orig" label to the original language so that it can be distinguished. # The subs are returned without "-orig" as well for compatibility process_language( @@ -4119,6 +4123,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor): automatic_captions, base_url, trans_code, trans_name, client_name, pot_params if orig_lang == orig_trans_code else {'tlang': trans_code, **pot_params}) + # Extract automatic captions when the language is not in 'translationLanguages' + # e.g. Cantonese [yue], see https://github.com/yt-dlp/yt-dlp/issues/14889 + lang_code = remove_start(lang_code, 'a-') + if is_manual_subs or not lang_code or lang_code in automatic_captions: + continue + lang_name = remove_end(lang_name, ' (auto-generated)') + if caption_track.get('isTranslatable'): + # We can assume this is the original audio language + set_audio_lang_from_orig_subs_lang(lang_code) + process_language( + automatic_captions, base_url, f'{lang_code}-orig', + f'{lang_name} (Original)', client_name, pot_params) + process_language( + automatic_captions, base_url, lang_code, lang_name, client_name, pot_params) + # Avoid duplication if we've already got everything we need need_subs_langs.difference_update(subtitles) need_caps_langs.difference_update(automatic_captions) From 4433b3a217c9f430dc057643bfd7b6769eff4a45 Mon Sep 17 00:00:00 2001 From: Zer0 Spectrum Date: Mon, 1 Dec 2025 05:24:17 +0530 Subject: [PATCH 169/175] [ie/fc2:live] Raise appropriate error when stream is offline (#15180) Closes #15179 Authored by: Zer0spectrum --- yt_dlp/extractor/fc2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yt_dlp/extractor/fc2.py b/yt_dlp/extractor/fc2.py index d343069fec..aa6ff6335d 100644 --- a/yt_dlp/extractor/fc2.py +++ b/yt_dlp/extractor/fc2.py @@ -5,6 +5,7 @@ from .common import InfoExtractor from ..networking import Request from ..utils import ( ExtractorError, + UserNotLive, js_to_json, traverse_obj, update_url_query, @@ -205,6 +206,9 @@ class FC2LiveIE(InfoExtractor): 'client_app': 'browser_hls', 'ipv6': '', }), headers={'X-Requested-With': 'XMLHttpRequest'}) + # A non-zero 'status' indicates the stream is not live, so check truthiness + if traverse_obj(control_server, ('status', {int})) and 'control_token' not in control_server: + raise UserNotLive(video_id=video_id) self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw']) ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']}) From 023e4db9afe0630c608621846856a1ca876d8bab Mon Sep 17 00:00:00 2001 From: thomasmllt <164891804+thomasmllt@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:59:28 +0100 Subject: [PATCH 170/175] [ie/patreon:campaign] Fix extractor (#15108) Closes #15094 Authored by: thomasmllt --- yt_dlp/extractor/patreon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py index 9038b4a7ff..b511994e8a 100644 --- a/yt_dlp/extractor/patreon.py +++ b/yt_dlp/extractor/patreon.py @@ -598,7 +598,8 @@ class PatreonCampaignIE(PatreonBaseIE): 'props', 'pageProps', 'bootstrapEnvelope', 'pageBootstrap', 'campaign', 'data', 'id', {str})) if not campaign_id: campaign_id = traverse_obj(self._search_nextjs_v13_data(webpage, vanity), ( - lambda _, v: v['type'] == 'campaign', 'id', {str}, any, {require('campaign ID')})) + ((..., 'value', 'campaign', 'data'), lambda _, v: v['type'] == 'campaign'), + 'id', {str}, any, {require('campaign ID')})) params = { 'json-api-use-default-includes': 'false', From 2a777ecbd598de19a4c691ba1f790ccbec9cdbc4 Mon Sep 17 00:00:00 2001 From: Zer0 Spectrum Date: Mon, 1 Dec 2025 06:03:14 +0530 Subject: [PATCH 171/175] [ie/tubitv:series] Fix extractor (#15018) Authored by: Zer0spectrum --- yt_dlp/extractor/tubitv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/tubitv.py b/yt_dlp/extractor/tubitv.py index 694a92fcd4..bb9a293b86 100644 --- a/yt_dlp/extractor/tubitv.py +++ b/yt_dlp/extractor/tubitv.py @@ -182,13 +182,13 @@ class TubiTvShowIE(InfoExtractor): webpage = self._download_webpage(show_url, playlist_id) data = self._search_json( - r'window\.__data\s*=', webpage, 'data', playlist_id, - transform_source=js_to_json)['video'] + r'window\.__REACT_QUERY_STATE__\s*=', webpage, 'data', playlist_id, + transform_source=js_to_json)['queries'][0]['state']['data'] # v['number'] is already a decimal string, but stringify to protect against API changes path = [lambda _, v: str(v['number']) == selected_season] if selected_season else [..., {dict}] - for season in traverse_obj(data, ('byId', lambda _, v: v['type'] == 's', 'seasons', *path)): + for season in traverse_obj(data, ('seasons', *path)): season_number = int_or_none(season.get('number')) for episode in traverse_obj(season, ('episodes', lambda _, v: v['id'])): episode_id = episode['id'] From 56ea3a00eabb45d926a6b993708acf1b9951e23a Mon Sep 17 00:00:00 2001 From: WhatAmISupposedToPutHere Date: Mon, 1 Dec 2025 02:02:58 +0100 Subject: [PATCH 172/175] [ie/youtube] Add `request_no_ads` extractor-arg (#15145) Default is `true` for unauthenticated users. Default is `false` if logged-in cookies have been passed to yt-dlp. Using `true` results in a loss of premium formats. Closes #15144 Authored by: WhatAmISupposedToPutHere --- README.md | 1 + yt_dlp/extractor/youtube/_video.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c98c69f418..6a97eb0eb2 100644 --- a/README.md +++ b/README.md @@ -1871,6 +1871,7 @@ The following extractors use this feature: * `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) * `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) * `jsc_trace`: Enable debug logging for JS Challenge fetching. Either `true` or `false` (default) +* `request_no_ads`: Skip preroll ads to eliminate the mandatory wait period before download. Either `true` (the default if unauthenticated) or `false`. The default is `false` when logged-in cookies have been passed to yt-dlp, since `true` will result in a loss of premium formats #### youtube-ejs * `jitless`: Run suported Javascript engines in JIT-less mode. Supported runtimes are `deno`, `node` and `bun`. Provides better security at the cost of performance/speed. Do note that `node` and `bun` are still considered unsecure. Either `true` or `false` (default) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index a792332046..0756ce2c40 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -2628,18 +2628,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _get_checkok_params(): return {'contentCheckOk': True, 'racyCheckOk': True} - @classmethod - def _generate_player_context(cls, sts=None): + def _generate_player_context(self, sts=None): context = { 'html5Preference': 'HTML5_PREF_WANTS', } if sts is not None: context['signatureTimestamp'] = sts + + playback_context = { + 'contentPlaybackContext': context, + } + + # The 'adPlaybackContext'/'request_no_ads' workaround results in a loss of premium formats. + # Only default to 'true' if the user is unauthenticated, since we can't reliably detect all + # types of premium accounts (e.g. YTMusic Premium), and since premium users don't have ads. + default_arg_value = 'false' if self.is_authenticated else 'true' + if self._configuration_arg('request_no_ads', [default_arg_value])[0] != 'false': + playback_context['adPlaybackContext'] = { + 'pyv': True, + } + return { - 'playbackContext': { - 'contentPlaybackContext': context, - }, - **cls._get_checkok_params(), + 'playbackContext': playback_context, + **self._get_checkok_params(), } def _get_config_po_token(self, client: str, context: _PoTokenContext): From 017d76edcf05186df2073bdb5b8351a87f596f4c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sun, 30 Nov 2025 23:01:22 -0600 Subject: [PATCH 173/175] [ie/youtube] Revert 56ea3a00eabb45d926a6b993708acf1b9951e23a Remove `request_no_ads` workaround (#15214) Closes #15212 Authored by: bashonly --- README.md | 1 - yt_dlp/extractor/youtube/_video.py | 23 ++++++----------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6a97eb0eb2..c98c69f418 100644 --- a/README.md +++ b/README.md @@ -1871,7 +1871,6 @@ The following extractors use this feature: * `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) * `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) * `jsc_trace`: Enable debug logging for JS Challenge fetching. Either `true` or `false` (default) -* `request_no_ads`: Skip preroll ads to eliminate the mandatory wait period before download. Either `true` (the default if unauthenticated) or `false`. The default is `false` when logged-in cookies have been passed to yt-dlp, since `true` will result in a loss of premium formats #### youtube-ejs * `jitless`: Run suported Javascript engines in JIT-less mode. Supported runtimes are `deno`, `node` and `bun`. Provides better security at the cost of performance/speed. Do note that `node` and `bun` are still considered unsecure. Either `true` or `false` (default) diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index 0756ce2c40..a792332046 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -2628,29 +2628,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _get_checkok_params(): return {'contentCheckOk': True, 'racyCheckOk': True} - def _generate_player_context(self, sts=None): + @classmethod + def _generate_player_context(cls, sts=None): context = { 'html5Preference': 'HTML5_PREF_WANTS', } if sts is not None: context['signatureTimestamp'] = sts - - playback_context = { - 'contentPlaybackContext': context, - } - - # The 'adPlaybackContext'/'request_no_ads' workaround results in a loss of premium formats. - # Only default to 'true' if the user is unauthenticated, since we can't reliably detect all - # types of premium accounts (e.g. YTMusic Premium), and since premium users don't have ads. - default_arg_value = 'false' if self.is_authenticated else 'true' - if self._configuration_arg('request_no_ads', [default_arg_value])[0] != 'false': - playback_context['adPlaybackContext'] = { - 'pyv': True, - } - return { - 'playbackContext': playback_context, - **self._get_checkok_params(), + 'playbackContext': { + 'contentPlaybackContext': context, + }, + **cls._get_checkok_params(), } def _get_config_po_token(self, client: str, context: _PoTokenContext): From f7acf3c1f42cc474927ecc452205d7877af36731 Mon Sep 17 00:00:00 2001 From: WhatAmISupposedToPutHere Date: Thu, 4 Dec 2025 00:26:20 +0100 Subject: [PATCH 174/175] [ie/youtube] Add `use_ad_playback_context` extractor-arg (#15220) Closes #15144 Authored by: WhatAmISupposedToPutHere --- README.md | 1 + yt_dlp/extractor/youtube/_base.py | 5 +++++ yt_dlp/extractor/youtube/_video.py | 23 ++++++++++++++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c98c69f418..48f8893411 100644 --- a/README.md +++ b/README.md @@ -1871,6 +1871,7 @@ The following extractors use this feature: * `pot_trace`: Enable debug logging for PO Token fetching. Either `true` or `false` (default) * `fetch_pot`: Policy to use for fetching a PO Token from providers. One of `always` (always try fetch a PO Token regardless if the client requires one for the given context), `never` (never fetch a PO Token), or `auto` (default; only fetch a PO Token if the client requires one for the given context) * `jsc_trace`: Enable debug logging for JS Challenge fetching. Either `true` or `false` (default) +* `use_ad_playback_context`: Skip preroll ads to eliminate the mandatory wait period before download. Do NOT use this when passing premium account cookies to yt-dlp, as it will result in a loss of premium formats. Only effective with the `web`, `web_safari`, `web_music` and `mweb` player clients. Either `true` or `false` (default) #### youtube-ejs * `jitless`: Run suported Javascript engines in JIT-less mode. Supported runtimes are `deno`, `node` and `bun`. Provides better security at the cost of performance/speed. Do note that `node` and `bun` are still considered unsecure. Either `true` or `false` (default) diff --git a/yt_dlp/extractor/youtube/_base.py b/yt_dlp/extractor/youtube/_base.py index 9ecce15553..114eee821b 100644 --- a/yt_dlp/extractor/youtube/_base.py +++ b/yt_dlp/extractor/youtube/_base.py @@ -104,6 +104,7 @@ INNERTUBE_CLIENTS = { }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, 'SUPPORTS_COOKIES': True, + 'SUPPORTS_AD_PLAYBACK_CONTEXT': True, **WEB_PO_TOKEN_POLICIES, }, # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats @@ -117,6 +118,7 @@ INNERTUBE_CLIENTS = { }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, 'SUPPORTS_COOKIES': True, + 'SUPPORTS_AD_PLAYBACK_CONTEXT': True, **WEB_PO_TOKEN_POLICIES, }, 'web_embedded': { @@ -157,6 +159,7 @@ INNERTUBE_CLIENTS = { ), }, 'SUPPORTS_COOKIES': True, + 'SUPPORTS_AD_PLAYBACK_CONTEXT': True, }, # This client now requires sign-in for every video 'web_creator': { @@ -313,6 +316,7 @@ INNERTUBE_CLIENTS = { ), }, 'SUPPORTS_COOKIES': True, + 'SUPPORTS_AD_PLAYBACK_CONTEXT': True, }, 'tv': { 'INNERTUBE_CONTEXT': { @@ -412,6 +416,7 @@ def build_innertube_clients(): ytcfg.setdefault('SUBS_PO_TOKEN_POLICY', SubsPoTokenPolicy()) ytcfg.setdefault('REQUIRE_AUTH', False) ytcfg.setdefault('SUPPORTS_COOKIES', False) + ytcfg.setdefault('SUPPORTS_AD_PLAYBACK_CONTEXT', False) ytcfg.setdefault('PLAYER_PARAMS', None) ytcfg.setdefault('AUTHENTICATED_USER_AGENT', None) ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en') diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py index a792332046..24c4458d61 100644 --- a/yt_dlp/extractor/youtube/_video.py +++ b/yt_dlp/extractor/youtube/_video.py @@ -2629,16 +2629,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return {'contentCheckOk': True, 'racyCheckOk': True} @classmethod - def _generate_player_context(cls, sts=None): + def _generate_player_context(cls, sts=None, use_ad_playback_context=False): context = { 'html5Preference': 'HTML5_PREF_WANTS', } if sts is not None: context['signatureTimestamp'] = sts + + playback_context = { + 'contentPlaybackContext': context, + } + if use_ad_playback_context: + playback_context['adPlaybackContext'] = { + 'pyv': True, + } + return { - 'playbackContext': { - 'contentPlaybackContext': context, - }, + 'playbackContext': playback_context, **cls._get_checkok_params(), } @@ -2866,7 +2873,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor): yt_query['serviceIntegrityDimensions'] = {'poToken': po_token} sts = self._extract_signature_timestamp(video_id, player_url, webpage_ytcfg, fatal=False) if player_url else None - yt_query.update(self._generate_player_context(sts)) + + use_ad_playback_context = ( + self._configuration_arg('use_ad_playback_context', ['false'])[0] != 'false' + and traverse_obj(INNERTUBE_CLIENTS, (client, 'SUPPORTS_AD_PLAYBACK_CONTEXT', {bool}))) + + yt_query.update(self._generate_player_context(sts, use_ad_playback_context)) + return self._extract_response( item_id=video_id, ep='player', query=yt_query, ytcfg=player_ytcfg, headers=headers, fatal=True, From 7ec6b9bc40ee8a21b11cce83a09a07a37014062e Mon Sep 17 00:00:00 2001 From: sepro Date: Thu, 4 Dec 2025 18:15:09 +0100 Subject: [PATCH 175/175] [ie/web.archive:youtube] Fix extractor (#15234) Closes #15233 Authored by: seproDev --- yt_dlp/extractor/archiveorg.py | 50 ++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py index 3746c58fb7..02c39beb68 100644 --- a/yt_dlp/extractor/archiveorg.py +++ b/yt_dlp/extractor/archiveorg.py @@ -704,6 +704,24 @@ class YoutubeWebArchiveIE(InfoExtractor): 'thumbnail': 'https://web.archive.org/web/20160108040020if_/https://i.ytimg.com/vi/SQCom7wjGDs/maxresdefault.jpg', 'upload_date': '20160107', }, + }, { + # dmuxed formats + 'url': 'https://web.archive.org/web/20240922160632/https://www.youtube.com/watch?v=z7hzvTL3k1k', + 'info_dict': { + 'id': 'z7hzvTL3k1k', + 'ext': 'webm', + 'title': 'Praise the Lord and Pass the Ammunition (BARRXN REMIX)', + 'description': 'md5:45dbf2c71c23b0734c8dfb82dd1e94b6', + 'uploader': 'Barrxn', + 'uploader_id': 'TheRockstar6086', + 'uploader_url': 'https://www.youtube.com/user/TheRockstar6086', + 'channel_id': 'UCjJPGUTtvR9uizmawn2ThqA', + 'channel_url': 'https://www.youtube.com/channel/UCjJPGUTtvR9uizmawn2ThqA', + 'duration': 125, + 'thumbnail': r're:https?://.*\.(jpg|webp)', + 'upload_date': '20201207', + }, + 'params': {'format': 'bv'}, }, { 'url': 'https://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFw', 'only_matching': True, @@ -1060,6 +1078,19 @@ class YoutubeWebArchiveIE(InfoExtractor): capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE]) return orderedSet(filter(None, capture_dates)) + def _parse_fmt(self, fmt, extra_info=None): + format_id = traverse_obj(fmt, ('url', {parse_qs}, 'itag', 0)) + return { + 'format_id': format_id, + **self._FORMATS.get(format_id, {}), + **traverse_obj(fmt, { + 'url': ('url', {lambda x: f'https://web.archive.org/web/2id_/{x}'}), + 'ext': ('ext', {str}), + 'filesize': ('url', {parse_qs}, 'clen', 0, {int_or_none}), + }), + **(extra_info or {}), + } + def _real_extract(self, url): video_id, url_date, url_date_2 = self._match_valid_url(url).group('id', 'date', 'date2') url_date = url_date or url_date_2 @@ -1090,17 +1121,14 @@ class YoutubeWebArchiveIE(InfoExtractor): info['thumbnails'] = self._extract_thumbnails(video_id) formats = [] - for fmt in traverse_obj(video_info, ('formats', lambda _, v: url_or_none(v['url']))): - format_id = traverse_obj(fmt, ('url', {parse_qs}, 'itag', 0)) - formats.append({ - 'format_id': format_id, - **self._FORMATS.get(format_id, {}), - **traverse_obj(fmt, { - 'url': ('url', {lambda x: f'https://web.archive.org/web/2id_/{x}'}), - 'ext': ('ext', {str}), - 'filesize': ('url', {parse_qs}, 'clen', 0, {int_or_none}), - }), - }) + if video_info.get('dmux'): + for vf in traverse_obj(video_info, ('formats', 'video', lambda _, v: url_or_none(v['url']))): + formats.append(self._parse_fmt(vf, {'acodec': 'none'})) + for af in traverse_obj(video_info, ('formats', 'audio', lambda _, v: url_or_none(v['url']))): + formats.append(self._parse_fmt(af, {'vcodec': 'none'})) + else: + for fmt in traverse_obj(video_info, ('formats', lambda _, v: url_or_none(v['url']))): + formats.append(self._parse_fmt(fmt)) info['formats'] = formats return info