diff --git a/.github/actionlint.yml b/.github/actionlint.yml index d6d5b9abd2..bdd3901a37 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -1,9 +1,3 @@ -self-hosted-runner: - labels: - # Workaround for the outdated runner list in actionlint v1.7.7 - # Ref: https://github.com/rhysd/actionlint/issues/533 - - windows-11-arm - config-variables: - KEEP_CACHE_WARM - PUSH_VERSION_COMMIT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69f8eb0c8d..b7487f1c2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -153,10 +153,12 @@ jobs: 'os': 'musllinux', 'arch': 'x86_64', 'runner': 'ubuntu-24.04', + 'python_version': '3.14', }, { 'os': 'musllinux', 'arch': 'aarch64', 'runner': 'ubuntu-24.04-arm', + 'python_version': '3.14', }], } INPUTS = json.loads(os.environ['INPUTS']) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index ae3dc95e1b..3cb17f2b7d 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -56,6 +56,8 @@ jobs: python-version: pypy-3.11 steps: - uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -65,6 +67,25 @@ jobs: - name: Run tests timeout-minutes: 15 continue-on-error: False + env: + source: ${{ (github.event_name == 'push' && github.event.before) || 'origin/master' }} + target: ${{ (github.event_name == 'push' && github.event.after) || 'HEAD' }} + shell: bash run: | + flags=() + # Check if a networking file is involved + patterns="\ + ^yt_dlp/networking/ + ^yt_dlp/utils/networking\.py$ + ^test/test_http_proxy\.py$ + ^test/test_networking\.py$ + ^test/test_networking_utils\.py$ + ^test/test_socks\.py$ + ^test/test_websockets\.py$ + ^pyproject\.toml$ + " + if git diff --name-only "${source}" "${target}" | grep -Ef <(printf '%s' "${patterns}"); then + flags+=(--flaky) + fi python3 -m yt_dlp -v || true # Print debug head - python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core + python3 -m devscripts.run_tests "${flags[@]}" --pytest-args '--reruns 2 --reruns-delay 3.0' core diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index 37bf044d69..d39ab8814b 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -17,8 +17,8 @@ on: permissions: contents: read env: - ACTIONLINT_VERSION: "1.7.7" - ACTIONLINT_SHA256SUM: 023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 + ACTIONLINT_VERSION: "1.7.8" + ACTIONLINT_SHA256SUM: be92c2652ab7b6d08425428797ceabeb16e31a781c07bc388456b4e592f3e36a ACTIONLINT_REPO: https://github.com/rhysd/actionlint jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89327581c0..99f18b2f32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -177,7 +177,7 @@ While it is strongly recommended to use `hatch` for yt-dlp development, if you a ```shell # To only install development dependencies: -$ python -m devscripts.install_deps --include dev +$ python -m devscripts.install_deps --include-group dev # Or, for an editable install plus dev dependencies: $ python -m pip install -e ".[default,dev]" diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 888d48d561..51369b35b6 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -818,3 +818,19 @@ robin-mu shssoichiro thanhtaivtt uoag +CaramelConnoisseur +ctengel +einstein95 +evilpie +i3p9 +JrM2628 +krystophny +matyb08 +pha1n0q +PierceLBrooks +sepro +TheQWERTYCodr +thomasmllt +w4grfw +WeidiDeng +Zer0spectrum diff --git a/Changelog.md b/Changelog.md index 8737441e86..b115fd045c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,71 @@ # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2025.11.12 + +#### Important changes +- **An external JavaScript runtime is now required for full YouTube support** +yt-dlp now requires users to have an external JavaScript runtime (e.g. Deno) installed in order to solve the JavaScript challenges presented by YouTube. [Read more](https://github.com/yt-dlp/yt-dlp/issues/15012) + +#### Core changes +- **cookies** + - [Allow `--cookies-from-browser` for Safari on iOS](https://github.com/yt-dlp/yt-dlp/commit/e6414d64e73d86d65bb357e5ad59d0ca080d5812) ([#14950](https://github.com/yt-dlp/yt-dlp/issues/14950)) by [pha1n0q](https://github.com/pha1n0q) + - [Support Firefox cookies database v17](https://github.com/yt-dlp/yt-dlp/commit/bf7e04e9d8bd3c4a4614b67ce617b7ae5d17d62a) ([#15010](https://github.com/yt-dlp/yt-dlp/issues/15010)) by [Grub4K](https://github.com/Grub4K) +- **sponsorblock**: [Add `hook` category](https://github.com/yt-dlp/yt-dlp/commit/52f3c56e83bbb25eec2496b0499768753732a093) ([#14845](https://github.com/yt-dlp/yt-dlp/issues/14845)) by [seproDev](https://github.com/seproDev) +- **update**: [Fix PyInstaller onedir variant detection](https://github.com/yt-dlp/yt-dlp/commit/1c2ad94353d1c9e03615d20b6bbfc293286c7a32) ([#14800](https://github.com/yt-dlp/yt-dlp/issues/14800)) by [bashonly](https://github.com/bashonly) + +#### Extractor changes +- **1tv**: live: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/19c5d7c53013440ec4f3f56ebbb067531b272f3f) ([#14299](https://github.com/yt-dlp/yt-dlp/issues/14299)) by [swayll](https://github.com/swayll) +- **ardaudiothek**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/0046fbcbfceee32fa2f68a8ea00cca02765470b6) ([#14309](https://github.com/yt-dlp/yt-dlp/issues/14309)) by [evilpie](https://github.com/evilpie), [marieell](https://github.com/marieell) +- **bunnycdn** + - [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/228ae9f0f2b441fa1296db2ed2b7afbd4a9a62a1) ([#14954](https://github.com/yt-dlp/yt-dlp/issues/14954)) by [seproDev](https://github.com/seproDev) + - [Support player subdomain URLs](https://github.com/yt-dlp/yt-dlp/commit/3ef867451cd9604b4195dfee00db768619629b2d) ([#14979](https://github.com/yt-dlp/yt-dlp/issues/14979)) by [einstein95](https://github.com/einstein95) +- **discoverynetworksde**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/10dea209d2460daf924c93835ddc2f0301cf2cd4) ([#14818](https://github.com/yt-dlp/yt-dlp/issues/14818)) by [dirkf](https://github.com/dirkf), [w4grfw](https://github.com/w4grfw) (With fixes in [f3c255b](https://github.com/yt-dlp/yt-dlp/commit/f3c255b63bd26069151fc3d3ba6dc626bb62ad6e) by [bashonly](https://github.com/bashonly)) +- **floatplane**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1ac7e6005cd3be9fff0b28be189c3a68ecd4c593) ([#14984](https://github.com/yt-dlp/yt-dlp/issues/14984)) by [i3p9](https://github.com/i3p9) +- **googledrive** + - [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/6d05cee4df30774ddce5c5c751fd2118f40c24fe) ([#14809](https://github.com/yt-dlp/yt-dlp/issues/14809)) by [seproDev](https://github.com/seproDev) + - [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/70f1098312fe53bc85358f7bd624370878b2fa28) ([#14746](https://github.com/yt-dlp/yt-dlp/issues/14746)) by [seproDev](https://github.com/seproDev) +- **kika**: [Do not extract non-existent subtitles](https://github.com/yt-dlp/yt-dlp/commit/79f9232ffbd57dde91c372b673b42801edaa9e53) ([#14813](https://github.com/yt-dlp/yt-dlp/issues/14813)) by [InvalidUsernameException](https://github.com/InvalidUsernameException) +- **mux**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a0bda3b78609593ce1127215fc035c1a308a89b6) ([#14914](https://github.com/yt-dlp/yt-dlp/issues/14914)) by [PierceLBrooks](https://github.com/PierceLBrooks), [seproDev](https://github.com/seproDev) +- **nascarclassics**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e8a6b1ca92f2a0ce2c187668165be23dc5506aab) ([#14866](https://github.com/yt-dlp/yt-dlp/issues/14866)) by [JrM2628](https://github.com/JrM2628) +- **nbc**: [Detect and discard DRM formats](https://github.com/yt-dlp/yt-dlp/commit/ee3a106f34124f0e2d28f062f5302863fd7639be) ([#14844](https://github.com/yt-dlp/yt-dlp/issues/14844)) by [bashonly](https://github.com/bashonly) +- **ntv.ru**: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/5dde0d0c9fcef2ce57e486b2e563e0dff9b2845a) ([#14934](https://github.com/yt-dlp/yt-dlp/issues/14934)) by [anlar](https://github.com/anlar), [seproDev](https://github.com/seproDev) (With fixes in [a86eeaa](https://github.com/yt-dlp/yt-dlp/commit/a86eeaadf236ceaf6bb232eb410cf21572538aa6) by [seproDev](https://github.com/seproDev)) +- **play.tv**: [Update extractor for new domain](https://github.com/yt-dlp/yt-dlp/commit/73fd850d170e01c47c31aaa6aa8fe90856d9ad18) ([#14905](https://github.com/yt-dlp/yt-dlp/issues/14905)) by [thomasmllt](https://github.com/thomasmllt) +- **tubetugraz**: [Support alternate URL format](https://github.com/yt-dlp/yt-dlp/commit/f3597cfafcab4d7d4c6d41bff3647681301f1e6b) ([#14718](https://github.com/yt-dlp/yt-dlp/issues/14718)) by [krystophny](https://github.com/krystophny) +- **twitch** + - [Fix playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/cb78440e468608fd55546280b537387d375335f2) ([#15008](https://github.com/yt-dlp/yt-dlp/issues/15008)) by [bashonly](https://github.com/bashonly), [ctengel](https://github.com/ctengel) + - stream: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7eff676183518175ce495ae63291c89f9b39f02a) ([#14988](https://github.com/yt-dlp/yt-dlp/issues/14988)) by [seproDev](https://github.com/seproDev) + - vod: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/b46c572b26be15683584102c5fb7e7bfde0c9821) ([#14999](https://github.com/yt-dlp/yt-dlp/issues/14999)) by [Zer0spectrum](https://github.com/Zer0spectrum) +- **urplay**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/808b1fed76fbd07840cc23a346c11334e3d34f43) ([#14785](https://github.com/yt-dlp/yt-dlp/issues/14785)) by [seproDev](https://github.com/seproDev) +- **web.archive**: youtube: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/d9e3011fd1c3a75871a50e78533afe78ad427ce3) ([#14753](https://github.com/yt-dlp/yt-dlp/issues/14753)) by [seproDev](https://github.com/seproDev) +- **xhamster**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/a1d6351c3fc82c07fa0ee70811ed84807f6bbb58) ([#14948](https://github.com/yt-dlp/yt-dlp/issues/14948)) by [CaramelConnoisseur](https://github.com/CaramelConnoisseur), [dhwz](https://github.com/dhwz) +- **youtube** + - [Add `tv_downgraded` client](https://github.com/yt-dlp/yt-dlp/commit/61cf34f5447177a73ba25ea9a47d7df516ca3b3b) ([#14887](https://github.com/yt-dlp/yt-dlp/issues/14887)) by [seproDev](https://github.com/seproDev) (With fixes in [fa35eb2](https://github.com/yt-dlp/yt-dlp/commit/fa35eb27eaf27df7b5854f527a89fc828c9e0ec0)) + - [Fix `web_embedded` client extraction](https://github.com/yt-dlp/yt-dlp/commit/d6ee67725397807bbb5edcd0b2c94f5bca62d3f4) ([#14843](https://github.com/yt-dlp/yt-dlp/issues/14843)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) + - [Fix auto-generated metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/a56217f9f6c594f6c419ce8dce9134198a9d90d0) ([#13896](https://github.com/yt-dlp/yt-dlp/issues/13896)) by [TheQWERTYCodr](https://github.com/TheQWERTYCodr) + - [Fix original language detection](https://github.com/yt-dlp/yt-dlp/commit/afc44022d0b736b2b3e87b52490bd35c53c53632) ([#14919](https://github.com/yt-dlp/yt-dlp/issues/14919)) by [bashonly](https://github.com/bashonly) + - [Implement external n/sig solver](https://github.com/yt-dlp/yt-dlp/commit/6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc) ([#14157](https://github.com/yt-dlp/yt-dlp/issues/14157)) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) (With fixes in [4b4223b](https://github.com/yt-dlp/yt-dlp/commit/4b4223b436fb03a12628679daed32ae4fc15ae4b), [ee98be4](https://github.com/yt-dlp/yt-dlp/commit/ee98be4ad767b77e4d8dd9bfd3c7d10f2e8397ff), [c0c9f30](https://github.com/yt-dlp/yt-dlp/commit/c0c9f30695db314df084e8701a7c376eb54f283c), [cacd163](https://github.com/yt-dlp/yt-dlp/commit/cacd1630a1a59e92f857d0d175c8730cffbf9801), [8636a9b](https://github.com/yt-dlp/yt-dlp/commit/8636a9bac3bed99984c1e297453660468ecf504b)) + - [Support collaborators](https://github.com/yt-dlp/yt-dlp/commit/f87cfadb5c3cba8e9dc4231c9554548e9edb3882) ([#14677](https://github.com/yt-dlp/yt-dlp/issues/14677)) by [seproDev](https://github.com/seproDev) + - tab: [Fix duration extraction for feeds](https://github.com/yt-dlp/yt-dlp/commit/1d2f0edaf978a5541cfb8f7e83fec433c65c1011) ([#14668](https://github.com/yt-dlp/yt-dlp/issues/14668)) by [WeidiDeng](https://github.com/WeidiDeng) + +#### Downloader changes +- **ffmpeg** + - [Apply `ffmpeg_args` for each format](https://github.com/yt-dlp/yt-dlp/commit/ffb7b7f446b6c67a28c66598ae91f4f2263e0d75) ([#14886](https://github.com/yt-dlp/yt-dlp/issues/14886)) by [bashonly](https://github.com/bashonly) + - [Limit read rate for DASH livestreams](https://github.com/yt-dlp/yt-dlp/commit/7af6d81f35aea8832023daa30ada10e6673a0529) ([#14918](https://github.com/yt-dlp/yt-dlp/issues/14918)) by [bashonly](https://github.com/bashonly) + +#### Networking changes +- [Ensure underlying file object is closed when fully read](https://github.com/yt-dlp/yt-dlp/commit/5767fb4ab108dddb07fc839a3b0f4d323a7c4bea) ([#14935](https://github.com/yt-dlp/yt-dlp/issues/14935)) by [coletdjnz](https://github.com/coletdjnz) + +#### Misc. changes +- [Fix zsh path argument completion](https://github.com/yt-dlp/yt-dlp/commit/c96e9291ab7bd6e7da66d33424982c8b0b4431c7) ([#14953](https://github.com/yt-dlp/yt-dlp/issues/14953)) by [matyb08](https://github.com/matyb08) +- **build**: [Bump musllinux Python version to 3.14](https://github.com/yt-dlp/yt-dlp/commit/646904cd3a79429ec5fdc43f904b3f57ae213f34) ([#14623](https://github.com/yt-dlp/yt-dlp/issues/14623)) by [bashonly](https://github.com/bashonly) +- **cleanup** + - Miscellaneous + - [c63b4e2](https://github.com/yt-dlp/yt-dlp/commit/c63b4e2a2b81cc78397c8709ef53ffd29bada213) by [bashonly](https://github.com/bashonly), [matyb08](https://github.com/matyb08), [sepro](https://github.com/sepro) + - [335653b](https://github.com/yt-dlp/yt-dlp/commit/335653be82d5ef999cfc2879d005397402eebec1) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev) +- **devscripts**: [Improve `install_deps` script](https://github.com/yt-dlp/yt-dlp/commit/73922e66e437fb4bb618bdc119a96375081bf508) ([#14766](https://github.com/yt-dlp/yt-dlp/issues/14766)) by [bashonly](https://github.com/bashonly) +- **test**: [Skip flaky tests if source unchanged](https://github.com/yt-dlp/yt-dlp/commit/ade8c2b36ff300edef87d48fd1ba835ac35c5b63) ([#14970](https://github.com/yt-dlp/yt-dlp/issues/14970)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K) + ### 2025.10.22 #### Important changes diff --git a/Maintainers.md b/Maintainers.md index 8b52daf5fa..515505d882 100644 --- a/Maintainers.md +++ b/Maintainers.md @@ -10,6 +10,8 @@ Core Maintainers are responsible for reviewing and merging contributions, publis **You can contact the core maintainers via `maintainers@yt-dlp.org`.** +This is **NOT** a support channel. [Open an issue](https://github.com/yt-dlp/yt-dlp/issues/new/choose) if you need help or want to report a bug. + ### [coletdjnz](https://github.com/coletdjnz) [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz) diff --git a/Makefile b/Makefile index 88727219b8..89aef9033b 100644 --- a/Makefile +++ b/Makefile @@ -202,9 +202,9 @@ CONTRIBUTORS: Changelog.md # The following EJS_-prefixed variables are auto-generated by devscripts/update_ejs.py # DO NOT EDIT! -EJS_VERSION = 0.3.0 -EJS_WHEEL_NAME = yt_dlp_ejs-0.3.0-py3-none-any.whl -EJS_WHEEL_HASH = sha256:abbf269fa1674cab7b7b266e51e89e0e60b01a11a0fdf3cd63528683190cdd07 +EJS_VERSION = 0.3.1 +EJS_WHEEL_NAME = yt_dlp_ejs-0.3.1-py3-none-any.whl +EJS_WHEEL_HASH = sha256:a6e3548874db7c774388931752bb46c7f4642c044b2a189e56968f3d5ecab622 EJS_PY_FOLDERS = yt_dlp_ejs yt_dlp_ejs/yt yt_dlp_ejs/yt/solver EJS_PY_FILES = yt_dlp_ejs/__init__.py yt_dlp_ejs/_version.py yt_dlp_ejs/yt/__init__.py yt_dlp_ejs/yt/solver/__init__.py EJS_JS_FOLDERS = yt_dlp_ejs/yt/solver diff --git a/README.md b/README.md index 4e7f442a60..8189015c72 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Example usage: yt-dlp --update-to nightly # To install nightly with pip: -python3 -m pip install -U --pre "yt-dlp[default]" +python -m pip install -U --pre "yt-dlp[default]" ``` When running a yt-dlp version that is older than 90 days, you will see a warning message suggesting to update to the latest version. @@ -265,12 +265,12 @@ To build the standalone executable, you must have Python and `pyinstaller` (plus You can run the following commands: ``` -python3 devscripts/install_deps.py --include pyinstaller -python3 devscripts/make_lazy_extractors.py -python3 -m bundle.pyinstaller +python devscripts/install_deps.py --include-group pyinstaller +python devscripts/make_lazy_extractors.py +python -m bundle.pyinstaller ``` -On some systems, you may need to use `py` or `python` instead of `python3`. +On some systems, you may need to use `py` or `python3` instead of `python`. `python -m bundle.pyinstaller` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate). @@ -360,7 +360,7 @@ Tip: Use `CTRL`+`F` (or `Command`+`F`) to search by keywords containing directory ("-" for stdin). Can be used multiple times and inside other configuration files - --plugin-dirs PATH Path to an additional directory to search + --plugin-dirs DIR Path to an additional directory to search for plugins. This option can be used multiple times to add multiple directories. Use "default" to search the default plugin @@ -369,22 +369,33 @@ Tip: Use `CTRL`+`F` (or `Command`+`F`) to search by keywords including defaults and those provided by previous --plugin-dirs --js-runtimes RUNTIME[:PATH] Additional JavaScript runtime to enable, - with an optional path to the runtime - location. This option can be used multiple - times to enable multiple runtimes. Supported - runtimes: deno, node, bun, quickjs. By - default, only "deno" runtime is enabled. + with an optional location for the runtime + (either the path to the binary or its + containing directory). This option can be + used multiple times to enable multiple + runtimes. Supported runtimes are (in order + of priority, from highest to lowest): deno, + node, quickjs, bun. Only "deno" is enabled + by default. The highest priority runtime + that is both enabled and available will be + used. In order to use a lower priority + runtime when "deno" is available, --no-js- + runtimes needs to be passed before enabling + other runtimes --no-js-runtimes Clear JavaScript runtimes to enable, including defaults and those provided by previous --js-runtimes --remote-components COMPONENT Remote components to allow yt-dlp to fetch - when required. You can use this option - multiple times to allow multiple components. - Supported values: ejs:npm (external - JavaScript components from npm), ejs:github - (external JavaScript components from yt-dlp- - ejs GitHub). By default, no remote - components are allowed. + when required. This option is currently not + needed if you are using an official + executable or have the requisite version of + the yt-dlp-ejs package installed. You can + use this option multiple times to allow + multiple components. Supported values: + ejs:npm (external JavaScript components from + npm), ejs:github (external JavaScript + components from yt-dlp-ejs GitHub). By + default, no remote components are allowed --no-remote-components Disallow fetching of all remote components, including any previously allowed by --remote-components or defaults. @@ -1105,11 +1116,12 @@ Make chapter entries for, or remove various segments (sponsor, for, separated by commas. Available categories are sponsor, intro, outro, selfpromo, preview, filler, interaction, - music_offtopic, poi_highlight, chapter, all - and default (=all). You can prefix the - category with a "-" to exclude it. See [1] - for descriptions of the categories. E.g. - --sponsorblock-mark all,-preview + music_offtopic, hook, poi_highlight, + chapter, all and default (=all). You can + prefix the category with a "-" to exclude + it. See [1] for descriptions of the + categories. E.g. --sponsorblock-mark + all,-preview [1] https://wiki.sponsor.ajay.app/w/Segment_Categories --sponsorblock-remove CATS SponsorBlock categories to be removed from the video file, separated by commas. If a @@ -1174,7 +1186,7 @@ Predefined aliases for convenience and ease of use. Note that future You can configure yt-dlp by placing any supported command line option in a configuration file. The configuration is loaded from the following locations: 1. **Main Configuration**: - * The file given to `--config-location` + * The file given to `--config-locations` 1. **Portable Configuration**: (Recommended for portable installations) * If using a binary, `yt-dlp.conf` in the same directory as the binary * If running from source-code, `yt-dlp.conf` in the parent directory of `yt_dlp` @@ -1256,7 +1268,7 @@ yt-dlp --netrc-cmd 'gpg --decrypt ~/.authinfo.gpg' 'https://www.youtube.com/watc ### Notes about environment variables * Environment variables are normally specified as `${VARIABLE}`/`$VARIABLE` on UNIX and `%VARIABLE%` on Windows; but is always shown as `${VARIABLE}` in this documentation -* yt-dlp also allows using UNIX-style variables on Windows for path-like options; e.g. `--output`, `--config-location` +* yt-dlp also allows using UNIX-style variables on Windows for path-like options; e.g. `--output`, `--config-locations` * If unset, `${XDG_CONFIG_HOME}` defaults to `~/.config` and `${XDG_CACHE_HOME}` to `~/.cache` * On Windows, `~` points to `${HOME}` if present; or, `${USERPROFILE}` or `${HOMEDRIVE}${HOMEPATH}` otherwise * On Windows, `${USERPROFILE}` generally points to `C:\Users\` and `${APPDATA}` to `${USERPROFILE}\AppData\Roaming` @@ -1840,7 +1852,7 @@ The following extractors use this feature: #### youtube * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube/_base.py](https://github.com/yt-dlp/yt-dlp/blob/415b4c9f955b1a0391204bd24a7132590e7b3bdb/yt_dlp/extractor/youtube/_base.py#L402-L409) for the list of supported content language codes * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively -* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_sdkless`, `android_vr`, `tv`, `tv_simply` and `tv_embedded`. By default, `tv,android_sdkless,web` is used. If no JavaScript runtime is available, then `android_sdkless,web_safari,web` is used. If logged-in cookies are passed to yt-dlp, then `tv,web_safari,web` is used for free accounts and `tv,web_creator,web` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios` +* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_sdkless`, `android_vr`, `tv`, `tv_simply`, `tv_downgraded`, and `tv_embedded`. By default, `tv,android_sdkless,web` is used. If no JavaScript runtime is available, then `android_sdkless,web_safari,web` is used. If logged-in cookies are passed to yt-dlp, then `tv_downgraded,web_safari,web` is used for free accounts and `tv_downgraded,web_creator,web` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios` * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details * `webpage_skip`: Skip extraction of embedded webpage data. One or both of `player_response`, `initial_data`. These options are for testing purposes and don't skip any network requests * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp. diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index e906838175..ba3f9518fb 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -308,5 +308,16 @@ "action": "add", "when": "2c9091e355a7ba5d1edb69796ecdca48199b77fb", "short": "[priority] **A stopgap release with a *TEMPORARY partial* fix for YouTube support**\nSome formats may still be unavailable, especially if cookies are passed to yt-dlp. The ***NEXT*** release, expected very soon, **will require an external JS runtime (e.g. Deno)** in order for YouTube downloads to work properly. [Read more](https://github.com/yt-dlp/yt-dlp/issues/14404)" + }, + { + "action": "change", + "when": "8636a9bac3bed99984c1e297453660468ecf504b", + "short": "Fix 6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc", + "authors": ["Grub4K"] + }, + { + "action": "add", + "when": "6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc", + "short": "[priority] **An external JavaScript runtime is now required for full YouTube support**\nyt-dlp now requires users to have an external JavaScript runtime (e.g. Deno) installed in order to solve the JavaScript challenges presented by YouTube. [Read more](https://github.com/yt-dlp/yt-dlp/issues/15012)" } ] diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index 0b2eb93b4e..88dbf74e4f 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -353,6 +353,13 @@ class CommitRange: continue commit = Commit(override_hash, override['short'], override.get('authors') or []) logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}') + if match := self.FIXES_RE.search(commit.short): + fix_commitish = match.group(1) + if fix_commitish in self._commits: + del self._commits[commit.hash] + self._fixes[fix_commitish].append(commit) + logger.info(f'Found fix for {fix_commitish[:HASH_LENGTH]}: {commit.hash[:HASH_LENGTH]}') + continue self._commits[commit.hash] = commit self._commits = dict(reversed(self._commits.items())) diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py index ebb3500b6c..3274abc39f 100755 --- a/devscripts/run_tests.py +++ b/devscripts/run_tests.py @@ -17,6 +17,18 @@ def parse_args(): parser = argparse.ArgumentParser(description='Run selected yt-dlp tests') parser.add_argument( 'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*') + parser.add_argument( + '--flaky', + action='store_true', + default=None, + help='Allow running flaky tests. (default: run, unless in CI)', + ) + parser.add_argument( + '--no-flaky', + action='store_false', + dest='flaky', + help=argparse.SUPPRESS, + ) parser.add_argument( '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION') parser.add_argument( @@ -24,10 +36,11 @@ def parse_args(): return parser.parse_args() -def run_tests(*tests, pattern=None, ci=False): +def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None): # XXX: hatch uses `tests` if no arguments are passed run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests) run_download = 'download' in tests + run_flaky = flaky or (flaky is None and not ci) pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '') arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)] @@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False): test if '/' in test else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}' for test in tests) + if not run_flaky: + arguments.append('--disallow-flaky') print(f'Running {arguments}', flush=True) try: @@ -72,6 +87,11 @@ if __name__ == '__main__': args = parse_args() os.chdir(Path(__file__).parent.parent) - sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI')))) + sys.exit(run_tests( + *args.test, + pattern=args.k, + ci=bool(os.getenv('CI')), + flaky=args.flaky, + )) except KeyboardInterrupt: pass diff --git a/devscripts/update_ejs.py b/devscripts/update_ejs.py old mode 100644 new mode 100755 index cffb1aa2b4..3aa76bd0ce --- a/devscripts/update_ejs.py +++ b/devscripts/update_ejs.py @@ -66,7 +66,9 @@ def list_wheel_contents( ) -> str: assert folders or files, 'at least one of "folders" or "files" must be True' - path_gen = (zinfo.filename for zinfo in zipfile.ZipFile(io.BytesIO(wheel_data)).infolist()) + with zipfile.ZipFile(io.BytesIO(wheel_data)) as zipf: + path_gen = (zinfo.filename for zinfo in zipf.infolist()) + filtered = filter(lambda path: path.startswith('yt_dlp_ejs/'), path_gen) if suffix: filtered = filter(lambda path: path.endswith(f'.{suffix}'), filtered) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 8e190c00cb..046e9231f1 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -18,6 +18,7 @@ def build_completion(opt_parser): for opt in group.option_list] opts_file = [opt for opt in opts if opt.metavar == 'FILE'] opts_dir = [opt for opt in opts if opt.metavar == 'DIR'] + opts_path = [opt for opt in opts if opt.metavar == 'PATH'] fileopts = [] for opt in opts_file: @@ -26,6 +27,12 @@ def build_completion(opt_parser): if opt._long_opts: fileopts.extend(opt._long_opts) + for opt in opts_path: + if opt._short_opts: + fileopts.extend(opt._short_opts) + if opt._long_opts: + fileopts.extend(opt._long_opts) + diropts = [] for opt in opts_dir: if opt._short_opts: diff --git a/pyproject.toml b/pyproject.toml index 0f6202ca08..d2c5745b95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ default = [ "requests>=2.32.2,<3", "urllib3>=2.0.2,<3", "websockets>=13.0", - "yt-dlp-ejs==0.3.0", + "yt-dlp-ejs==0.3.1", ] curl-cffi = [ "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.14; implementation_name=='cpython'", diff --git a/supportedsites.md b/supportedsites.md index a546819286..9ab6d26335 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -12,6 +12,7 @@ The only reliable way to check if a site is supported is to try it. - **17live:vod** - **1News**: 1news.co.nz article videos - **1tv**: Первый канал + - **1tv:live**: Первый канал (прямой эфир) - **20min**: (**Currently broken**) - **23video** - **247sports**: (**Currently broken**) @@ -93,6 +94,8 @@ The only reliable way to check if a site is supported is to try it. - **archive.org**: archive.org video and audio - **ArcPublishing** - **ARD** + - **ARDAudiothek** + - **ARDAudiothekPlaylist** - **ARDMediathek** - **ARDMediathekCollection** - **Art19** @@ -533,7 +536,6 @@ The only reliable way to check if a site is supported is to try it. - **google:​podcasts:feed** - **GoogleDrive** - **GoogleDrive:Folder** - - **GoPlay**: [*goplay*](## "netrc machine") - **GoPro** - **Goshgay** - **GoToStage** @@ -844,6 +846,7 @@ The only reliable way to check if a site is supported is to try it. - **MusicdexArtist** - **MusicdexPlaylist** - **MusicdexSong** + - **Mux** - **Mx3** - **Mx3Neo** - **Mx3Volksmusik** @@ -858,6 +861,7 @@ The only reliable way to check if a site is supported is to try it. - **n-tv.de** - **N1Info:article** - **N1InfoAsset** + - **NascarClassics** - **Nate** - **NateProgram** - **natgeo:video** @@ -1071,6 +1075,7 @@ The only reliable way to check if a site is supported is to try it. - **PlanetMarathi** - **Platzi**: [*platzi*](## "netrc machine") - **PlatziCourse**: [*platzi*](## "netrc machine") + - **play.tv**: [*goplay*](## "netrc machine") PLAY (formerly goplay.be) - **player.sky.it** - **PlayerFm** - **playeur** @@ -1559,12 +1564,12 @@ The only reliable way to check if a site is supported is to try it. - **TwitCastingLive** - **TwitCastingUser** - **twitch:clips**: [*twitch*](## "netrc machine") + - **twitch:collection**: [*twitch*](## "netrc machine") - **twitch:stream**: [*twitch*](## "netrc machine") + - **twitch:videos**: [*twitch*](## "netrc machine") + - **twitch:​videos:clips**: [*twitch*](## "netrc machine") + - **twitch:​videos:collections**: [*twitch*](## "netrc machine") - **twitch:vod**: [*twitch*](## "netrc machine") - - **TwitchCollection**: [*twitch*](## "netrc machine") - - **TwitchVideos**: [*twitch*](## "netrc machine") - - **TwitchVideosClips**: [*twitch*](## "netrc machine") - - **TwitchVideosCollections**: [*twitch*](## "netrc machine") - **twitter**: [*twitter*](## "netrc machine") - **twitter:amplify**: [*twitter*](## "netrc machine") - **twitter:broadcast**: [*twitter*](## "netrc machine") diff --git a/test/conftest.py b/test/conftest.py index a8b92f811e..9d31986196 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -52,6 +52,33 @@ def skip_handlers_if(request, handler): pytest.skip(marker.args[1] if len(marker.args) > 1 else '') +@pytest.fixture(autouse=True) +def handler_flaky(request, handler): + """Mark a certain handler as being flaky. + + This will skip the test if pytest does not get run using `--allow-flaky` + + usage: + pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason') + """ + for marker in request.node.iter_markers(handler_flaky.__name__): + if ( + marker.args[0] == handler.RH_KEY + and (not marker.args[1:] or any(marker.args[1:])) + and request.config.getoption('disallow_flaky') + ): + reason = marker.kwargs.get('reason') + pytest.skip(f'flaky: {reason}' if reason else 'flaky') + + +def pytest_addoption(parser, pluginmanager): + parser.addoption( + '--disallow-flaky', + action='store_true', + help='disallow flaky tests from running.', + ) + + def pytest_configure(config): config.addinivalue_line( 'markers', 'skip_handler(handler): skip test for the given handler', @@ -62,3 +89,6 @@ def pytest_configure(config): config.addinivalue_line( 'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true', ) + config.addinivalue_line( + 'markers', 'handler_flaky(handler): mark handler as flaky if condition is true', + ) diff --git a/test/test_http_proxy.py b/test/test_http_proxy.py index e903ff8beb..22ce3ca5d7 100644 --- a/test/test_http_proxy.py +++ b/test/test_http_proxy.py @@ -247,6 +247,7 @@ def ctx(request): @pytest.mark.parametrize( 'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') @pytest.mark.parametrize('ctx', ['http'], indirect=True) # pure http proxy can only support http class TestHTTPProxy: def test_http_no_auth(self, handler, ctx): @@ -315,6 +316,7 @@ class TestHTTPProxy: ('Requests', 'https'), ('CurlCFFI', 'https'), ], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults') class TestHTTPConnectProxy: def test_http_connect_no_auth(self, handler, ctx): with ctx.http_server(HTTPConnectProxyHandler) as server_address: diff --git a/test/test_networking.py b/test/test_networking.py index afdd0c7aa7..f78a8d5770 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -3,6 +3,7 @@ # Allow direct execution import os import sys +from unittest.mock import MagicMock import pytest @@ -311,6 +312,7 @@ class TestRequestHandlerBase: @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) +@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults') class TestHTTPRequestHandler(TestRequestHandlerBase): def test_verify_cert(self, handler): @@ -614,8 +616,11 @@ class TestHTTPRequestHandler(TestRequestHandlerBase): @pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi') def test_gzip_trailing_garbage(self, handler): with handler() as rh: - data = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode() + res = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')) + data = res.read().decode() assert data == '