mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2025-12-06 14:55:02 +01:00
Merge https://github.com/yt-dlp/yt-dlp into fix/ie/plutotv
This commit is contained in:
commit
620240264b
63 changed files with 1805 additions and 559 deletions
6
.github/actionlint.yml
vendored
6
.github/actionlint.yml
vendored
|
|
@ -1,9 +1,3 @@
|
||||||
self-hosted-runner:
|
|
||||||
labels:
|
|
||||||
# Workaround for the outdated runner list in actionlint v1.7.7
|
|
||||||
# Ref: https://github.com/rhysd/actionlint/issues/533
|
|
||||||
- windows-11-arm
|
|
||||||
|
|
||||||
config-variables:
|
config-variables:
|
||||||
- KEEP_CACHE_WARM
|
- KEEP_CACHE_WARM
|
||||||
- PUSH_VERSION_COMMIT
|
- PUSH_VERSION_COMMIT
|
||||||
|
|
|
||||||
2
.github/workflows/build.yml
vendored
2
.github/workflows/build.yml
vendored
|
|
@ -153,10 +153,12 @@ jobs:
|
||||||
'os': 'musllinux',
|
'os': 'musllinux',
|
||||||
'arch': 'x86_64',
|
'arch': 'x86_64',
|
||||||
'runner': 'ubuntu-24.04',
|
'runner': 'ubuntu-24.04',
|
||||||
|
'python_version': '3.14',
|
||||||
}, {
|
}, {
|
||||||
'os': 'musllinux',
|
'os': 'musllinux',
|
||||||
'arch': 'aarch64',
|
'arch': 'aarch64',
|
||||||
'runner': 'ubuntu-24.04-arm',
|
'runner': 'ubuntu-24.04-arm',
|
||||||
|
'python_version': '3.14',
|
||||||
}],
|
}],
|
||||||
}
|
}
|
||||||
INPUTS = json.loads(os.environ['INPUTS'])
|
INPUTS = json.loads(os.environ['INPUTS'])
|
||||||
|
|
|
||||||
23
.github/workflows/core.yml
vendored
23
.github/workflows/core.yml
vendored
|
|
@ -56,6 +56,8 @@ jobs:
|
||||||
python-version: pypy-3.11
|
python-version: pypy-3.11
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
|
|
@ -65,6 +67,25 @@ jobs:
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
timeout-minutes: 15
|
timeout-minutes: 15
|
||||||
continue-on-error: False
|
continue-on-error: False
|
||||||
|
env:
|
||||||
|
source: ${{ (github.event_name == 'push' && github.event.before) || 'origin/master' }}
|
||||||
|
target: ${{ (github.event_name == 'push' && github.event.after) || 'HEAD' }}
|
||||||
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
|
flags=()
|
||||||
|
# Check if a networking file is involved
|
||||||
|
patterns="\
|
||||||
|
^yt_dlp/networking/
|
||||||
|
^yt_dlp/utils/networking\.py$
|
||||||
|
^test/test_http_proxy\.py$
|
||||||
|
^test/test_networking\.py$
|
||||||
|
^test/test_networking_utils\.py$
|
||||||
|
^test/test_socks\.py$
|
||||||
|
^test/test_websockets\.py$
|
||||||
|
^pyproject\.toml$
|
||||||
|
"
|
||||||
|
if git diff --name-only "${source}" "${target}" | grep -Ef <(printf '%s' "${patterns}"); then
|
||||||
|
flags+=(--flaky)
|
||||||
|
fi
|
||||||
python3 -m yt_dlp -v || true # Print debug head
|
python3 -m yt_dlp -v || true # Print debug head
|
||||||
python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core
|
python3 -m devscripts.run_tests "${flags[@]}" --pytest-args '--reruns 2 --reruns-delay 3.0' core
|
||||||
|
|
|
||||||
4
.github/workflows/test-workflows.yml
vendored
4
.github/workflows/test-workflows.yml
vendored
|
|
@ -17,8 +17,8 @@ on:
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
env:
|
env:
|
||||||
ACTIONLINT_VERSION: "1.7.7"
|
ACTIONLINT_VERSION: "1.7.8"
|
||||||
ACTIONLINT_SHA256SUM: 023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757
|
ACTIONLINT_SHA256SUM: be92c2652ab7b6d08425428797ceabeb16e31a781c07bc388456b4e592f3e36a
|
||||||
ACTIONLINT_REPO: https://github.com/rhysd/actionlint
|
ACTIONLINT_REPO: https://github.com/rhysd/actionlint
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
|
||||||
|
|
@ -177,7 +177,7 @@ While it is strongly recommended to use `hatch` for yt-dlp development, if you a
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
# To only install development dependencies:
|
# To only install development dependencies:
|
||||||
$ python -m devscripts.install_deps --include dev
|
$ python -m devscripts.install_deps --include-group dev
|
||||||
|
|
||||||
# Or, for an editable install plus dev dependencies:
|
# Or, for an editable install plus dev dependencies:
|
||||||
$ python -m pip install -e ".[default,dev]"
|
$ python -m pip install -e ".[default,dev]"
|
||||||
|
|
|
||||||
16
CONTRIBUTORS
16
CONTRIBUTORS
|
|
@ -818,3 +818,19 @@ robin-mu
|
||||||
shssoichiro
|
shssoichiro
|
||||||
thanhtaivtt
|
thanhtaivtt
|
||||||
uoag
|
uoag
|
||||||
|
CaramelConnoisseur
|
||||||
|
ctengel
|
||||||
|
einstein95
|
||||||
|
evilpie
|
||||||
|
i3p9
|
||||||
|
JrM2628
|
||||||
|
krystophny
|
||||||
|
matyb08
|
||||||
|
pha1n0q
|
||||||
|
PierceLBrooks
|
||||||
|
sepro
|
||||||
|
TheQWERTYCodr
|
||||||
|
thomasmllt
|
||||||
|
w4grfw
|
||||||
|
WeidiDeng
|
||||||
|
Zer0spectrum
|
||||||
|
|
|
||||||
65
Changelog.md
65
Changelog.md
|
|
@ -4,6 +4,71 @@
|
||||||
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
|
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
|
||||||
-->
|
-->
|
||||||
|
|
||||||
|
### 2025.11.12
|
||||||
|
|
||||||
|
#### Important changes
|
||||||
|
- **An external JavaScript runtime is now required for full YouTube support**
|
||||||
|
yt-dlp now requires users to have an external JavaScript runtime (e.g. Deno) installed in order to solve the JavaScript challenges presented by YouTube. [Read more](https://github.com/yt-dlp/yt-dlp/issues/15012)
|
||||||
|
|
||||||
|
#### Core changes
|
||||||
|
- **cookies**
|
||||||
|
- [Allow `--cookies-from-browser` for Safari on iOS](https://github.com/yt-dlp/yt-dlp/commit/e6414d64e73d86d65bb357e5ad59d0ca080d5812) ([#14950](https://github.com/yt-dlp/yt-dlp/issues/14950)) by [pha1n0q](https://github.com/pha1n0q)
|
||||||
|
- [Support Firefox cookies database v17](https://github.com/yt-dlp/yt-dlp/commit/bf7e04e9d8bd3c4a4614b67ce617b7ae5d17d62a) ([#15010](https://github.com/yt-dlp/yt-dlp/issues/15010)) by [Grub4K](https://github.com/Grub4K)
|
||||||
|
- **sponsorblock**: [Add `hook` category](https://github.com/yt-dlp/yt-dlp/commit/52f3c56e83bbb25eec2496b0499768753732a093) ([#14845](https://github.com/yt-dlp/yt-dlp/issues/14845)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- **update**: [Fix PyInstaller onedir variant detection](https://github.com/yt-dlp/yt-dlp/commit/1c2ad94353d1c9e03615d20b6bbfc293286c7a32) ([#14800](https://github.com/yt-dlp/yt-dlp/issues/14800)) by [bashonly](https://github.com/bashonly)
|
||||||
|
|
||||||
|
#### Extractor changes
|
||||||
|
- **1tv**: live: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/19c5d7c53013440ec4f3f56ebbb067531b272f3f) ([#14299](https://github.com/yt-dlp/yt-dlp/issues/14299)) by [swayll](https://github.com/swayll)
|
||||||
|
- **ardaudiothek**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/0046fbcbfceee32fa2f68a8ea00cca02765470b6) ([#14309](https://github.com/yt-dlp/yt-dlp/issues/14309)) by [evilpie](https://github.com/evilpie), [marieell](https://github.com/marieell)
|
||||||
|
- **bunnycdn**
|
||||||
|
- [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/228ae9f0f2b441fa1296db2ed2b7afbd4a9a62a1) ([#14954](https://github.com/yt-dlp/yt-dlp/issues/14954)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- [Support player subdomain URLs](https://github.com/yt-dlp/yt-dlp/commit/3ef867451cd9604b4195dfee00db768619629b2d) ([#14979](https://github.com/yt-dlp/yt-dlp/issues/14979)) by [einstein95](https://github.com/einstein95)
|
||||||
|
- **discoverynetworksde**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/10dea209d2460daf924c93835ddc2f0301cf2cd4) ([#14818](https://github.com/yt-dlp/yt-dlp/issues/14818)) by [dirkf](https://github.com/dirkf), [w4grfw](https://github.com/w4grfw) (With fixes in [f3c255b](https://github.com/yt-dlp/yt-dlp/commit/f3c255b63bd26069151fc3d3ba6dc626bb62ad6e) by [bashonly](https://github.com/bashonly))
|
||||||
|
- **floatplane**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/1ac7e6005cd3be9fff0b28be189c3a68ecd4c593) ([#14984](https://github.com/yt-dlp/yt-dlp/issues/14984)) by [i3p9](https://github.com/i3p9)
|
||||||
|
- **googledrive**
|
||||||
|
- [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/6d05cee4df30774ddce5c5c751fd2118f40c24fe) ([#14809](https://github.com/yt-dlp/yt-dlp/issues/14809)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/70f1098312fe53bc85358f7bd624370878b2fa28) ([#14746](https://github.com/yt-dlp/yt-dlp/issues/14746)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- **kika**: [Do not extract non-existent subtitles](https://github.com/yt-dlp/yt-dlp/commit/79f9232ffbd57dde91c372b673b42801edaa9e53) ([#14813](https://github.com/yt-dlp/yt-dlp/issues/14813)) by [InvalidUsernameException](https://github.com/InvalidUsernameException)
|
||||||
|
- **mux**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a0bda3b78609593ce1127215fc035c1a308a89b6) ([#14914](https://github.com/yt-dlp/yt-dlp/issues/14914)) by [PierceLBrooks](https://github.com/PierceLBrooks), [seproDev](https://github.com/seproDev)
|
||||||
|
- **nascarclassics**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/e8a6b1ca92f2a0ce2c187668165be23dc5506aab) ([#14866](https://github.com/yt-dlp/yt-dlp/issues/14866)) by [JrM2628](https://github.com/JrM2628)
|
||||||
|
- **nbc**: [Detect and discard DRM formats](https://github.com/yt-dlp/yt-dlp/commit/ee3a106f34124f0e2d28f062f5302863fd7639be) ([#14844](https://github.com/yt-dlp/yt-dlp/issues/14844)) by [bashonly](https://github.com/bashonly)
|
||||||
|
- **ntv.ru**: [Rework extractor](https://github.com/yt-dlp/yt-dlp/commit/5dde0d0c9fcef2ce57e486b2e563e0dff9b2845a) ([#14934](https://github.com/yt-dlp/yt-dlp/issues/14934)) by [anlar](https://github.com/anlar), [seproDev](https://github.com/seproDev) (With fixes in [a86eeaa](https://github.com/yt-dlp/yt-dlp/commit/a86eeaadf236ceaf6bb232eb410cf21572538aa6) by [seproDev](https://github.com/seproDev))
|
||||||
|
- **play.tv**: [Update extractor for new domain](https://github.com/yt-dlp/yt-dlp/commit/73fd850d170e01c47c31aaa6aa8fe90856d9ad18) ([#14905](https://github.com/yt-dlp/yt-dlp/issues/14905)) by [thomasmllt](https://github.com/thomasmllt)
|
||||||
|
- **tubetugraz**: [Support alternate URL format](https://github.com/yt-dlp/yt-dlp/commit/f3597cfafcab4d7d4c6d41bff3647681301f1e6b) ([#14718](https://github.com/yt-dlp/yt-dlp/issues/14718)) by [krystophny](https://github.com/krystophny)
|
||||||
|
- **twitch**
|
||||||
|
- [Fix playlist extraction](https://github.com/yt-dlp/yt-dlp/commit/cb78440e468608fd55546280b537387d375335f2) ([#15008](https://github.com/yt-dlp/yt-dlp/issues/15008)) by [bashonly](https://github.com/bashonly), [ctengel](https://github.com/ctengel)
|
||||||
|
- stream: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/7eff676183518175ce495ae63291c89f9b39f02a) ([#14988](https://github.com/yt-dlp/yt-dlp/issues/14988)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- vod: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/b46c572b26be15683584102c5fb7e7bfde0c9821) ([#14999](https://github.com/yt-dlp/yt-dlp/issues/14999)) by [Zer0spectrum](https://github.com/Zer0spectrum)
|
||||||
|
- **urplay**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/808b1fed76fbd07840cc23a346c11334e3d34f43) ([#14785](https://github.com/yt-dlp/yt-dlp/issues/14785)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- **web.archive**: youtube: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/d9e3011fd1c3a75871a50e78533afe78ad427ce3) ([#14753](https://github.com/yt-dlp/yt-dlp/issues/14753)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- **xhamster**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/a1d6351c3fc82c07fa0ee70811ed84807f6bbb58) ([#14948](https://github.com/yt-dlp/yt-dlp/issues/14948)) by [CaramelConnoisseur](https://github.com/CaramelConnoisseur), [dhwz](https://github.com/dhwz)
|
||||||
|
- **youtube**
|
||||||
|
- [Add `tv_downgraded` client](https://github.com/yt-dlp/yt-dlp/commit/61cf34f5447177a73ba25ea9a47d7df516ca3b3b) ([#14887](https://github.com/yt-dlp/yt-dlp/issues/14887)) by [seproDev](https://github.com/seproDev) (With fixes in [fa35eb2](https://github.com/yt-dlp/yt-dlp/commit/fa35eb27eaf27df7b5854f527a89fc828c9e0ec0))
|
||||||
|
- [Fix `web_embedded` client extraction](https://github.com/yt-dlp/yt-dlp/commit/d6ee67725397807bbb5edcd0b2c94f5bca62d3f4) ([#14843](https://github.com/yt-dlp/yt-dlp/issues/14843)) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev)
|
||||||
|
- [Fix auto-generated metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/a56217f9f6c594f6c419ce8dce9134198a9d90d0) ([#13896](https://github.com/yt-dlp/yt-dlp/issues/13896)) by [TheQWERTYCodr](https://github.com/TheQWERTYCodr)
|
||||||
|
- [Fix original language detection](https://github.com/yt-dlp/yt-dlp/commit/afc44022d0b736b2b3e87b52490bd35c53c53632) ([#14919](https://github.com/yt-dlp/yt-dlp/issues/14919)) by [bashonly](https://github.com/bashonly)
|
||||||
|
- [Implement external n/sig solver](https://github.com/yt-dlp/yt-dlp/commit/6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc) ([#14157](https://github.com/yt-dlp/yt-dlp/issues/14157)) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev) (With fixes in [4b4223b](https://github.com/yt-dlp/yt-dlp/commit/4b4223b436fb03a12628679daed32ae4fc15ae4b), [ee98be4](https://github.com/yt-dlp/yt-dlp/commit/ee98be4ad767b77e4d8dd9bfd3c7d10f2e8397ff), [c0c9f30](https://github.com/yt-dlp/yt-dlp/commit/c0c9f30695db314df084e8701a7c376eb54f283c), [cacd163](https://github.com/yt-dlp/yt-dlp/commit/cacd1630a1a59e92f857d0d175c8730cffbf9801), [8636a9b](https://github.com/yt-dlp/yt-dlp/commit/8636a9bac3bed99984c1e297453660468ecf504b))
|
||||||
|
- [Support collaborators](https://github.com/yt-dlp/yt-dlp/commit/f87cfadb5c3cba8e9dc4231c9554548e9edb3882) ([#14677](https://github.com/yt-dlp/yt-dlp/issues/14677)) by [seproDev](https://github.com/seproDev)
|
||||||
|
- tab: [Fix duration extraction for feeds](https://github.com/yt-dlp/yt-dlp/commit/1d2f0edaf978a5541cfb8f7e83fec433c65c1011) ([#14668](https://github.com/yt-dlp/yt-dlp/issues/14668)) by [WeidiDeng](https://github.com/WeidiDeng)
|
||||||
|
|
||||||
|
#### Downloader changes
|
||||||
|
- **ffmpeg**
|
||||||
|
- [Apply `ffmpeg_args` for each format](https://github.com/yt-dlp/yt-dlp/commit/ffb7b7f446b6c67a28c66598ae91f4f2263e0d75) ([#14886](https://github.com/yt-dlp/yt-dlp/issues/14886)) by [bashonly](https://github.com/bashonly)
|
||||||
|
- [Limit read rate for DASH livestreams](https://github.com/yt-dlp/yt-dlp/commit/7af6d81f35aea8832023daa30ada10e6673a0529) ([#14918](https://github.com/yt-dlp/yt-dlp/issues/14918)) by [bashonly](https://github.com/bashonly)
|
||||||
|
|
||||||
|
#### Networking changes
|
||||||
|
- [Ensure underlying file object is closed when fully read](https://github.com/yt-dlp/yt-dlp/commit/5767fb4ab108dddb07fc839a3b0f4d323a7c4bea) ([#14935](https://github.com/yt-dlp/yt-dlp/issues/14935)) by [coletdjnz](https://github.com/coletdjnz)
|
||||||
|
|
||||||
|
#### Misc. changes
|
||||||
|
- [Fix zsh path argument completion](https://github.com/yt-dlp/yt-dlp/commit/c96e9291ab7bd6e7da66d33424982c8b0b4431c7) ([#14953](https://github.com/yt-dlp/yt-dlp/issues/14953)) by [matyb08](https://github.com/matyb08)
|
||||||
|
- **build**: [Bump musllinux Python version to 3.14](https://github.com/yt-dlp/yt-dlp/commit/646904cd3a79429ec5fdc43f904b3f57ae213f34) ([#14623](https://github.com/yt-dlp/yt-dlp/issues/14623)) by [bashonly](https://github.com/bashonly)
|
||||||
|
- **cleanup**
|
||||||
|
- Miscellaneous
|
||||||
|
- [c63b4e2](https://github.com/yt-dlp/yt-dlp/commit/c63b4e2a2b81cc78397c8709ef53ffd29bada213) by [bashonly](https://github.com/bashonly), [matyb08](https://github.com/matyb08), [sepro](https://github.com/sepro)
|
||||||
|
- [335653b](https://github.com/yt-dlp/yt-dlp/commit/335653be82d5ef999cfc2879d005397402eebec1) by [bashonly](https://github.com/bashonly), [seproDev](https://github.com/seproDev)
|
||||||
|
- **devscripts**: [Improve `install_deps` script](https://github.com/yt-dlp/yt-dlp/commit/73922e66e437fb4bb618bdc119a96375081bf508) ([#14766](https://github.com/yt-dlp/yt-dlp/issues/14766)) by [bashonly](https://github.com/bashonly)
|
||||||
|
- **test**: [Skip flaky tests if source unchanged](https://github.com/yt-dlp/yt-dlp/commit/ade8c2b36ff300edef87d48fd1ba835ac35c5b63) ([#14970](https://github.com/yt-dlp/yt-dlp/issues/14970)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
|
||||||
|
|
||||||
### 2025.10.22
|
### 2025.10.22
|
||||||
|
|
||||||
#### Important changes
|
#### Important changes
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@ Core Maintainers are responsible for reviewing and merging contributions, publis
|
||||||
|
|
||||||
**You can contact the core maintainers via `maintainers@yt-dlp.org`.**
|
**You can contact the core maintainers via `maintainers@yt-dlp.org`.**
|
||||||
|
|
||||||
|
This is **NOT** a support channel. [Open an issue](https://github.com/yt-dlp/yt-dlp/issues/new/choose) if you need help or want to report a bug.
|
||||||
|
|
||||||
### [coletdjnz](https://github.com/coletdjnz)
|
### [coletdjnz](https://github.com/coletdjnz)
|
||||||
|
|
||||||
[](https://github.com/sponsors/coletdjnz)
|
[](https://github.com/sponsors/coletdjnz)
|
||||||
|
|
|
||||||
6
Makefile
6
Makefile
|
|
@ -202,9 +202,9 @@ CONTRIBUTORS: Changelog.md
|
||||||
|
|
||||||
# The following EJS_-prefixed variables are auto-generated by devscripts/update_ejs.py
|
# The following EJS_-prefixed variables are auto-generated by devscripts/update_ejs.py
|
||||||
# DO NOT EDIT!
|
# DO NOT EDIT!
|
||||||
EJS_VERSION = 0.3.0
|
EJS_VERSION = 0.3.1
|
||||||
EJS_WHEEL_NAME = yt_dlp_ejs-0.3.0-py3-none-any.whl
|
EJS_WHEEL_NAME = yt_dlp_ejs-0.3.1-py3-none-any.whl
|
||||||
EJS_WHEEL_HASH = sha256:abbf269fa1674cab7b7b266e51e89e0e60b01a11a0fdf3cd63528683190cdd07
|
EJS_WHEEL_HASH = sha256:a6e3548874db7c774388931752bb46c7f4642c044b2a189e56968f3d5ecab622
|
||||||
EJS_PY_FOLDERS = yt_dlp_ejs yt_dlp_ejs/yt yt_dlp_ejs/yt/solver
|
EJS_PY_FOLDERS = yt_dlp_ejs yt_dlp_ejs/yt yt_dlp_ejs/yt/solver
|
||||||
EJS_PY_FILES = yt_dlp_ejs/__init__.py yt_dlp_ejs/_version.py yt_dlp_ejs/yt/__init__.py yt_dlp_ejs/yt/solver/__init__.py
|
EJS_PY_FILES = yt_dlp_ejs/__init__.py yt_dlp_ejs/_version.py yt_dlp_ejs/yt/__init__.py yt_dlp_ejs/yt/solver/__init__.py
|
||||||
EJS_JS_FOLDERS = yt_dlp_ejs/yt/solver
|
EJS_JS_FOLDERS = yt_dlp_ejs/yt/solver
|
||||||
|
|
|
||||||
64
README.md
64
README.md
|
|
@ -189,7 +189,7 @@ Example usage:
|
||||||
yt-dlp --update-to nightly
|
yt-dlp --update-to nightly
|
||||||
|
|
||||||
# To install nightly with pip:
|
# To install nightly with pip:
|
||||||
python3 -m pip install -U --pre "yt-dlp[default]"
|
python -m pip install -U --pre "yt-dlp[default]"
|
||||||
```
|
```
|
||||||
|
|
||||||
When running a yt-dlp version that is older than 90 days, you will see a warning message suggesting to update to the latest version.
|
When running a yt-dlp version that is older than 90 days, you will see a warning message suggesting to update to the latest version.
|
||||||
|
|
@ -265,12 +265,12 @@ To build the standalone executable, you must have Python and `pyinstaller` (plus
|
||||||
You can run the following commands:
|
You can run the following commands:
|
||||||
|
|
||||||
```
|
```
|
||||||
python3 devscripts/install_deps.py --include pyinstaller
|
python devscripts/install_deps.py --include-group pyinstaller
|
||||||
python3 devscripts/make_lazy_extractors.py
|
python devscripts/make_lazy_extractors.py
|
||||||
python3 -m bundle.pyinstaller
|
python -m bundle.pyinstaller
|
||||||
```
|
```
|
||||||
|
|
||||||
On some systems, you may need to use `py` or `python` instead of `python3`.
|
On some systems, you may need to use `py` or `python3` instead of `python`.
|
||||||
|
|
||||||
`python -m bundle.pyinstaller` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
|
`python -m bundle.pyinstaller` accepts any arguments that can be passed to `pyinstaller`, such as `--onefile/-F` or `--onedir/-D`, which is further [documented here](https://pyinstaller.org/en/stable/usage.html#what-to-generate).
|
||||||
|
|
||||||
|
|
@ -360,7 +360,7 @@ Tip: Use `CTRL`+`F` (or `Command`+`F`) to search by keywords
|
||||||
containing directory ("-" for stdin). Can be
|
containing directory ("-" for stdin). Can be
|
||||||
used multiple times and inside other
|
used multiple times and inside other
|
||||||
configuration files
|
configuration files
|
||||||
--plugin-dirs PATH Path to an additional directory to search
|
--plugin-dirs DIR Path to an additional directory to search
|
||||||
for plugins. This option can be used
|
for plugins. This option can be used
|
||||||
multiple times to add multiple directories.
|
multiple times to add multiple directories.
|
||||||
Use "default" to search the default plugin
|
Use "default" to search the default plugin
|
||||||
|
|
@ -369,22 +369,33 @@ Tip: Use `CTRL`+`F` (or `Command`+`F`) to search by keywords
|
||||||
including defaults and those provided by
|
including defaults and those provided by
|
||||||
previous --plugin-dirs
|
previous --plugin-dirs
|
||||||
--js-runtimes RUNTIME[:PATH] Additional JavaScript runtime to enable,
|
--js-runtimes RUNTIME[:PATH] Additional JavaScript runtime to enable,
|
||||||
with an optional path to the runtime
|
with an optional location for the runtime
|
||||||
location. This option can be used multiple
|
(either the path to the binary or its
|
||||||
times to enable multiple runtimes. Supported
|
containing directory). This option can be
|
||||||
runtimes: deno, node, bun, quickjs. By
|
used multiple times to enable multiple
|
||||||
default, only "deno" runtime is enabled.
|
runtimes. Supported runtimes are (in order
|
||||||
|
of priority, from highest to lowest): deno,
|
||||||
|
node, quickjs, bun. Only "deno" is enabled
|
||||||
|
by default. The highest priority runtime
|
||||||
|
that is both enabled and available will be
|
||||||
|
used. In order to use a lower priority
|
||||||
|
runtime when "deno" is available, --no-js-
|
||||||
|
runtimes needs to be passed before enabling
|
||||||
|
other runtimes
|
||||||
--no-js-runtimes Clear JavaScript runtimes to enable,
|
--no-js-runtimes Clear JavaScript runtimes to enable,
|
||||||
including defaults and those provided by
|
including defaults and those provided by
|
||||||
previous --js-runtimes
|
previous --js-runtimes
|
||||||
--remote-components COMPONENT Remote components to allow yt-dlp to fetch
|
--remote-components COMPONENT Remote components to allow yt-dlp to fetch
|
||||||
when required. You can use this option
|
when required. This option is currently not
|
||||||
multiple times to allow multiple components.
|
needed if you are using an official
|
||||||
Supported values: ejs:npm (external
|
executable or have the requisite version of
|
||||||
JavaScript components from npm), ejs:github
|
the yt-dlp-ejs package installed. You can
|
||||||
(external JavaScript components from yt-dlp-
|
use this option multiple times to allow
|
||||||
ejs GitHub). By default, no remote
|
multiple components. Supported values:
|
||||||
components are allowed.
|
ejs:npm (external JavaScript components from
|
||||||
|
npm), ejs:github (external JavaScript
|
||||||
|
components from yt-dlp-ejs GitHub). By
|
||||||
|
default, no remote components are allowed
|
||||||
--no-remote-components Disallow fetching of all remote components,
|
--no-remote-components Disallow fetching of all remote components,
|
||||||
including any previously allowed by
|
including any previously allowed by
|
||||||
--remote-components or defaults.
|
--remote-components or defaults.
|
||||||
|
|
@ -1105,11 +1116,12 @@ Make chapter entries for, or remove various segments (sponsor,
|
||||||
for, separated by commas. Available
|
for, separated by commas. Available
|
||||||
categories are sponsor, intro, outro,
|
categories are sponsor, intro, outro,
|
||||||
selfpromo, preview, filler, interaction,
|
selfpromo, preview, filler, interaction,
|
||||||
music_offtopic, poi_highlight, chapter, all
|
music_offtopic, hook, poi_highlight,
|
||||||
and default (=all). You can prefix the
|
chapter, all and default (=all). You can
|
||||||
category with a "-" to exclude it. See [1]
|
prefix the category with a "-" to exclude
|
||||||
for descriptions of the categories. E.g.
|
it. See [1] for descriptions of the
|
||||||
--sponsorblock-mark all,-preview
|
categories. E.g. --sponsorblock-mark
|
||||||
|
all,-preview
|
||||||
[1] https://wiki.sponsor.ajay.app/w/Segment_Categories
|
[1] https://wiki.sponsor.ajay.app/w/Segment_Categories
|
||||||
--sponsorblock-remove CATS SponsorBlock categories to be removed from
|
--sponsorblock-remove CATS SponsorBlock categories to be removed from
|
||||||
the video file, separated by commas. If a
|
the video file, separated by commas. If a
|
||||||
|
|
@ -1174,7 +1186,7 @@ Predefined aliases for convenience and ease of use. Note that future
|
||||||
You can configure yt-dlp by placing any supported command line option in a configuration file. The configuration is loaded from the following locations:
|
You can configure yt-dlp by placing any supported command line option in a configuration file. The configuration is loaded from the following locations:
|
||||||
|
|
||||||
1. **Main Configuration**:
|
1. **Main Configuration**:
|
||||||
* The file given to `--config-location`
|
* The file given to `--config-locations`
|
||||||
1. **Portable Configuration**: (Recommended for portable installations)
|
1. **Portable Configuration**: (Recommended for portable installations)
|
||||||
* If using a binary, `yt-dlp.conf` in the same directory as the binary
|
* If using a binary, `yt-dlp.conf` in the same directory as the binary
|
||||||
* If running from source-code, `yt-dlp.conf` in the parent directory of `yt_dlp`
|
* If running from source-code, `yt-dlp.conf` in the parent directory of `yt_dlp`
|
||||||
|
|
@ -1256,7 +1268,7 @@ yt-dlp --netrc-cmd 'gpg --decrypt ~/.authinfo.gpg' 'https://www.youtube.com/watc
|
||||||
|
|
||||||
### Notes about environment variables
|
### Notes about environment variables
|
||||||
* Environment variables are normally specified as `${VARIABLE}`/`$VARIABLE` on UNIX and `%VARIABLE%` on Windows; but is always shown as `${VARIABLE}` in this documentation
|
* Environment variables are normally specified as `${VARIABLE}`/`$VARIABLE` on UNIX and `%VARIABLE%` on Windows; but is always shown as `${VARIABLE}` in this documentation
|
||||||
* yt-dlp also allows using UNIX-style variables on Windows for path-like options; e.g. `--output`, `--config-location`
|
* yt-dlp also allows using UNIX-style variables on Windows for path-like options; e.g. `--output`, `--config-locations`
|
||||||
* If unset, `${XDG_CONFIG_HOME}` defaults to `~/.config` and `${XDG_CACHE_HOME}` to `~/.cache`
|
* If unset, `${XDG_CONFIG_HOME}` defaults to `~/.config` and `${XDG_CACHE_HOME}` to `~/.cache`
|
||||||
* On Windows, `~` points to `${HOME}` if present; or, `${USERPROFILE}` or `${HOMEDRIVE}${HOMEPATH}` otherwise
|
* On Windows, `~` points to `${HOME}` if present; or, `${USERPROFILE}` or `${HOMEDRIVE}${HOMEPATH}` otherwise
|
||||||
* On Windows, `${USERPROFILE}` generally points to `C:\Users\<user name>` and `${APPDATA}` to `${USERPROFILE}\AppData\Roaming`
|
* On Windows, `${USERPROFILE}` generally points to `C:\Users\<user name>` and `${APPDATA}` to `${USERPROFILE}\AppData\Roaming`
|
||||||
|
|
@ -1840,7 +1852,7 @@ The following extractors use this feature:
|
||||||
#### youtube
|
#### youtube
|
||||||
* `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube/_base.py](https://github.com/yt-dlp/yt-dlp/blob/415b4c9f955b1a0391204bd24a7132590e7b3bdb/yt_dlp/extractor/youtube/_base.py#L402-L409) for the list of supported content language codes
|
* `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube/_base.py](https://github.com/yt-dlp/yt-dlp/blob/415b4c9f955b1a0391204bd24a7132590e7b3bdb/yt_dlp/extractor/youtube/_base.py#L402-L409) for the list of supported content language codes
|
||||||
* `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
|
* `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
|
||||||
* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_sdkless`, `android_vr`, `tv`, `tv_simply` and `tv_embedded`. By default, `tv,android_sdkless,web` is used. If no JavaScript runtime is available, then `android_sdkless,web_safari,web` is used. If logged-in cookies are passed to yt-dlp, then `tv,web_safari,web` is used for free accounts and `tv,web_creator,web` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios`
|
* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `android`, `android_sdkless`, `android_vr`, `tv`, `tv_simply`, `tv_downgraded`, and `tv_embedded`. By default, `tv,android_sdkless,web` is used. If no JavaScript runtime is available, then `android_sdkless,web_safari,web` is used. If logged-in cookies are passed to yt-dlp, then `tv_downgraded,web_safari,web` is used for free accounts and `tv_downgraded,web_creator,web` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only works if the video is embeddable. The `tv_embedded` and `web_creator` clients are added for age-restricted videos if account age-verification is required. Some clients, such as `web` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-ios`
|
||||||
* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details
|
* `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details
|
||||||
* `webpage_skip`: Skip extraction of embedded webpage data. One or both of `player_response`, `initial_data`. These options are for testing purposes and don't skip any network requests
|
* `webpage_skip`: Skip extraction of embedded webpage data. One or both of `player_response`, `initial_data`. These options are for testing purposes and don't skip any network requests
|
||||||
* `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
|
* `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
|
||||||
|
|
|
||||||
|
|
@ -308,5 +308,16 @@
|
||||||
"action": "add",
|
"action": "add",
|
||||||
"when": "2c9091e355a7ba5d1edb69796ecdca48199b77fb",
|
"when": "2c9091e355a7ba5d1edb69796ecdca48199b77fb",
|
||||||
"short": "[priority] **A stopgap release with a *TEMPORARY partial* fix for YouTube support**\nSome formats may still be unavailable, especially if cookies are passed to yt-dlp. The ***NEXT*** release, expected very soon, **will require an external JS runtime (e.g. Deno)** in order for YouTube downloads to work properly. [Read more](https://github.com/yt-dlp/yt-dlp/issues/14404)"
|
"short": "[priority] **A stopgap release with a *TEMPORARY partial* fix for YouTube support**\nSome formats may still be unavailable, especially if cookies are passed to yt-dlp. The ***NEXT*** release, expected very soon, **will require an external JS runtime (e.g. Deno)** in order for YouTube downloads to work properly. [Read more](https://github.com/yt-dlp/yt-dlp/issues/14404)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "change",
|
||||||
|
"when": "8636a9bac3bed99984c1e297453660468ecf504b",
|
||||||
|
"short": "Fix 6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc",
|
||||||
|
"authors": ["Grub4K"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "add",
|
||||||
|
"when": "6224a3898821965a7d6a2cb9cc2de40a0fd6e6bc",
|
||||||
|
"short": "[priority] **An external JavaScript runtime is now required for full YouTube support**\nyt-dlp now requires users to have an external JavaScript runtime (e.g. Deno) installed in order to solve the JavaScript challenges presented by YouTube. [Read more](https://github.com/yt-dlp/yt-dlp/issues/15012)"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -353,6 +353,13 @@ class CommitRange:
|
||||||
continue
|
continue
|
||||||
commit = Commit(override_hash, override['short'], override.get('authors') or [])
|
commit = Commit(override_hash, override['short'], override.get('authors') or [])
|
||||||
logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
|
logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}')
|
||||||
|
if match := self.FIXES_RE.search(commit.short):
|
||||||
|
fix_commitish = match.group(1)
|
||||||
|
if fix_commitish in self._commits:
|
||||||
|
del self._commits[commit.hash]
|
||||||
|
self._fixes[fix_commitish].append(commit)
|
||||||
|
logger.info(f'Found fix for {fix_commitish[:HASH_LENGTH]}: {commit.hash[:HASH_LENGTH]}')
|
||||||
|
continue
|
||||||
self._commits[commit.hash] = commit
|
self._commits[commit.hash] = commit
|
||||||
|
|
||||||
self._commits = dict(reversed(self._commits.items()))
|
self._commits = dict(reversed(self._commits.items()))
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,18 @@ def parse_args():
|
||||||
parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
|
parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*')
|
'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*')
|
||||||
|
parser.add_argument(
|
||||||
|
'--flaky',
|
||||||
|
action='store_true',
|
||||||
|
default=None,
|
||||||
|
help='Allow running flaky tests. (default: run, unless in CI)',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--no-flaky',
|
||||||
|
action='store_false',
|
||||||
|
dest='flaky',
|
||||||
|
help=argparse.SUPPRESS,
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
|
'-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|
@ -24,10 +36,11 @@ def parse_args():
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def run_tests(*tests, pattern=None, ci=False):
|
def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None):
|
||||||
# XXX: hatch uses `tests` if no arguments are passed
|
# XXX: hatch uses `tests` if no arguments are passed
|
||||||
run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests)
|
run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests)
|
||||||
run_download = 'download' in tests
|
run_download = 'download' in tests
|
||||||
|
run_flaky = flaky or (flaky is None and not ci)
|
||||||
|
|
||||||
pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
|
pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
|
||||||
arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
|
arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
|
||||||
|
|
@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False):
|
||||||
test if '/' in test
|
test if '/' in test
|
||||||
else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}'
|
else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}'
|
||||||
for test in tests)
|
for test in tests)
|
||||||
|
if not run_flaky:
|
||||||
|
arguments.append('--disallow-flaky')
|
||||||
|
|
||||||
print(f'Running {arguments}', flush=True)
|
print(f'Running {arguments}', flush=True)
|
||||||
try:
|
try:
|
||||||
|
|
@ -72,6 +87,11 @@ if __name__ == '__main__':
|
||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
|
||||||
os.chdir(Path(__file__).parent.parent)
|
os.chdir(Path(__file__).parent.parent)
|
||||||
sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
|
sys.exit(run_tests(
|
||||||
|
*args.test,
|
||||||
|
pattern=args.k,
|
||||||
|
ci=bool(os.getenv('CI')),
|
||||||
|
flaky=args.flaky,
|
||||||
|
))
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
pass
|
pass
|
||||||
|
|
|
||||||
4
devscripts/update_ejs.py
Normal file → Executable file
4
devscripts/update_ejs.py
Normal file → Executable file
|
|
@ -66,7 +66,9 @@ def list_wheel_contents(
|
||||||
) -> str:
|
) -> str:
|
||||||
assert folders or files, 'at least one of "folders" or "files" must be True'
|
assert folders or files, 'at least one of "folders" or "files" must be True'
|
||||||
|
|
||||||
path_gen = (zinfo.filename for zinfo in zipfile.ZipFile(io.BytesIO(wheel_data)).infolist())
|
with zipfile.ZipFile(io.BytesIO(wheel_data)) as zipf:
|
||||||
|
path_gen = (zinfo.filename for zinfo in zipf.infolist())
|
||||||
|
|
||||||
filtered = filter(lambda path: path.startswith('yt_dlp_ejs/'), path_gen)
|
filtered = filter(lambda path: path.startswith('yt_dlp_ejs/'), path_gen)
|
||||||
if suffix:
|
if suffix:
|
||||||
filtered = filter(lambda path: path.endswith(f'.{suffix}'), filtered)
|
filtered = filter(lambda path: path.endswith(f'.{suffix}'), filtered)
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ def build_completion(opt_parser):
|
||||||
for opt in group.option_list]
|
for opt in group.option_list]
|
||||||
opts_file = [opt for opt in opts if opt.metavar == 'FILE']
|
opts_file = [opt for opt in opts if opt.metavar == 'FILE']
|
||||||
opts_dir = [opt for opt in opts if opt.metavar == 'DIR']
|
opts_dir = [opt for opt in opts if opt.metavar == 'DIR']
|
||||||
|
opts_path = [opt for opt in opts if opt.metavar == 'PATH']
|
||||||
|
|
||||||
fileopts = []
|
fileopts = []
|
||||||
for opt in opts_file:
|
for opt in opts_file:
|
||||||
|
|
@ -26,6 +27,12 @@ def build_completion(opt_parser):
|
||||||
if opt._long_opts:
|
if opt._long_opts:
|
||||||
fileopts.extend(opt._long_opts)
|
fileopts.extend(opt._long_opts)
|
||||||
|
|
||||||
|
for opt in opts_path:
|
||||||
|
if opt._short_opts:
|
||||||
|
fileopts.extend(opt._short_opts)
|
||||||
|
if opt._long_opts:
|
||||||
|
fileopts.extend(opt._long_opts)
|
||||||
|
|
||||||
diropts = []
|
diropts = []
|
||||||
for opt in opts_dir:
|
for opt in opts_dir:
|
||||||
if opt._short_opts:
|
if opt._short_opts:
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ default = [
|
||||||
"requests>=2.32.2,<3",
|
"requests>=2.32.2,<3",
|
||||||
"urllib3>=2.0.2,<3",
|
"urllib3>=2.0.2,<3",
|
||||||
"websockets>=13.0",
|
"websockets>=13.0",
|
||||||
"yt-dlp-ejs==0.3.0",
|
"yt-dlp-ejs==0.3.1",
|
||||||
]
|
]
|
||||||
curl-cffi = [
|
curl-cffi = [
|
||||||
"curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.14; implementation_name=='cpython'",
|
"curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.14; implementation_name=='cpython'",
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **17live:vod**
|
- **17live:vod**
|
||||||
- **1News**: 1news.co.nz article videos
|
- **1News**: 1news.co.nz article videos
|
||||||
- **1tv**: Первый канал
|
- **1tv**: Первый канал
|
||||||
|
- **1tv:live**: Первый канал (прямой эфир)
|
||||||
- **20min**: (**Currently broken**)
|
- **20min**: (**Currently broken**)
|
||||||
- **23video**
|
- **23video**
|
||||||
- **247sports**: (**Currently broken**)
|
- **247sports**: (**Currently broken**)
|
||||||
|
|
@ -93,6 +94,8 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **archive.org**: archive.org video and audio
|
- **archive.org**: archive.org video and audio
|
||||||
- **ArcPublishing**
|
- **ArcPublishing**
|
||||||
- **ARD**
|
- **ARD**
|
||||||
|
- **ARDAudiothek**
|
||||||
|
- **ARDAudiothekPlaylist**
|
||||||
- **ARDMediathek**
|
- **ARDMediathek**
|
||||||
- **ARDMediathekCollection**
|
- **ARDMediathekCollection**
|
||||||
- **Art19**
|
- **Art19**
|
||||||
|
|
@ -533,7 +536,6 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **google:podcasts:feed**
|
- **google:podcasts:feed**
|
||||||
- **GoogleDrive**
|
- **GoogleDrive**
|
||||||
- **GoogleDrive:Folder**
|
- **GoogleDrive:Folder**
|
||||||
- **GoPlay**: [*goplay*](## "netrc machine")
|
|
||||||
- **GoPro**
|
- **GoPro**
|
||||||
- **Goshgay**
|
- **Goshgay**
|
||||||
- **GoToStage**
|
- **GoToStage**
|
||||||
|
|
@ -844,6 +846,7 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **MusicdexArtist**
|
- **MusicdexArtist**
|
||||||
- **MusicdexPlaylist**
|
- **MusicdexPlaylist**
|
||||||
- **MusicdexSong**
|
- **MusicdexSong**
|
||||||
|
- **Mux**
|
||||||
- **Mx3**
|
- **Mx3**
|
||||||
- **Mx3Neo**
|
- **Mx3Neo**
|
||||||
- **Mx3Volksmusik**
|
- **Mx3Volksmusik**
|
||||||
|
|
@ -858,6 +861,7 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **n-tv.de**
|
- **n-tv.de**
|
||||||
- **N1Info:article**
|
- **N1Info:article**
|
||||||
- **N1InfoAsset**
|
- **N1InfoAsset**
|
||||||
|
- **NascarClassics**
|
||||||
- **Nate**
|
- **Nate**
|
||||||
- **NateProgram**
|
- **NateProgram**
|
||||||
- **natgeo:video**
|
- **natgeo:video**
|
||||||
|
|
@ -1071,6 +1075,7 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **PlanetMarathi**
|
- **PlanetMarathi**
|
||||||
- **Platzi**: [*platzi*](## "netrc machine")
|
- **Platzi**: [*platzi*](## "netrc machine")
|
||||||
- **PlatziCourse**: [*platzi*](## "netrc machine")
|
- **PlatziCourse**: [*platzi*](## "netrc machine")
|
||||||
|
- **play.tv**: [*goplay*](## "netrc machine") PLAY (formerly goplay.be)
|
||||||
- **player.sky.it**
|
- **player.sky.it**
|
||||||
- **PlayerFm**
|
- **PlayerFm**
|
||||||
- **playeur**
|
- **playeur**
|
||||||
|
|
@ -1559,12 +1564,12 @@ The only reliable way to check if a site is supported is to try it.
|
||||||
- **TwitCastingLive**
|
- **TwitCastingLive**
|
||||||
- **TwitCastingUser**
|
- **TwitCastingUser**
|
||||||
- **twitch:clips**: [*twitch*](## "netrc machine")
|
- **twitch:clips**: [*twitch*](## "netrc machine")
|
||||||
|
- **twitch:collection**: [*twitch*](## "netrc machine")
|
||||||
- **twitch:stream**: [*twitch*](## "netrc machine")
|
- **twitch:stream**: [*twitch*](## "netrc machine")
|
||||||
|
- **twitch:videos**: [*twitch*](## "netrc machine")
|
||||||
|
- **twitch:videos:clips**: [*twitch*](## "netrc machine")
|
||||||
|
- **twitch:videos:collections**: [*twitch*](## "netrc machine")
|
||||||
- **twitch:vod**: [*twitch*](## "netrc machine")
|
- **twitch:vod**: [*twitch*](## "netrc machine")
|
||||||
- **TwitchCollection**: [*twitch*](## "netrc machine")
|
|
||||||
- **TwitchVideos**: [*twitch*](## "netrc machine")
|
|
||||||
- **TwitchVideosClips**: [*twitch*](## "netrc machine")
|
|
||||||
- **TwitchVideosCollections**: [*twitch*](## "netrc machine")
|
|
||||||
- **twitter**: [*twitter*](## "netrc machine")
|
- **twitter**: [*twitter*](## "netrc machine")
|
||||||
- **twitter:amplify**: [*twitter*](## "netrc machine")
|
- **twitter:amplify**: [*twitter*](## "netrc machine")
|
||||||
- **twitter:broadcast**: [*twitter*](## "netrc machine")
|
- **twitter:broadcast**: [*twitter*](## "netrc machine")
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,33 @@ def skip_handlers_if(request, handler):
|
||||||
pytest.skip(marker.args[1] if len(marker.args) > 1 else '')
|
pytest.skip(marker.args[1] if len(marker.args) > 1 else '')
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def handler_flaky(request, handler):
|
||||||
|
"""Mark a certain handler as being flaky.
|
||||||
|
|
||||||
|
This will skip the test if pytest does not get run using `--allow-flaky`
|
||||||
|
|
||||||
|
usage:
|
||||||
|
pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason')
|
||||||
|
"""
|
||||||
|
for marker in request.node.iter_markers(handler_flaky.__name__):
|
||||||
|
if (
|
||||||
|
marker.args[0] == handler.RH_KEY
|
||||||
|
and (not marker.args[1:] or any(marker.args[1:]))
|
||||||
|
and request.config.getoption('disallow_flaky')
|
||||||
|
):
|
||||||
|
reason = marker.kwargs.get('reason')
|
||||||
|
pytest.skip(f'flaky: {reason}' if reason else 'flaky')
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser, pluginmanager):
|
||||||
|
parser.addoption(
|
||||||
|
'--disallow-flaky',
|
||||||
|
action='store_true',
|
||||||
|
help='disallow flaky tests from running.',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
config.addinivalue_line(
|
config.addinivalue_line(
|
||||||
'markers', 'skip_handler(handler): skip test for the given handler',
|
'markers', 'skip_handler(handler): skip test for the given handler',
|
||||||
|
|
@ -62,3 +89,6 @@ def pytest_configure(config):
|
||||||
config.addinivalue_line(
|
config.addinivalue_line(
|
||||||
'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true',
|
'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true',
|
||||||
)
|
)
|
||||||
|
config.addinivalue_line(
|
||||||
|
'markers', 'handler_flaky(handler): mark handler as flaky if condition is true',
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -247,6 +247,7 @@ def ctx(request):
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
|
||||||
@pytest.mark.parametrize('ctx', ['http'], indirect=True) # pure http proxy can only support http
|
@pytest.mark.parametrize('ctx', ['http'], indirect=True) # pure http proxy can only support http
|
||||||
class TestHTTPProxy:
|
class TestHTTPProxy:
|
||||||
def test_http_no_auth(self, handler, ctx):
|
def test_http_no_auth(self, handler, ctx):
|
||||||
|
|
@ -315,6 +316,7 @@ class TestHTTPProxy:
|
||||||
('Requests', 'https'),
|
('Requests', 'https'),
|
||||||
('CurlCFFI', 'https'),
|
('CurlCFFI', 'https'),
|
||||||
], indirect=True)
|
], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
|
||||||
class TestHTTPConnectProxy:
|
class TestHTTPConnectProxy:
|
||||||
def test_http_connect_no_auth(self, handler, ctx):
|
def test_http_connect_no_auth(self, handler, ctx):
|
||||||
with ctx.http_server(HTTPConnectProxyHandler) as server_address:
|
with ctx.http_server(HTTPConnectProxyHandler) as server_address:
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
@ -311,6 +312,7 @@ class TestRequestHandlerBase:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
|
||||||
class TestHTTPRequestHandler(TestRequestHandlerBase):
|
class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
|
|
||||||
def test_verify_cert(self, handler):
|
def test_verify_cert(self, handler):
|
||||||
|
|
@ -614,8 +616,11 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
@pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
|
@pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
|
||||||
def test_gzip_trailing_garbage(self, handler):
|
def test_gzip_trailing_garbage(self, handler):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
data = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage')).read().decode()
|
res = validate_and_send(rh, Request(f'http://localhost:{self.http_port}/trailing_garbage'))
|
||||||
|
data = res.read().decode()
|
||||||
assert data == '<html><video src="/vid.mp4" /></html>'
|
assert data == '<html><video src="/vid.mp4" /></html>'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
@pytest.mark.skip_handler('CurlCFFI', 'not applicable to curl-cffi')
|
@pytest.mark.skip_handler('CurlCFFI', 'not applicable to curl-cffi')
|
||||||
@pytest.mark.skipif(not brotli, reason='brotli support is not installed')
|
@pytest.mark.skipif(not brotli, reason='brotli support is not installed')
|
||||||
|
|
@ -627,6 +632,8 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
headers={'ytdl-encoding': 'br'}))
|
headers={'ytdl-encoding': 'br'}))
|
||||||
assert res.headers.get('Content-Encoding') == 'br'
|
assert res.headers.get('Content-Encoding') == 'br'
|
||||||
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_deflate(self, handler):
|
def test_deflate(self, handler):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
|
|
@ -636,6 +643,8 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
headers={'ytdl-encoding': 'deflate'}))
|
headers={'ytdl-encoding': 'deflate'}))
|
||||||
assert res.headers.get('Content-Encoding') == 'deflate'
|
assert res.headers.get('Content-Encoding') == 'deflate'
|
||||||
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_gzip(self, handler):
|
def test_gzip(self, handler):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
|
|
@ -645,6 +654,8 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
headers={'ytdl-encoding': 'gzip'}))
|
headers={'ytdl-encoding': 'gzip'}))
|
||||||
assert res.headers.get('Content-Encoding') == 'gzip'
|
assert res.headers.get('Content-Encoding') == 'gzip'
|
||||||
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_multiple_encodings(self, handler):
|
def test_multiple_encodings(self, handler):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
|
|
@ -655,6 +666,8 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
headers={'ytdl-encoding': pair}))
|
headers={'ytdl-encoding': pair}))
|
||||||
assert res.headers.get('Content-Encoding') == pair
|
assert res.headers.get('Content-Encoding') == pair
|
||||||
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
assert res.read() == b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
@pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
|
@pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
|
||||||
def test_unsupported_encoding(self, handler):
|
def test_unsupported_encoding(self, handler):
|
||||||
|
|
@ -665,6 +678,8 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
headers={'ytdl-encoding': 'unsupported', 'Accept-Encoding': '*'}))
|
headers={'ytdl-encoding': 'unsupported', 'Accept-Encoding': '*'}))
|
||||||
assert res.headers.get('Content-Encoding') == 'unsupported'
|
assert res.headers.get('Content-Encoding') == 'unsupported'
|
||||||
assert res.read() == b'raw'
|
assert res.read() == b'raw'
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_read(self, handler):
|
def test_read(self, handler):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
|
|
@ -672,9 +687,13 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
rh, Request(f'http://127.0.0.1:{self.http_port}/headers'))
|
rh, Request(f'http://127.0.0.1:{self.http_port}/headers'))
|
||||||
assert res.readable()
|
assert res.readable()
|
||||||
assert res.read(1) == b'H'
|
assert res.read(1) == b'H'
|
||||||
|
# Ensure we don't close the adaptor yet
|
||||||
|
assert not res.closed
|
||||||
assert res.read(3) == b'ost'
|
assert res.read(3) == b'ost'
|
||||||
assert res.read().decode().endswith('\n\n')
|
assert res.read().decode().endswith('\n\n')
|
||||||
assert res.read() == b''
|
assert res.read() == b''
|
||||||
|
# Should auto-close and mark the response adaptor as closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_request_disable_proxy(self, handler):
|
def test_request_disable_proxy(self, handler):
|
||||||
for proxy_proto in handler._SUPPORTED_PROXY_SCHEMES or ['http']:
|
for proxy_proto in handler._SUPPORTED_PROXY_SCHEMES or ['http']:
|
||||||
|
|
@ -736,8 +755,20 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
||||||
assert res.read(0) == b''
|
assert res.read(0) == b''
|
||||||
assert res.read() == b'<video src="/vid.mp4" /></html>'
|
assert res.read() == b'<video src="/vid.mp4" /></html>'
|
||||||
|
|
||||||
|
def test_partial_read_greater_than_response_then_full_read(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
for encoding in ('', 'gzip', 'deflate'):
|
||||||
|
res = validate_and_send(rh, Request(
|
||||||
|
f'http://127.0.0.1:{self.http_port}/content-encoding',
|
||||||
|
headers={'ytdl-encoding': encoding}))
|
||||||
|
assert res.headers.get('Content-Encoding') == encoding
|
||||||
|
assert res.read(512) == b'<html><video src="/vid.mp4" /></html>'
|
||||||
|
assert res.read(0) == b''
|
||||||
|
assert res.read() == b''
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
|
||||||
class TestClientCertificate:
|
class TestClientCertificate:
|
||||||
@classmethod
|
@classmethod
|
||||||
def setup_class(cls):
|
def setup_class(cls):
|
||||||
|
|
@ -875,11 +906,53 @@ class TestUrllibRequestHandler(TestRequestHandlerBase):
|
||||||
|
|
||||||
with handler(enable_file_urls=True) as rh:
|
with handler(enable_file_urls=True) as rh:
|
||||||
res = validate_and_send(rh, req)
|
res = validate_and_send(rh, req)
|
||||||
assert res.read() == b'foobar'
|
assert res.read(1) == b'f'
|
||||||
res.close()
|
assert not res.fp.closed
|
||||||
|
assert res.read() == b'oobar'
|
||||||
|
# Should automatically close the underlying file object
|
||||||
|
assert res.fp.closed
|
||||||
|
|
||||||
os.unlink(tf.name)
|
os.unlink(tf.name)
|
||||||
|
|
||||||
|
def test_data_uri_auto_close(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request('data:text/plain,hello%20world'))
|
||||||
|
assert res.read() == b'hello world'
|
||||||
|
# Should automatically close the underlying file object
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
def test_http_response_auto_close(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/gen_200'))
|
||||||
|
assert res.read() == b'<html></html>'
|
||||||
|
# Should automatically close the underlying file object in the HTTP Response
|
||||||
|
assert isinstance(res.fp, http.client.HTTPResponse)
|
||||||
|
assert res.fp.fp is None
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
def test_data_uri_partial_read_then_full_read(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request('data:text/plain,hello%20world'))
|
||||||
|
assert res.read(6) == b'hello '
|
||||||
|
assert res.read(0) == b''
|
||||||
|
assert res.read() == b'world'
|
||||||
|
# Should automatically close the underlying file object
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
def test_data_uri_partial_read_greater_than_response_then_full_read(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request('data:text/plain,hello%20world'))
|
||||||
|
assert res.read(512) == b'hello world'
|
||||||
|
# Response and its underlying file object should already be closed now
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
assert res.read(0) == b''
|
||||||
|
assert res.read() == b''
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
def test_http_error_returns_content(self, handler):
|
def test_http_error_returns_content(self, handler):
|
||||||
# urllib HTTPError will try close the underlying response if reference to the HTTPError object is lost
|
# urllib HTTPError will try close the underlying response if reference to the HTTPError object is lost
|
||||||
def get_response():
|
def get_response():
|
||||||
|
|
@ -1012,8 +1085,17 @@ class TestRequestsRequestHandler(TestRequestHandlerBase):
|
||||||
rh.close()
|
rh.close()
|
||||||
assert called
|
assert called
|
||||||
|
|
||||||
|
def test_http_response_auto_close(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/gen_200'))
|
||||||
|
assert res.read() == b'<html></html>'
|
||||||
|
# Should automatically close the underlying file object in the HTTP Response
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
|
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
|
||||||
class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
|
class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
|
||||||
|
|
||||||
@pytest.mark.parametrize('params,extensions', [
|
@pytest.mark.parametrize('params,extensions', [
|
||||||
|
|
@ -1177,6 +1259,14 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
|
||||||
assert res4.closed
|
assert res4.closed
|
||||||
assert res4._buffer == b''
|
assert res4._buffer == b''
|
||||||
|
|
||||||
|
def test_http_response_auto_close(self, handler):
|
||||||
|
with handler() as rh:
|
||||||
|
res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/gen_200'))
|
||||||
|
assert res.read() == b'<html></html>'
|
||||||
|
# Should automatically close the underlying file object in the HTTP Response
|
||||||
|
assert res.fp.closed
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
|
||||||
def run_validation(handler, error, req, **handler_kwargs):
|
def run_validation(handler, error, req, **handler_kwargs):
|
||||||
with handler(**handler_kwargs) as rh:
|
with handler(**handler_kwargs) as rh:
|
||||||
|
|
@ -2032,6 +2122,30 @@ class TestResponse:
|
||||||
assert res.info() is res.headers
|
assert res.info() is res.headers
|
||||||
assert res.getheader('test') == res.get_header('test')
|
assert res.getheader('test') == res.get_header('test')
|
||||||
|
|
||||||
|
def test_auto_close(self):
|
||||||
|
# Should mark the response as closed if the underlying file is closed
|
||||||
|
class AutoCloseBytesIO(io.BytesIO):
|
||||||
|
def read(self, size=-1, /):
|
||||||
|
data = super().read(size)
|
||||||
|
self.close()
|
||||||
|
return data
|
||||||
|
|
||||||
|
fp = AutoCloseBytesIO(b'test')
|
||||||
|
res = Response(fp, url='test://', headers={}, status=200)
|
||||||
|
assert not res.closed
|
||||||
|
res.read()
|
||||||
|
assert res.closed
|
||||||
|
|
||||||
|
def test_close(self):
|
||||||
|
# Should not call close() on the underlying file when already closed
|
||||||
|
fp = MagicMock()
|
||||||
|
fp.closed = False
|
||||||
|
res = Response(fp, url='test://', headers={}, status=200)
|
||||||
|
res.close()
|
||||||
|
fp.closed = True
|
||||||
|
res.close()
|
||||||
|
assert fp.close.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
class TestImpersonateTarget:
|
class TestImpersonateTarget:
|
||||||
@pytest.mark.parametrize('target_str,expected', [
|
@pytest.mark.parametrize('target_str,expected', [
|
||||||
|
|
|
||||||
|
|
@ -295,6 +295,7 @@ def ctx(request):
|
||||||
('Websockets', 'ws'),
|
('Websockets', 'ws'),
|
||||||
('CurlCFFI', 'http'),
|
('CurlCFFI', 'http'),
|
||||||
], indirect=True)
|
], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
|
||||||
class TestSocks4Proxy:
|
class TestSocks4Proxy:
|
||||||
def test_socks4_no_auth(self, handler, ctx):
|
def test_socks4_no_auth(self, handler, ctx):
|
||||||
with handler() as rh:
|
with handler() as rh:
|
||||||
|
|
@ -370,6 +371,7 @@ class TestSocks4Proxy:
|
||||||
('Websockets', 'ws'),
|
('Websockets', 'ws'),
|
||||||
('CurlCFFI', 'http'),
|
('CurlCFFI', 'http'),
|
||||||
], indirect=True)
|
], indirect=True)
|
||||||
|
@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
|
||||||
class TestSocks5Proxy:
|
class TestSocks5Proxy:
|
||||||
|
|
||||||
def test_socks5_no_auth(self, handler, ctx):
|
def test_socks5_no_auth(self, handler, ctx):
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,13 @@ from yt_dlp.utils.networking import HTTPHeaderDict
|
||||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.handler_flaky(
|
||||||
|
'Websockets',
|
||||||
|
os.name == 'nt' or sys.implementation.name == 'pypy',
|
||||||
|
reason='segfaults',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def websocket_handler(websocket):
|
def websocket_handler(websocket):
|
||||||
for message in websocket:
|
for message in websocket:
|
||||||
if isinstance(message, bytes):
|
if isinstance(message, bytes):
|
||||||
|
|
|
||||||
|
|
@ -125,7 +125,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(),
|
||||||
|
|
||||||
|
|
||||||
def _extract_firefox_cookies(profile, container, logger):
|
def _extract_firefox_cookies(profile, container, logger):
|
||||||
MAX_SUPPORTED_DB_SCHEMA_VERSION = 16
|
MAX_SUPPORTED_DB_SCHEMA_VERSION = 17
|
||||||
|
|
||||||
logger.info('Extracting cookies from firefox')
|
logger.info('Extracting cookies from firefox')
|
||||||
if not sqlite3:
|
if not sqlite3:
|
||||||
|
|
@ -166,6 +166,8 @@ def _extract_firefox_cookies(profile, container, logger):
|
||||||
db_schema_version = cursor.execute('PRAGMA user_version;').fetchone()[0]
|
db_schema_version = cursor.execute('PRAGMA user_version;').fetchone()[0]
|
||||||
if db_schema_version > MAX_SUPPORTED_DB_SCHEMA_VERSION:
|
if db_schema_version > MAX_SUPPORTED_DB_SCHEMA_VERSION:
|
||||||
logger.warning(f'Possibly unsupported firefox cookies database version: {db_schema_version}')
|
logger.warning(f'Possibly unsupported firefox cookies database version: {db_schema_version}')
|
||||||
|
else:
|
||||||
|
logger.debug(f'Firefox cookies database version: {db_schema_version}')
|
||||||
if isinstance(container_id, int):
|
if isinstance(container_id, int):
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f'Only loading cookies from firefox container "{container}", ID {container_id}')
|
f'Only loading cookies from firefox container "{container}", ID {container_id}')
|
||||||
|
|
@ -557,7 +559,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor):
|
||||||
|
|
||||||
|
|
||||||
def _extract_safari_cookies(profile, logger):
|
def _extract_safari_cookies(profile, logger):
|
||||||
if sys.platform != 'darwin':
|
if sys.platform not in ('darwin', 'ios'):
|
||||||
raise ValueError(f'unsupported platform: {sys.platform}')
|
raise ValueError(f'unsupported platform: {sys.platform}')
|
||||||
|
|
||||||
if profile:
|
if profile:
|
||||||
|
|
|
||||||
|
|
@ -461,7 +461,8 @@ class FileDownloader:
|
||||||
min_sleep_interval = self.params.get('sleep_interval') or 0
|
min_sleep_interval = self.params.get('sleep_interval') or 0
|
||||||
max_sleep_interval = self.params.get('max_sleep_interval') or 0
|
max_sleep_interval = self.params.get('max_sleep_interval') or 0
|
||||||
|
|
||||||
if available_at := info_dict.get('available_at'):
|
requested_formats = info_dict.get('requested_formats') or [info_dict]
|
||||||
|
if available_at := max(f.get('available_at') or 0 for f in requested_formats):
|
||||||
forced_sleep_interval = available_at - int(time.time())
|
forced_sleep_interval = available_at - int(time.time())
|
||||||
if forced_sleep_interval > min_sleep_interval:
|
if forced_sleep_interval > min_sleep_interval:
|
||||||
sleep_note = 'as required by the site'
|
sleep_note = 'as required by the site'
|
||||||
|
|
|
||||||
|
|
@ -457,6 +457,8 @@ class FFmpegFD(ExternalFD):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls, path=None):
|
def available(cls, path=None):
|
||||||
|
# TODO: Fix path for ffmpeg
|
||||||
|
# Fixme: This may be wrong when --ffmpeg-location is used
|
||||||
return FFmpegPostProcessor().available
|
return FFmpegPostProcessor().available
|
||||||
|
|
||||||
def on_process_started(self, proc, stdin):
|
def on_process_started(self, proc, stdin):
|
||||||
|
|
@ -488,20 +490,6 @@ class FFmpegFD(ExternalFD):
|
||||||
if not self.params.get('verbose'):
|
if not self.params.get('verbose'):
|
||||||
args += ['-hide_banner']
|
args += ['-hide_banner']
|
||||||
|
|
||||||
args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args', ...))
|
|
||||||
|
|
||||||
# These exists only for compatibility. Extractors should use
|
|
||||||
# info_dict['downloader_options']['ffmpeg_args'] instead
|
|
||||||
args += info_dict.get('_ffmpeg_args') or []
|
|
||||||
seekable = info_dict.get('_seekable')
|
|
||||||
if seekable is not None:
|
|
||||||
# setting -seekable prevents ffmpeg from guessing if the server
|
|
||||||
# supports seeking(by adding the header `Range: bytes=0-`), which
|
|
||||||
# can cause problems in some cases
|
|
||||||
# https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
|
|
||||||
# http://trac.ffmpeg.org/ticket/6125#comment:10
|
|
||||||
args += ['-seekable', '1' if seekable else '0']
|
|
||||||
|
|
||||||
env = None
|
env = None
|
||||||
proxy = self.params.get('proxy')
|
proxy = self.params.get('proxy')
|
||||||
if proxy:
|
if proxy:
|
||||||
|
|
@ -521,39 +509,10 @@ class FFmpegFD(ExternalFD):
|
||||||
env['HTTP_PROXY'] = proxy
|
env['HTTP_PROXY'] = proxy
|
||||||
env['http_proxy'] = proxy
|
env['http_proxy'] = proxy
|
||||||
|
|
||||||
protocol = info_dict.get('protocol')
|
|
||||||
|
|
||||||
if protocol == 'rtmp':
|
|
||||||
player_url = info_dict.get('player_url')
|
|
||||||
page_url = info_dict.get('page_url')
|
|
||||||
app = info_dict.get('app')
|
|
||||||
play_path = info_dict.get('play_path')
|
|
||||||
tc_url = info_dict.get('tc_url')
|
|
||||||
flash_version = info_dict.get('flash_version')
|
|
||||||
live = info_dict.get('rtmp_live', False)
|
|
||||||
conn = info_dict.get('rtmp_conn')
|
|
||||||
if player_url is not None:
|
|
||||||
args += ['-rtmp_swfverify', player_url]
|
|
||||||
if page_url is not None:
|
|
||||||
args += ['-rtmp_pageurl', page_url]
|
|
||||||
if app is not None:
|
|
||||||
args += ['-rtmp_app', app]
|
|
||||||
if play_path is not None:
|
|
||||||
args += ['-rtmp_playpath', play_path]
|
|
||||||
if tc_url is not None:
|
|
||||||
args += ['-rtmp_tcurl', tc_url]
|
|
||||||
if flash_version is not None:
|
|
||||||
args += ['-rtmp_flashver', flash_version]
|
|
||||||
if live:
|
|
||||||
args += ['-rtmp_live', 'live']
|
|
||||||
if isinstance(conn, list):
|
|
||||||
for entry in conn:
|
|
||||||
args += ['-rtmp_conn', entry]
|
|
||||||
elif isinstance(conn, str):
|
|
||||||
args += ['-rtmp_conn', conn]
|
|
||||||
|
|
||||||
start_time, end_time = info_dict.get('section_start') or 0, info_dict.get('section_end')
|
start_time, end_time = info_dict.get('section_start') or 0, info_dict.get('section_end')
|
||||||
|
|
||||||
|
fallback_input_args = traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args', ...))
|
||||||
|
|
||||||
selected_formats = info_dict.get('requested_formats') or [info_dict]
|
selected_formats = info_dict.get('requested_formats') or [info_dict]
|
||||||
for i, fmt in enumerate(selected_formats):
|
for i, fmt in enumerate(selected_formats):
|
||||||
is_http = re.match(r'https?://', fmt['url'])
|
is_http = re.match(r'https?://', fmt['url'])
|
||||||
|
|
@ -572,6 +531,44 @@ class FFmpegFD(ExternalFD):
|
||||||
if end_time:
|
if end_time:
|
||||||
args += ['-t', str(end_time - start_time)]
|
args += ['-t', str(end_time - start_time)]
|
||||||
|
|
||||||
|
protocol = fmt.get('protocol')
|
||||||
|
|
||||||
|
if protocol == 'rtmp':
|
||||||
|
player_url = fmt.get('player_url')
|
||||||
|
page_url = fmt.get('page_url')
|
||||||
|
app = fmt.get('app')
|
||||||
|
play_path = fmt.get('play_path')
|
||||||
|
tc_url = fmt.get('tc_url')
|
||||||
|
flash_version = fmt.get('flash_version')
|
||||||
|
live = fmt.get('rtmp_live', False)
|
||||||
|
conn = fmt.get('rtmp_conn')
|
||||||
|
if player_url is not None:
|
||||||
|
args += ['-rtmp_swfverify', player_url]
|
||||||
|
if page_url is not None:
|
||||||
|
args += ['-rtmp_pageurl', page_url]
|
||||||
|
if app is not None:
|
||||||
|
args += ['-rtmp_app', app]
|
||||||
|
if play_path is not None:
|
||||||
|
args += ['-rtmp_playpath', play_path]
|
||||||
|
if tc_url is not None:
|
||||||
|
args += ['-rtmp_tcurl', tc_url]
|
||||||
|
if flash_version is not None:
|
||||||
|
args += ['-rtmp_flashver', flash_version]
|
||||||
|
if live:
|
||||||
|
args += ['-rtmp_live', 'live']
|
||||||
|
if isinstance(conn, list):
|
||||||
|
for entry in conn:
|
||||||
|
args += ['-rtmp_conn', entry]
|
||||||
|
elif isinstance(conn, str):
|
||||||
|
args += ['-rtmp_conn', conn]
|
||||||
|
|
||||||
|
elif protocol == 'http_dash_segments' and info_dict.get('is_live'):
|
||||||
|
# ffmpeg may try to read past the latest available segments for
|
||||||
|
# live DASH streams unless we pass `-re`. In modern ffmpeg, this
|
||||||
|
# is an alias of `-readrate 1`, but `-readrate` was not added
|
||||||
|
# until ffmpeg 5.0, so we must stick to using `-re`
|
||||||
|
args += ['-re']
|
||||||
|
|
||||||
url = fmt['url']
|
url = fmt['url']
|
||||||
if self.params.get('enable_file_urls') and url.startswith('file:'):
|
if self.params.get('enable_file_urls') and url.startswith('file:'):
|
||||||
# The default protocol_whitelist is 'file,crypto,data' when reading local m3u8 URLs,
|
# The default protocol_whitelist is 'file,crypto,data' when reading local m3u8 URLs,
|
||||||
|
|
@ -586,6 +583,7 @@ class FFmpegFD(ExternalFD):
|
||||||
# https://trac.ffmpeg.org/ticket/2702
|
# https://trac.ffmpeg.org/ticket/2702
|
||||||
url = re.sub(r'^file://(?:localhost)?/', 'file:' if os.name == 'nt' else 'file:/', url)
|
url = re.sub(r'^file://(?:localhost)?/', 'file:' if os.name == 'nt' else 'file:/', url)
|
||||||
|
|
||||||
|
args += traverse_obj(fmt, ('downloader_options', 'ffmpeg_args', ...)) or fallback_input_args
|
||||||
args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', url]
|
args += [*self._configuration_args((f'_i{i + 1}', '_i')), '-i', url]
|
||||||
|
|
||||||
if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'):
|
if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'):
|
||||||
|
|
|
||||||
|
|
@ -268,6 +268,7 @@ from .bitchute import (
|
||||||
BitChuteChannelIE,
|
BitChuteChannelIE,
|
||||||
BitChuteIE,
|
BitChuteIE,
|
||||||
)
|
)
|
||||||
|
from .bitmovin import BitmovinIE
|
||||||
from .blackboardcollaborate import (
|
from .blackboardcollaborate import (
|
||||||
BlackboardCollaborateIE,
|
BlackboardCollaborateIE,
|
||||||
BlackboardCollaborateLaunchIE,
|
BlackboardCollaborateLaunchIE,
|
||||||
|
|
@ -640,7 +641,10 @@ from .filmon import (
|
||||||
FilmOnIE,
|
FilmOnIE,
|
||||||
)
|
)
|
||||||
from .filmweb import FilmwebIE
|
from .filmweb import FilmwebIE
|
||||||
from .firsttv import FirstTVIE
|
from .firsttv import (
|
||||||
|
FirstTVIE,
|
||||||
|
FirstTVLiveIE,
|
||||||
|
)
|
||||||
from .fivetv import FiveTVIE
|
from .fivetv import FiveTVIE
|
||||||
from .flextv import FlexTVIE
|
from .flextv import FlexTVIE
|
||||||
from .flickr import FlickrIE
|
from .flickr import FlickrIE
|
||||||
|
|
@ -687,6 +691,10 @@ from .frontendmasters import (
|
||||||
FrontendMastersIE,
|
FrontendMastersIE,
|
||||||
FrontendMastersLessonIE,
|
FrontendMastersLessonIE,
|
||||||
)
|
)
|
||||||
|
from .frontro import (
|
||||||
|
TheChosenGroupIE,
|
||||||
|
TheChosenIE,
|
||||||
|
)
|
||||||
from .fujitv import FujiTVFODPlus7IE
|
from .fujitv import FujiTVFODPlus7IE
|
||||||
from .funk import FunkIE
|
from .funk import FunkIE
|
||||||
from .funker530 import Funker530IE
|
from .funker530 import Funker530IE
|
||||||
|
|
@ -1090,7 +1098,10 @@ from .markiza import (
|
||||||
from .massengeschmacktv import MassengeschmackTVIE
|
from .massengeschmacktv import MassengeschmackTVIE
|
||||||
from .masters import MastersIE
|
from .masters import MastersIE
|
||||||
from .matchtv import MatchTVIE
|
from .matchtv import MatchTVIE
|
||||||
from .mave import MaveIE
|
from .mave import (
|
||||||
|
MaveChannelIE,
|
||||||
|
MaveIE,
|
||||||
|
)
|
||||||
from .mbn import MBNIE
|
from .mbn import MBNIE
|
||||||
from .mdr import MDRIE
|
from .mdr import MDRIE
|
||||||
from .medaltv import MedalTVIE
|
from .medaltv import MedalTVIE
|
||||||
|
|
@ -1197,6 +1208,7 @@ from .musicdex import (
|
||||||
MusicdexPlaylistIE,
|
MusicdexPlaylistIE,
|
||||||
MusicdexSongIE,
|
MusicdexSongIE,
|
||||||
)
|
)
|
||||||
|
from .mux import MuxIE
|
||||||
from .mx3 import (
|
from .mx3 import (
|
||||||
Mx3IE,
|
Mx3IE,
|
||||||
Mx3NeoIE,
|
Mx3NeoIE,
|
||||||
|
|
@ -1272,6 +1284,10 @@ from .nest import (
|
||||||
NestClipIE,
|
NestClipIE,
|
||||||
NestIE,
|
NestIE,
|
||||||
)
|
)
|
||||||
|
from .netapp import (
|
||||||
|
NetAppCollectionIE,
|
||||||
|
NetAppVideoIE,
|
||||||
|
)
|
||||||
from .neteasemusic import (
|
from .neteasemusic import (
|
||||||
NetEaseMusicAlbumIE,
|
NetEaseMusicAlbumIE,
|
||||||
NetEaseMusicDjRadioIE,
|
NetEaseMusicDjRadioIE,
|
||||||
|
|
@ -1364,6 +1380,7 @@ from .nova import (
|
||||||
NovaIE,
|
NovaIE,
|
||||||
)
|
)
|
||||||
from .novaplay import NovaPlayIE
|
from .novaplay import NovaPlayIE
|
||||||
|
from .nowcanal import NowCanalIE
|
||||||
from .nowness import (
|
from .nowness import (
|
||||||
NownessIE,
|
NownessIE,
|
||||||
NownessPlaylistIE,
|
NownessPlaylistIE,
|
||||||
|
|
@ -2517,6 +2534,7 @@ from .yappy import (
|
||||||
YappyIE,
|
YappyIE,
|
||||||
YappyProfileIE,
|
YappyProfileIE,
|
||||||
)
|
)
|
||||||
|
from .yfanefa import YfanefaIE
|
||||||
from .yle_areena import YleAreenaIE
|
from .yle_areena import YleAreenaIE
|
||||||
from .youjizz import YouJizzIE
|
from .youjizz import YouJizzIE
|
||||||
from .youku import (
|
from .youku import (
|
||||||
|
|
|
||||||
|
|
@ -321,6 +321,8 @@ class ABCIViewIE(InfoExtractor):
|
||||||
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
|
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
|
||||||
if formats:
|
if formats:
|
||||||
break
|
break
|
||||||
|
else:
|
||||||
|
formats = []
|
||||||
|
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
src_vtt = stream.get('captions', {}).get('src-vtt')
|
src_vtt = stream.get('captions', {}).get('src-vtt')
|
||||||
|
|
|
||||||
74
yt_dlp/extractor/bitmovin.py
Normal file
74
yt_dlp/extractor/bitmovin.py
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
|
class BitmovinIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://streams\.bitmovin\.com/(?P<id>\w+)'
|
||||||
|
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//streams\.bitmovin\.com/(?P<id>\w+)[^"\']+)']
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://streams.bitmovin.com/cqkl1t5giv3lrce7pjbg/embed',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cqkl1t5giv3lrce7pjbg',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Developing Osteopathic Residents as Faculty',
|
||||||
|
'thumbnail': 'https://streams.bitmovin.com/cqkl1t5giv3lrce7pjbg/poster',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
|
}, {
|
||||||
|
'url': 'https://streams.bitmovin.com/cgl9rh94uvs51rqc8jhg/share',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cgl9rh94uvs51rqc8jhg',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Big Buck Bunny (Streams Docs)',
|
||||||
|
'thumbnail': 'https://streams.bitmovin.com/cgl9rh94uvs51rqc8jhg/poster',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
|
}]
|
||||||
|
_WEBPAGE_TESTS = [{
|
||||||
|
# bitmovin-stream web component
|
||||||
|
'url': 'https://www.institutionalinvestor.com/article/2bsw1in1l9k68mp9kritc/video-war-stories-over-board-games/best-case-i-get-fired-war-stories',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cuiumeil6g115lc4li3g',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '[media] War Stories over Board Games: “Best Case: I Get Fired” ',
|
||||||
|
'thumbnail': 'https://streams.bitmovin.com/cuiumeil6g115lc4li3g/poster',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
|
}, {
|
||||||
|
# iframe embed
|
||||||
|
'url': 'https://www.clearblueionizer.com/en/pool-ionizers/mineral-pool-vs-saltwater-pool/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'cvpvfsm1pf7itg7cfvtg',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Pool Ionizer vs. Salt Chlorinator',
|
||||||
|
'thumbnail': 'https://streams.bitmovin.com/cvpvfsm1pf7itg7cfvtg/poster',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
|
}]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _extract_embed_urls(cls, url, webpage):
|
||||||
|
yield from super()._extract_embed_urls(url, webpage)
|
||||||
|
for stream_id in re.findall(r'<bitmovin-stream\b[^>]*\bstream-id=["\'](?P<id>\w+)', webpage):
|
||||||
|
yield f'https://streams.bitmovin.com/{stream_id}'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
player_config = self._download_json(
|
||||||
|
f'https://streams.bitmovin.com/{video_id}/config', video_id)['sources']
|
||||||
|
|
||||||
|
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||||
|
player_config['hls'], video_id, 'mp4')
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
**traverse_obj(player_config, {
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'thumbnail': ('poster', {str}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
@ -16,7 +16,7 @@ from ..utils.traversal import find_element, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class BunnyCdnIE(InfoExtractor):
|
class BunnyCdnIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:iframe\.mediadelivery\.net|video\.bunnycdn\.com)/(?:embed|play)/(?P<library_id>\d+)/(?P<id>[\da-f-]+)'
|
_VALID_URL = r'https?://(?:(?:iframe|player)\.mediadelivery\.net|video\.bunnycdn\.com)/(?:embed|play)/(?P<library_id>\d+)/(?P<id>[\da-f-]+)'
|
||||||
_EMBED_REGEX = [rf'<iframe[^>]+src=[\'"](?P<url>{_VALID_URL}[^\'"]*)[\'"]']
|
_EMBED_REGEX = [rf'<iframe[^>]+src=[\'"](?P<url>{_VALID_URL}[^\'"]*)[\'"]']
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://iframe.mediadelivery.net/embed/113933/e73edec1-e381-4c8b-ae73-717a140e0924',
|
'url': 'https://iframe.mediadelivery.net/embed/113933/e73edec1-e381-4c8b-ae73-717a140e0924',
|
||||||
|
|
@ -39,7 +39,7 @@ class BunnyCdnIE(InfoExtractor):
|
||||||
'timestamp': 1691145748,
|
'timestamp': 1691145748,
|
||||||
'thumbnail': r're:^https?://.*\.b-cdn\.net/32e34c4b-0d72-437c-9abb-05e67657da34/thumbnail_9172dc16\.jpg',
|
'thumbnail': r're:^https?://.*\.b-cdn\.net/32e34c4b-0d72-437c-9abb-05e67657da34/thumbnail_9172dc16\.jpg',
|
||||||
'duration': 106.0,
|
'duration': 106.0,
|
||||||
'description': 'md5:981a3e899a5c78352b21ed8b2f1efd81',
|
'description': 'md5:11452bcb31f379ee3eaf1234d3264e44',
|
||||||
'upload_date': '20230804',
|
'upload_date': '20230804',
|
||||||
'title': 'Sanela ist Teil der #arbeitsmarktkraft',
|
'title': 'Sanela ist Teil der #arbeitsmarktkraft',
|
||||||
},
|
},
|
||||||
|
|
@ -58,6 +58,23 @@ class BunnyCdnIE(InfoExtractor):
|
||||||
'thumbnail': r're:^https?://.*\.b-cdn\.net/2e8545ec-509d-4571-b855-4cf0235ccd75/thumbnail\.jpg',
|
'thumbnail': r're:^https?://.*\.b-cdn\.net/2e8545ec-509d-4571-b855-4cf0235ccd75/thumbnail\.jpg',
|
||||||
},
|
},
|
||||||
'params': {'skip_download': True},
|
'params': {'skip_download': True},
|
||||||
|
}, {
|
||||||
|
# Requires any Referer
|
||||||
|
'url': 'https://iframe.mediadelivery.net/embed/289162/6372f5a3-68df-4ef7-a115-e1110186c477',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '6372f5a3-68df-4ef7-a115-e1110186c477',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '12-Creating Small Asset Blockouts -Timelapse.mp4',
|
||||||
|
'description': '',
|
||||||
|
'duration': 263.0,
|
||||||
|
'timestamp': 1724485440,
|
||||||
|
'upload_date': '20240824',
|
||||||
|
'thumbnail': r're:^https?://.*\.b-cdn\.net/6372f5a3-68df-4ef7-a115-e1110186c477/thumbnail\.jpg',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': True},
|
||||||
|
}, {
|
||||||
|
'url': 'https://player.mediadelivery.net/embed/519128/875880a9-bcc2-4038-9e05-e5024bba9b70',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
_WEBPAGE_TESTS = [{
|
_WEBPAGE_TESTS = [{
|
||||||
# Stream requires Referer
|
# Stream requires Referer
|
||||||
|
|
@ -100,7 +117,7 @@ class BunnyCdnIE(InfoExtractor):
|
||||||
video_id, library_id = self._match_valid_url(url).group('id', 'library_id')
|
video_id, library_id = self._match_valid_url(url).group('id', 'library_id')
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
f'https://iframe.mediadelivery.net/embed/{library_id}/{video_id}', video_id,
|
f'https://iframe.mediadelivery.net/embed/{library_id}/{video_id}', video_id,
|
||||||
headers=traverse_obj(smuggled_data, {'Referer': 'Referer'}),
|
headers={'Referer': smuggled_data.get('Referer') or 'https://iframe.mediadelivery.net/'},
|
||||||
query=traverse_obj(parse_qs(url), {'token': 'token', 'expires': 'expires'}))
|
query=traverse_obj(parse_qs(url), {'token': 'token', 'expires': 'expires'}))
|
||||||
|
|
||||||
if html_title := self._html_extract_title(webpage, default=None) == '403':
|
if html_title := self._html_extract_title(webpage, default=None) == '403':
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import int_or_none
|
from ..utils import int_or_none, url_or_none
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class DigitekaIE(InfoExtractor):
|
class DigitekaIE(InfoExtractor):
|
||||||
|
|
@ -25,74 +26,56 @@ class DigitekaIE(InfoExtractor):
|
||||||
)/(?P<id>[\d+a-z]+)'''
|
)/(?P<id>[\d+a-z]+)'''
|
||||||
_EMBED_REGEX = [r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)']
|
_EMBED_REGEX = [r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)']
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
# news
|
'url': 'https://www.ultimedia.com/default/index/videogeneric/id/3x5x55k',
|
||||||
'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
|
|
||||||
'md5': '276a0e49de58c7e85d32b057837952a2',
|
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 's8uk0r',
|
'id': '3x5x55k',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
|
'title': 'Il est passionné de DS',
|
||||||
'thumbnail': r're:^https?://.*\.jpg',
|
'thumbnail': r're:^https?://.*\.jpg',
|
||||||
'duration': 74,
|
'duration': 89,
|
||||||
'upload_date': '20150317',
|
'upload_date': '20251012',
|
||||||
'timestamp': 1426604939,
|
'timestamp': 1760285363,
|
||||||
'uploader_id': '3fszv',
|
'uploader_id': '3pz33',
|
||||||
},
|
},
|
||||||
}, {
|
'params': {'skip_download': True},
|
||||||
# music
|
|
||||||
'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
|
|
||||||
'md5': '2ea3513813cf230605c7e2ffe7eca61c',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'xvpfp8',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'Two - C\'est La Vie (clip)',
|
|
||||||
'thumbnail': r're:^https?://.*\.jpg',
|
|
||||||
'duration': 233,
|
|
||||||
'upload_date': '20150224',
|
|
||||||
'timestamp': 1424760500,
|
|
||||||
'uploader_id': '3rfzk',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes',
|
|
||||||
'only_matching': True,
|
|
||||||
}]
|
}]
|
||||||
|
_IFRAME_MD_ID = '01836272' # One static ID working for Ultimedia iframes
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = self._match_valid_url(url)
|
video_id = self._match_id(url)
|
||||||
video_id = mobj.group('id')
|
|
||||||
video_type = mobj.group('embed_type') or mobj.group('site_type')
|
|
||||||
if video_type == 'music':
|
|
||||||
video_type = 'musique'
|
|
||||||
|
|
||||||
deliver_info = self._download_json(
|
video_info = self._download_json(
|
||||||
f'http://www.ultimedia.com/deliver/video?video={video_id}&topic={video_type}',
|
f'https://www.ultimedia.com/player/getConf/{self._IFRAME_MD_ID}/1/{video_id}', video_id,
|
||||||
video_id)
|
note='Downloading player configuration')['video']
|
||||||
|
|
||||||
yt_id = deliver_info.get('yt_id')
|
|
||||||
if yt_id:
|
|
||||||
return self.url_result(yt_id, 'Youtube')
|
|
||||||
|
|
||||||
jwconf = deliver_info['jwconf']
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for source in jwconf['playlist'][0]['sources']:
|
subtitles = {}
|
||||||
formats.append({
|
|
||||||
'url': source['file'],
|
|
||||||
'format_id': source.get('label'),
|
|
||||||
})
|
|
||||||
|
|
||||||
title = deliver_info['title']
|
if hls_url := traverse_obj(video_info, ('media_sources', 'hls', 'hls_auto', {url_or_none})):
|
||||||
thumbnail = jwconf.get('image')
|
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||||
duration = int_or_none(deliver_info.get('duration'))
|
hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
|
||||||
timestamp = int_or_none(deliver_info.get('release_time'))
|
formats.extend(fmts)
|
||||||
uploader_id = deliver_info.get('owner_id')
|
self._merge_subtitles(subs, target=subtitles)
|
||||||
|
|
||||||
|
for format_id, mp4_url in traverse_obj(video_info, ('media_sources', 'mp4', {dict.items}, ...)):
|
||||||
|
if not mp4_url:
|
||||||
|
continue
|
||||||
|
formats.append({
|
||||||
|
'url': mp4_url,
|
||||||
|
'format_id': format_id,
|
||||||
|
'height': int_or_none(format_id.partition('_')[2]),
|
||||||
|
'ext': 'mp4',
|
||||||
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
|
||||||
'thumbnail': thumbnail,
|
|
||||||
'duration': duration,
|
|
||||||
'timestamp': timestamp,
|
|
||||||
'uploader_id': uploader_id,
|
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
**traverse_obj(video_info, {
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'thumbnail': ('image', {url_or_none}),
|
||||||
|
'duration': ('duration', {int_or_none}),
|
||||||
|
'timestamp': ('creationDate', {int_or_none}),
|
||||||
|
'uploader_id': ('ownerId', {str}),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1063,7 +1063,7 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'German Gold',
|
'title': 'German Gold',
|
||||||
'description': 'md5:f3073306553a8d9b40e6ac4cdbf09fc6',
|
'description': 'md5:f3073306553a8d9b40e6ac4cdbf09fc6',
|
||||||
'display_id': 'german-gold',
|
'display_id': 'goldrausch-in-australien/german-gold',
|
||||||
'episode': 'Episode 1',
|
'episode': 'Episode 1',
|
||||||
'episode_number': 1,
|
'episode_number': 1,
|
||||||
'season': 'Season 5',
|
'season': 'Season 5',
|
||||||
|
|
@ -1112,7 +1112,7 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '24 Stunden auf der Feuerwache 3',
|
'title': '24 Stunden auf der Feuerwache 3',
|
||||||
'description': 'md5:f3084ef6170bfb79f9a6e0c030e09330',
|
'description': 'md5:f3084ef6170bfb79f9a6e0c030e09330',
|
||||||
'display_id': '24-stunden-auf-der-feuerwache-3',
|
'display_id': 'feuerwache-3-alarm-in-muenchen/24-stunden-auf-der-feuerwache-3',
|
||||||
'episode': 'Episode 1',
|
'episode': 'Episode 1',
|
||||||
'episode_number': 1,
|
'episode_number': 1,
|
||||||
'season': 'Season 1',
|
'season': 'Season 1',
|
||||||
|
|
@ -1134,7 +1134,7 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Der Poltergeist im Kostümladen',
|
'title': 'Der Poltergeist im Kostümladen',
|
||||||
'description': 'md5:20b52b9736a0a3a7873d19a238fad7fc',
|
'description': 'md5:20b52b9736a0a3a7873d19a238fad7fc',
|
||||||
'display_id': 'der-poltergeist-im-kostumladen',
|
'display_id': 'ghost-adventures/der-poltergeist-im-kostumladen',
|
||||||
'episode': 'Episode 1',
|
'episode': 'Episode 1',
|
||||||
'episode_number': 1,
|
'episode_number': 1,
|
||||||
'season': 'Season 25',
|
'season': 'Season 25',
|
||||||
|
|
@ -1156,7 +1156,7 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Das Geheimnis meines Bruders',
|
'title': 'Das Geheimnis meines Bruders',
|
||||||
'description': 'md5:3167550bb582eb9c92875c86a0a20882',
|
'description': 'md5:3167550bb582eb9c92875c86a0a20882',
|
||||||
'display_id': 'das-geheimnis-meines-bruders',
|
'display_id': 'evil-gesichter-des-boesen/das-geheimnis-meines-bruders',
|
||||||
'episode': 'Episode 1',
|
'episode': 'Episode 1',
|
||||||
'episode_number': 1,
|
'episode_number': 1,
|
||||||
'season': 'Season 1',
|
'season': 'Season 1',
|
||||||
|
|
@ -1175,18 +1175,19 @@ class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
domain, programme, alternate_id = self._match_valid_url(url).groups()
|
domain, programme, alternate_id = self._match_valid_url(url).groups()
|
||||||
|
display_id = f'{programme}/{alternate_id}'
|
||||||
meta = self._download_json(
|
meta = self._download_json(
|
||||||
f'https://de-api.loma-cms.com/feloma/videos/{alternate_id}/',
|
f'https://de-api.loma-cms.com/feloma/videos/{alternate_id}/',
|
||||||
alternate_id, query={
|
display_id, query={
|
||||||
'environment': domain.split('.')[0],
|
'environment': domain.split('.')[0],
|
||||||
'v': '2',
|
'v': '2',
|
||||||
'filter[show.slug]': programme,
|
'filter[show.slug]': programme,
|
||||||
}, fatal=False)
|
}, fatal=False)
|
||||||
video_id = traverse_obj(meta, ('uid', {str}, {lambda s: s[-7:]})) or alternate_id
|
video_id = traverse_obj(meta, ('uid', {str}, {lambda s: s[-7:]})) or display_id
|
||||||
|
|
||||||
disco_api_info = self._get_disco_api_info(
|
disco_api_info = self._get_disco_api_info(
|
||||||
url, video_id, 'eu1-prod.disco-api.com', domain.replace('.', ''), 'DE')
|
url, video_id, 'eu1-prod.disco-api.com', domain.replace('.', ''), 'DE')
|
||||||
disco_api_info['display_id'] = alternate_id
|
disco_api_info['display_id'] = display_id
|
||||||
disco_api_info['categories'] = traverse_obj(meta, (
|
disco_api_info['categories'] = traverse_obj(meta, (
|
||||||
'taxonomies', lambda _, v: v['category'] == 'genre', 'title', {str.strip}, filter, all, filter))
|
'taxonomies', lambda _, v: v['category'] == 'genre', 'title', {str.strip}, filter, all, filter))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ from ..utils import (
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
)
|
)
|
||||||
from ..utils.traversal import traverse_obj
|
from ..utils.traversal import require, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class FirstTVIE(InfoExtractor):
|
class FirstTVIE(InfoExtractor):
|
||||||
|
|
@ -129,3 +129,36 @@ class FirstTVIE(InfoExtractor):
|
||||||
return self.playlist_result(
|
return self.playlist_result(
|
||||||
self._entries(items), display_id, self._og_search_title(webpage, default=None),
|
self._entries(items), display_id, self._og_search_title(webpage, default=None),
|
||||||
thumbnail=self._og_search_thumbnail(webpage, default=None))
|
thumbnail=self._og_search_thumbnail(webpage, default=None))
|
||||||
|
|
||||||
|
|
||||||
|
class FirstTVLiveIE(InfoExtractor):
|
||||||
|
IE_NAME = '1tv:live'
|
||||||
|
IE_DESC = 'Первый канал (прямой эфир)'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?1tv\.ru/live'
|
||||||
|
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.1tv.ru/live',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'live',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': r're:ПЕРВЫЙ КАНАЛ ПРЯМОЙ ЭФИР СМОТРЕТЬ ОНЛАЙН \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
|
||||||
|
'live_status': 'is_live',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'livestream'},
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
display_id = 'live'
|
||||||
|
webpage = self._download_webpage(url, display_id, fatal=False)
|
||||||
|
|
||||||
|
streams_list = self._download_json('https://stream.1tv.ru/api/playlist/1tvch-v1_as_array.json', display_id)
|
||||||
|
mpd_url = traverse_obj(streams_list, ('mpd', ..., {url_or_none}, any, {require('mpd url')}))
|
||||||
|
# FFmpeg needs to be passed -re to not seek past live window. This is handled by core
|
||||||
|
formats, _ = self._extract_mpd_formats_and_subtitles(mpd_url, display_id, mpd_id='dash')
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': display_id,
|
||||||
|
'title': self._html_extract_title(webpage),
|
||||||
|
'formats': formats,
|
||||||
|
'is_live': True,
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,15 +6,15 @@ from ..utils import (
|
||||||
OnDemandPagedList,
|
OnDemandPagedList,
|
||||||
clean_html,
|
clean_html,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
|
float_or_none,
|
||||||
format_field,
|
format_field,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
join_nonempty,
|
join_nonempty,
|
||||||
parse_codecs,
|
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
url_or_none,
|
url_or_none,
|
||||||
urljoin,
|
urljoin,
|
||||||
)
|
)
|
||||||
from ..utils.traversal import traverse_obj
|
from ..utils.traversal import require, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class FloatplaneBaseIE(InfoExtractor):
|
class FloatplaneBaseIE(InfoExtractor):
|
||||||
|
|
@ -50,37 +50,31 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||||
media_id = media['id']
|
media_id = media['id']
|
||||||
media_typ = media.get('type') or 'video'
|
media_typ = media.get('type') or 'video'
|
||||||
|
|
||||||
metadata = self._download_json(
|
|
||||||
f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id, query={'id': media_id},
|
|
||||||
note=f'Downloading {media_typ} metadata', impersonate=self._IMPERSONATE_TARGET)
|
|
||||||
|
|
||||||
stream = self._download_json(
|
stream = self._download_json(
|
||||||
f'{self._BASE_URL}/api/v2/cdn/delivery', media_id, query={
|
f'{self._BASE_URL}/api/v3/delivery/info', media_id,
|
||||||
'type': 'vod' if media_typ == 'video' else 'aod',
|
query={'scenario': 'onDemand', 'entityId': media_id},
|
||||||
'guid': metadata['guid'],
|
note=f'Downloading {media_typ} stream data',
|
||||||
}, note=f'Downloading {media_typ} stream data',
|
|
||||||
impersonate=self._IMPERSONATE_TARGET)
|
impersonate=self._IMPERSONATE_TARGET)
|
||||||
|
|
||||||
path_template = traverse_obj(stream, ('resource', 'uri', {str}))
|
metadata = self._download_json(
|
||||||
|
f'{self._BASE_URL}/api/v3/content/{media_typ}', media_id,
|
||||||
|
f'Downloading {media_typ} metadata', query={'id': media_id},
|
||||||
|
fatal=False, impersonate=self._IMPERSONATE_TARGET)
|
||||||
|
|
||||||
def format_path(params):
|
cdn_base_url = traverse_obj(stream, (
|
||||||
path = path_template
|
'groups', 0, 'origins', ..., 'url', {url_or_none}, any, {require('cdn base url')}))
|
||||||
for i, val in (params or {}).items():
|
|
||||||
path = path.replace(f'{{qualityLevelParams.{i}}}', val)
|
|
||||||
return path
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for quality in traverse_obj(stream, ('resource', 'data', 'qualityLevels', ...)):
|
for variant in traverse_obj(stream, ('groups', 0, 'variants', lambda _, v: v['url'])):
|
||||||
url = urljoin(stream['cdn'], format_path(traverse_obj(
|
format_url = urljoin(cdn_base_url, variant['url'])
|
||||||
stream, ('resource', 'data', 'qualityLevelParams', quality['name'], {dict}))))
|
format_id = traverse_obj(variant, ('name', {str}))
|
||||||
format_id = traverse_obj(quality, ('name', {str}))
|
|
||||||
hls_aes = {}
|
hls_aes = {}
|
||||||
m3u8_data = None
|
m3u8_data = None
|
||||||
|
|
||||||
# If we need impersonation for the API, then we need it for HLS keys too: extract in advance
|
# If we need impersonation for the API, then we need it for HLS keys too: extract in advance
|
||||||
if self._IMPERSONATE_TARGET is not None:
|
if self._IMPERSONATE_TARGET is not None:
|
||||||
m3u8_data = self._download_webpage(
|
m3u8_data = self._download_webpage(
|
||||||
url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
|
format_url, media_id, fatal=False, impersonate=self._IMPERSONATE_TARGET, headers=self._HEADERS,
|
||||||
note=join_nonempty('Downloading', format_id, 'm3u8 information', delim=' '),
|
note=join_nonempty('Downloading', format_id, 'm3u8 information', delim=' '),
|
||||||
errnote=join_nonempty('Failed to download', format_id, 'm3u8 information', delim=' '))
|
errnote=join_nonempty('Failed to download', format_id, 'm3u8 information', delim=' '))
|
||||||
if not m3u8_data:
|
if not m3u8_data:
|
||||||
|
|
@ -98,18 +92,34 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||||
hls_aes['key'] = urlh.read().hex()
|
hls_aes['key'] = urlh.read().hex()
|
||||||
|
|
||||||
formats.append({
|
formats.append({
|
||||||
**traverse_obj(quality, {
|
**traverse_obj(variant, {
|
||||||
'format_note': ('label', {str}),
|
'format_note': ('label', {str}),
|
||||||
'width': ('width', {int}),
|
'width': ('meta', 'video', 'width', {int_or_none}),
|
||||||
'height': ('height', {int}),
|
'height': ('meta', 'video', 'height', {int_or_none}),
|
||||||
|
'vcodec': ('meta', 'video', 'codec', {str}),
|
||||||
|
'acodec': ('meta', 'audio', 'codec', {str}),
|
||||||
|
'vbr': ('meta', 'video', 'bitrate', 'average', {int_or_none(scale=1000)}),
|
||||||
|
'abr': ('meta', 'audio', 'bitrate', 'average', {int_or_none(scale=1000)}),
|
||||||
|
'audio_channels': ('meta', 'audio', 'channelCount', {int_or_none}),
|
||||||
|
'fps': ('meta', 'video', 'fps', {float_or_none}),
|
||||||
}),
|
}),
|
||||||
**parse_codecs(quality.get('codecs')),
|
'url': format_url,
|
||||||
'url': url,
|
'ext': determine_ext(format_url.partition('/chunk.m3u8')[0], 'mp4'),
|
||||||
'ext': determine_ext(url.partition('/chunk.m3u8')[0], 'mp4'),
|
|
||||||
'format_id': format_id,
|
'format_id': format_id,
|
||||||
'hls_media_playlist_data': m3u8_data,
|
'hls_media_playlist_data': m3u8_data,
|
||||||
'hls_aes': hls_aes or None,
|
'hls_aes': hls_aes or None,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
automatic_captions = {}
|
||||||
|
for sub_data in traverse_obj(metadata, ('textTracks', lambda _, v: url_or_none(v['src']))):
|
||||||
|
sub_lang = sub_data.get('language') or 'en'
|
||||||
|
sub_entry = {'url': sub_data['src']}
|
||||||
|
if sub_data.get('generated'):
|
||||||
|
automatic_captions.setdefault(sub_lang, []).append(sub_entry)
|
||||||
|
else:
|
||||||
|
subtitles.setdefault(sub_lang, []).append(sub_entry)
|
||||||
|
|
||||||
items.append({
|
items.append({
|
||||||
**common_info,
|
**common_info,
|
||||||
'id': media_id,
|
'id': media_id,
|
||||||
|
|
@ -119,6 +129,8 @@ class FloatplaneBaseIE(InfoExtractor):
|
||||||
'thumbnail': ('thumbnail', 'path', {url_or_none}),
|
'thumbnail': ('thumbnail', 'path', {url_or_none}),
|
||||||
}),
|
}),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
'automatic_captions': automatic_captions,
|
||||||
})
|
})
|
||||||
|
|
||||||
post_info = {
|
post_info = {
|
||||||
|
|
|
||||||
164
yt_dlp/extractor/frontro.py
Normal file
164
yt_dlp/extractor/frontro.py
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import int_or_none, parse_iso8601, url_or_none
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
|
class FrontoBaseIE(InfoExtractor):
|
||||||
|
def _get_auth_headers(self, url):
|
||||||
|
return traverse_obj(self._get_cookies(url), {
|
||||||
|
'authorization': ('frAccessToken', 'value', {lambda token: f'Bearer {token}' if token else None}),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
class FrontroVideoBaseIE(FrontoBaseIE):
|
||||||
|
_CHANNEL_ID = None
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
metadata = self._download_json(
|
||||||
|
'https://api.frontrow.cc/query', video_id, data=json.dumps({
|
||||||
|
'operationName': 'Video',
|
||||||
|
'variables': {'channelID': self._CHANNEL_ID, 'videoID': video_id},
|
||||||
|
'query': '''query Video($channelID: ID!, $videoID: ID!) {
|
||||||
|
video(ChannelID: $channelID, VideoID: $videoID) {
|
||||||
|
... on Video {title description updatedAt thumbnail createdAt duration likeCount comments views url hasAccess}
|
||||||
|
}
|
||||||
|
}''',
|
||||||
|
}).encode(), headers={
|
||||||
|
'content-type': 'application/json',
|
||||||
|
**self._get_auth_headers(url),
|
||||||
|
})['data']['video']
|
||||||
|
if not traverse_obj(metadata, 'hasAccess'):
|
||||||
|
self.raise_login_required()
|
||||||
|
|
||||||
|
formats, subtitles = self._extract_m3u8_formats_and_subtitles(metadata['url'], video_id)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
**traverse_obj(metadata, {
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'description': ('description', {str}),
|
||||||
|
'thumbnail': ('thumbnail', {url_or_none}),
|
||||||
|
'timestamp': ('createdAt', {parse_iso8601}),
|
||||||
|
'modified_timestamp': ('updatedAt', {parse_iso8601}),
|
||||||
|
'duration': ('duration', {int_or_none}),
|
||||||
|
'like_count': ('likeCount', {int_or_none}),
|
||||||
|
'comment_count': ('comments', {int_or_none}),
|
||||||
|
'view_count': ('views', {int_or_none}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FrontroGroupBaseIE(FrontoBaseIE):
|
||||||
|
_CHANNEL_ID = None
|
||||||
|
_VIDEO_EXTRACTOR = None
|
||||||
|
_VIDEO_URL_TMPL = None
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
group_id = self._match_id(url)
|
||||||
|
|
||||||
|
metadata = self._download_json(
|
||||||
|
'https://api.frontrow.cc/query', group_id, note='Downloading playlist metadata',
|
||||||
|
data=json.dumps({
|
||||||
|
'operationName': 'PaginatedStaticPageContainer',
|
||||||
|
'variables': {'channelID': self._CHANNEL_ID, 'first': 500, 'pageContainerID': group_id},
|
||||||
|
'query': '''query PaginatedStaticPageContainer($channelID: ID!, $pageContainerID: ID!) {
|
||||||
|
pageContainer(ChannelID: $channelID, PageContainerID: $pageContainerID) {
|
||||||
|
... on StaticPageContainer { id title updatedAt createdAt itemRefs {edges {node {
|
||||||
|
id contentItem { ... on ItemVideo { videoItem: item {
|
||||||
|
id
|
||||||
|
}}}
|
||||||
|
}}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}''',
|
||||||
|
}).encode(), headers={
|
||||||
|
'content-type': 'application/json',
|
||||||
|
**self._get_auth_headers(url),
|
||||||
|
})['data']['pageContainer']
|
||||||
|
|
||||||
|
entries = []
|
||||||
|
for video_id in traverse_obj(metadata, (
|
||||||
|
'itemRefs', 'edges', ..., 'node', 'contentItem', 'videoItem', 'id', {str}),
|
||||||
|
):
|
||||||
|
entries.append(self.url_result(
|
||||||
|
self._VIDEO_URL_TMPL % video_id, self._VIDEO_EXTRACTOR, video_id))
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'playlist',
|
||||||
|
'id': group_id,
|
||||||
|
'entries': entries,
|
||||||
|
**traverse_obj(metadata, {
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'timestamp': ('createdAt', {parse_iso8601}),
|
||||||
|
'modified_timestamp': ('updatedAt', {parse_iso8601}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TheChosenIE(FrontroVideoBaseIE):
|
||||||
|
_CHANNEL_ID = '12884901895'
|
||||||
|
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/video/(?P<id>[0-9]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://watch.thechosen.tv/video/184683594325',
|
||||||
|
'md5': '3f878b689588c71b38ec9943c54ff5b0',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '184683594325',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Season 3 Episode 2: Two by Two',
|
||||||
|
'description': 'md5:174c373756ecc8df46b403f4fcfbaf8c',
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'duration': 4212,
|
||||||
|
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683594325/',
|
||||||
|
'timestamp': 1698954546,
|
||||||
|
'upload_date': '20231102',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'modified_date': str,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://watch.thechosen.tv/video/184683596189',
|
||||||
|
'md5': 'd581562f9d29ce82f5b7770415334151',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '184683596189',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Season 4 Episode 8: Humble',
|
||||||
|
'description': 'md5:20a57bead43da1cf77cd5b0fe29bbc76',
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'duration': 5092,
|
||||||
|
'thumbnail': r're:https://fastly\.frontrowcdn\.com/channels/12884901895/VIDEO_THUMBNAIL/184683596189/',
|
||||||
|
'timestamp': 1715019474,
|
||||||
|
'upload_date': '20240506',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'modified_date': str,
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
class TheChosenGroupIE(FrontroGroupBaseIE):
|
||||||
|
_CHANNEL_ID = '12884901895'
|
||||||
|
_VIDEO_EXTRACTOR = TheChosenIE
|
||||||
|
_VIDEO_URL_TMPL = 'https://watch.thechosen.tv/video/%s'
|
||||||
|
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?watch\.thechosen\.tv/group/(?P<id>[0-9]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://watch.thechosen.tv/group/309237658592',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '309237658592',
|
||||||
|
'title': 'Season 3',
|
||||||
|
'timestamp': 1746203969,
|
||||||
|
'upload_date': '20250502',
|
||||||
|
'modified_timestamp': int,
|
||||||
|
'modified_date': str,
|
||||||
|
},
|
||||||
|
'playlist_count': 8,
|
||||||
|
}]
|
||||||
|
|
@ -13,12 +13,14 @@ from ..utils.traversal import get_first, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class GoPlayIE(InfoExtractor):
|
class GoPlayIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
|
IE_NAME = 'play.tv'
|
||||||
|
IE_DESC = 'PLAY (formerly goplay.be)'
|
||||||
|
_VALID_URL = r'https?://(www\.)?play\.tv/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
|
||||||
|
|
||||||
_NETRC_MACHINE = 'goplay'
|
_NETRC_MACHINE = 'goplay'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://www.goplay.be/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
|
'url': 'https://www.play.tv/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '2baa4560-87a0-421b-bffc-359914e3c387',
|
'id': '2baa4560-87a0-421b-bffc-359914e3c387',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
|
@ -33,7 +35,7 @@ class GoPlayIE(InfoExtractor):
|
||||||
'params': {'skip_download': True},
|
'params': {'skip_download': True},
|
||||||
'skip': 'This video is only available for registered users',
|
'skip': 'This video is only available for registered users',
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.goplay.be/video/1917',
|
'url': 'https://www.play.tv/video/1917',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
|
'id': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
|
@ -43,7 +45,7 @@ class GoPlayIE(InfoExtractor):
|
||||||
'params': {'skip_download': True},
|
'params': {'skip_download': True},
|
||||||
'skip': 'This video is only available for registered users',
|
'skip': 'This video is only available for registered users',
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
|
'url': 'https://www.play.tv/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
|
'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
|
@ -101,7 +103,7 @@ class GoPlayIE(InfoExtractor):
|
||||||
break
|
break
|
||||||
|
|
||||||
api = self._download_json(
|
api = self._download_json(
|
||||||
f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',
|
f'https://api.play.tv/web/v1/videos/long-form/{video_id}',
|
||||||
video_id, headers={
|
video_id, headers={
|
||||||
'Authorization': f'Bearer {self._id_token}',
|
'Authorization': f'Bearer {self._id_token}',
|
||||||
**self.geo_verification_headers(),
|
**self.geo_verification_headers(),
|
||||||
|
|
|
||||||
|
|
@ -98,7 +98,7 @@ class JTBCIE(InfoExtractor):
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for stream_url in traverse_obj(playback_data, ('sources', 'HLS', ..., 'file', {url_or_none})):
|
for stream_url in traverse_obj(playback_data, ('sources', 'HLS', ..., 'file', {url_or_none})):
|
||||||
stream_url = re.sub(r'/playlist(?:_pd\d+)?\.m3u8', '/index.m3u8', stream_url)
|
stream_url = re.sub(r'/playlist_pd\d+\.m3u8', '/playlist.m3u8', stream_url)
|
||||||
formats.extend(self._extract_m3u8_formats(stream_url, video_id, fatal=False))
|
formats.extend(self._extract_m3u8_formats(stream_url, video_id, fatal=False))
|
||||||
|
|
||||||
metadata = self._download_json(
|
metadata = self._download_json(
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
import re
|
import functools
|
||||||
|
import math
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
InAdvancePagedList,
|
||||||
clean_html,
|
clean_html,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
|
@ -10,15 +12,64 @@ from ..utils import (
|
||||||
from ..utils.traversal import require, traverse_obj
|
from ..utils.traversal import require, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class MaveIE(InfoExtractor):
|
class MaveBaseIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?P<channel>[\w-]+)\.mave\.digital/(?P<id>ep-\d+)'
|
_API_BASE_URL = 'https://api.mave.digital/v1/website'
|
||||||
|
_API_BASE_STORAGE_URL = 'https://store.cloud.mts.ru/mave/'
|
||||||
|
|
||||||
|
def _load_channel_meta(self, channel_id, display_id):
|
||||||
|
return traverse_obj(self._download_json(
|
||||||
|
f'{self._API_BASE_URL}/{channel_id}/', display_id,
|
||||||
|
note='Downloading channel metadata'), 'podcast')
|
||||||
|
|
||||||
|
def _load_episode_meta(self, channel_id, episode_code, display_id):
|
||||||
|
return self._download_json(
|
||||||
|
f'{self._API_BASE_URL}/{channel_id}/episodes/{episode_code}',
|
||||||
|
display_id, note='Downloading episode metadata')
|
||||||
|
|
||||||
|
def _create_entry(self, channel_id, channel_meta, episode_meta):
|
||||||
|
episode_code = traverse_obj(episode_meta, ('code', {int}, {require('episode code')}))
|
||||||
|
return {
|
||||||
|
'display_id': f'{channel_id}-{episode_code}',
|
||||||
|
'extractor_key': MaveIE.ie_key(),
|
||||||
|
'extractor': MaveIE.IE_NAME,
|
||||||
|
'webpage_url': f'https://{channel_id}.mave.digital/ep-{episode_code}',
|
||||||
|
'channel_id': channel_id,
|
||||||
|
'channel_url': f'https://{channel_id}.mave.digital/',
|
||||||
|
'vcodec': 'none',
|
||||||
|
**traverse_obj(episode_meta, {
|
||||||
|
'id': ('id', {str}),
|
||||||
|
'url': ('audio', {urljoin(self._API_BASE_STORAGE_URL)}),
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'description': ('description', {clean_html}),
|
||||||
|
'thumbnail': ('image', {urljoin(self._API_BASE_STORAGE_URL)}),
|
||||||
|
'duration': ('duration', {int_or_none}),
|
||||||
|
'season_number': ('season', {int_or_none}),
|
||||||
|
'episode_number': ('number', {int_or_none}),
|
||||||
|
'view_count': ('listenings', {int_or_none}),
|
||||||
|
'like_count': ('reactions', lambda _, v: v['type'] == 'like', 'count', {int_or_none}, any),
|
||||||
|
'dislike_count': ('reactions', lambda _, v: v['type'] == 'dislike', 'count', {int_or_none}, any),
|
||||||
|
'age_limit': ('is_explicit', {bool}, {lambda x: 18 if x else None}),
|
||||||
|
'timestamp': ('publish_date', {parse_iso8601}),
|
||||||
|
}),
|
||||||
|
**traverse_obj(channel_meta, {
|
||||||
|
'series_id': ('id', {str}),
|
||||||
|
'series': ('title', {str}),
|
||||||
|
'channel': ('title', {str}),
|
||||||
|
'uploader': ('author', {str}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MaveIE(MaveBaseIE):
|
||||||
|
IE_NAME = 'mave'
|
||||||
|
_VALID_URL = r'https?://(?P<channel_id>[\w-]+)\.mave\.digital/ep-(?P<episode_code>\d+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://ochenlichnoe.mave.digital/ep-25',
|
'url': 'https://ochenlichnoe.mave.digital/ep-25',
|
||||||
'md5': 'aa3e513ef588b4366df1520657cbc10c',
|
'md5': 'aa3e513ef588b4366df1520657cbc10c',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '4035f587-914b-44b6-aa5a-d76685ad9bc2',
|
'id': '4035f587-914b-44b6-aa5a-d76685ad9bc2',
|
||||||
'ext': 'mp3',
|
'ext': 'mp3',
|
||||||
'display_id': 'ochenlichnoe-ep-25',
|
'display_id': 'ochenlichnoe-25',
|
||||||
'title': 'Между мной и миром: психология самооценки',
|
'title': 'Между мной и миром: психология самооценки',
|
||||||
'description': 'md5:4b7463baaccb6982f326bce5c700382a',
|
'description': 'md5:4b7463baaccb6982f326bce5c700382a',
|
||||||
'uploader': 'Самарский университет',
|
'uploader': 'Самарский университет',
|
||||||
|
|
@ -45,7 +96,7 @@ class MaveIE(InfoExtractor):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '41898bb5-ff57-4797-9236-37a8e537aa21',
|
'id': '41898bb5-ff57-4797-9236-37a8e537aa21',
|
||||||
'ext': 'mp3',
|
'ext': 'mp3',
|
||||||
'display_id': 'budem-ep-12',
|
'display_id': 'budem-12',
|
||||||
'title': 'Екатерина Михайлова: "Горе от ума" не про женщин написана',
|
'title': 'Екатерина Михайлова: "Горе от ума" не про женщин написана',
|
||||||
'description': 'md5:fa3bdd59ee829dfaf16e3efcb13f1d19',
|
'description': 'md5:fa3bdd59ee829dfaf16e3efcb13f1d19',
|
||||||
'uploader': 'Полина Цветкова+Евгения Акопова',
|
'uploader': 'Полина Цветкова+Евгения Акопова',
|
||||||
|
|
@ -68,40 +119,72 @@ class MaveIE(InfoExtractor):
|
||||||
'upload_date': '20241230',
|
'upload_date': '20241230',
|
||||||
},
|
},
|
||||||
}]
|
}]
|
||||||
_API_BASE_URL = 'https://api.mave.digital/'
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
channel_id, slug = self._match_valid_url(url).group('channel', 'id')
|
channel_id, episode_code = self._match_valid_url(url).group(
|
||||||
display_id = f'{channel_id}-{slug}'
|
'channel_id', 'episode_code')
|
||||||
webpage = self._download_webpage(url, display_id)
|
display_id = f'{channel_id}-{episode_code}'
|
||||||
data = traverse_obj(
|
|
||||||
self._search_nuxt_json(webpage, display_id),
|
channel_meta = self._load_channel_meta(channel_id, display_id)
|
||||||
('data', lambda _, v: v['activeEpisodeData'], any, {require('podcast data')}))
|
episode_meta = self._load_episode_meta(channel_id, episode_code, display_id)
|
||||||
|
|
||||||
|
return self._create_entry(channel_id, channel_meta, episode_meta)
|
||||||
|
|
||||||
|
|
||||||
|
class MaveChannelIE(MaveBaseIE):
|
||||||
|
IE_NAME = 'mave:channel'
|
||||||
|
_VALID_URL = r'https?://(?P<id>[\w-]+)\.mave\.digital/?(?:$|[?#])'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://budem.mave.digital/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'budem',
|
||||||
|
'title': 'Все там будем',
|
||||||
|
'description': 'md5:f04ae12a42be0f1d765c5e326b41987a',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 15,
|
||||||
|
}, {
|
||||||
|
'url': 'https://ochenlichnoe.mave.digital/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'ochenlichnoe',
|
||||||
|
'title': 'Очень личное',
|
||||||
|
'description': 'md5:ee36a6a52546b91b487fe08c552fdbb2',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 20,
|
||||||
|
}, {
|
||||||
|
'url': 'https://geekcity.mave.digital/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'geekcity',
|
||||||
|
'title': 'Мужчины в трико',
|
||||||
|
'description': 'md5:4164d425d60a0d97abdce9d1f6f8e049',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 80,
|
||||||
|
}]
|
||||||
|
_PAGE_SIZE = 50
|
||||||
|
|
||||||
|
def _entries(self, channel_id, channel_meta, page_num):
|
||||||
|
page_data = self._download_json(
|
||||||
|
f'{self._API_BASE_URL}/{channel_id}/episodes', channel_id, query={
|
||||||
|
'view': 'all',
|
||||||
|
'page': page_num + 1,
|
||||||
|
'sort': 'newest',
|
||||||
|
'format': 'all',
|
||||||
|
}, note=f'Downloading page {page_num + 1}')
|
||||||
|
for ep in traverse_obj(page_data, ('episodes', lambda _, v: v['audio'] and v['id'])):
|
||||||
|
yield self._create_entry(channel_id, channel_meta, ep)
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
channel_id = self._match_id(url)
|
||||||
|
|
||||||
|
channel_meta = self._load_channel_meta(channel_id, channel_id)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'display_id': display_id,
|
'_type': 'playlist',
|
||||||
'channel_id': channel_id,
|
'id': channel_id,
|
||||||
'channel_url': f'https://{channel_id}.mave.digital/',
|
**traverse_obj(channel_meta, {
|
||||||
'vcodec': 'none',
|
|
||||||
'thumbnail': re.sub(r'_\d+(?=\.(?:jpg|png))', '', self._og_search_thumbnail(webpage, default='')) or None,
|
|
||||||
**traverse_obj(data, ('activeEpisodeData', {
|
|
||||||
'url': ('audio', {urljoin(self._API_BASE_URL)}),
|
|
||||||
'id': ('id', {str}),
|
|
||||||
'title': ('title', {str}),
|
'title': ('title', {str}),
|
||||||
'description': ('description', {clean_html}),
|
'description': ('description', {str}),
|
||||||
'duration': ('duration', {int_or_none}),
|
}),
|
||||||
'season_number': ('season', {int_or_none}),
|
'entries': InAdvancePagedList(
|
||||||
'episode_number': ('number', {int_or_none}),
|
functools.partial(self._entries, channel_id, channel_meta),
|
||||||
'view_count': ('listenings', {int_or_none}),
|
math.ceil(channel_meta['episodes_count'] / self._PAGE_SIZE), self._PAGE_SIZE),
|
||||||
'like_count': ('reactions', lambda _, v: v['type'] == 'like', 'count', {int_or_none}, any),
|
|
||||||
'dislike_count': ('reactions', lambda _, v: v['type'] == 'dislike', 'count', {int_or_none}, any),
|
|
||||||
'age_limit': ('is_explicit', {bool}, {lambda x: 18 if x else None}),
|
|
||||||
'timestamp': ('publish_date', {parse_iso8601}),
|
|
||||||
})),
|
|
||||||
**traverse_obj(data, ('podcast', 'podcast', {
|
|
||||||
'series_id': ('id', {str}),
|
|
||||||
'series': ('title', {str}),
|
|
||||||
'channel': ('title', {str}),
|
|
||||||
'uploader': ('author', {str}),
|
|
||||||
})),
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,9 @@
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
|
||||||
float_or_none,
|
|
||||||
format_field,
|
|
||||||
int_or_none,
|
int_or_none,
|
||||||
str_or_none,
|
url_or_none,
|
||||||
traverse_obj,
|
|
||||||
)
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class MedalTVIE(InfoExtractor):
|
class MedalTVIE(InfoExtractor):
|
||||||
|
|
@ -30,25 +25,8 @@ class MedalTVIE(InfoExtractor):
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
'like_count': int,
|
'like_count': int,
|
||||||
'duration': 13,
|
'duration': 13,
|
||||||
},
|
'thumbnail': r're:https://cdn\.medal\.tv/ugcp/content-thumbnail/.*\.jpg',
|
||||||
}, {
|
'tags': ['headshot', 'valorant', '4k', 'clutch', 'mornu'],
|
||||||
'url': 'https://medal.tv/games/cod-cold-war/clips/2mA60jWAGQCBH',
|
|
||||||
'md5': 'fc7a3e4552ae8993c1c4006db46be447',
|
|
||||||
'info_dict': {
|
|
||||||
'id': '2mA60jWAGQCBH',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'Quad Cold',
|
|
||||||
'description': 'Medal,https://medal.tv/desktop/',
|
|
||||||
'uploader': 'MowgliSB',
|
|
||||||
'timestamp': 1603165266,
|
|
||||||
'upload_date': '20201020',
|
|
||||||
'uploader_id': '10619174',
|
|
||||||
'thumbnail': 'https://cdn.medal.tv/10619174/thumbnail-34934644-720p.jpg?t=1080p&c=202042&missing',
|
|
||||||
'uploader_url': 'https://medal.tv/users/10619174',
|
|
||||||
'comment_count': int,
|
|
||||||
'view_count': int,
|
|
||||||
'like_count': int,
|
|
||||||
'duration': 23,
|
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://medal.tv/games/cod-cold-war/clips/2um24TWdty0NA',
|
'url': 'https://medal.tv/games/cod-cold-war/clips/2um24TWdty0NA',
|
||||||
|
|
@ -57,12 +35,12 @@ class MedalTVIE(InfoExtractor):
|
||||||
'id': '2um24TWdty0NA',
|
'id': '2um24TWdty0NA',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'u tk me i tk u bigger',
|
'title': 'u tk me i tk u bigger',
|
||||||
'description': 'Medal,https://medal.tv/desktop/',
|
'description': '',
|
||||||
'uploader': 'Mimicc',
|
'uploader': 'zahl',
|
||||||
'timestamp': 1605580939,
|
'timestamp': 1605580939,
|
||||||
'upload_date': '20201117',
|
'upload_date': '20201117',
|
||||||
'uploader_id': '5156321',
|
'uploader_id': '5156321',
|
||||||
'thumbnail': 'https://cdn.medal.tv/5156321/thumbnail-36787208-360p.jpg?t=1080p&c=202046&missing',
|
'thumbnail': r're:https://cdn\.medal\.tv/source/.*\.png',
|
||||||
'uploader_url': 'https://medal.tv/users/5156321',
|
'uploader_url': 'https://medal.tv/users/5156321',
|
||||||
'comment_count': int,
|
'comment_count': int,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
|
|
@ -70,91 +48,77 @@ class MedalTVIE(InfoExtractor):
|
||||||
'duration': 9,
|
'duration': 9,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://medal.tv/games/valorant/clips/37rMeFpryCC-9',
|
# API requires auth
|
||||||
'only_matching': True,
|
|
||||||
}, {
|
|
||||||
'url': 'https://medal.tv/games/valorant/clips/2WRj40tpY_EU9',
|
'url': 'https://medal.tv/games/valorant/clips/2WRj40tpY_EU9',
|
||||||
|
'md5': '6c6bb6569777fd8b4ef7b33c09de8dcf',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '2WRj40tpY_EU9',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '1v5 clutch',
|
||||||
|
'description': '',
|
||||||
|
'uploader': 'adny',
|
||||||
|
'uploader_id': '6256941',
|
||||||
|
'uploader_url': 'https://medal.tv/users/6256941',
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'duration': 25,
|
||||||
|
'thumbnail': r're:https://cdn\.medal\.tv/source/.*\.jpg',
|
||||||
|
'timestamp': 1612896680,
|
||||||
|
'upload_date': '20210209',
|
||||||
|
},
|
||||||
|
'expected_warnings': ['Video formats are not available through API'],
|
||||||
|
}, {
|
||||||
|
'url': 'https://medal.tv/games/valorant/clips/37rMeFpryCC-9',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id, query={'mobilebypass': 'true'})
|
content_data = self._download_json(
|
||||||
|
f'https://medal.tv/api/content/{video_id}', video_id,
|
||||||
hydration_data = self._search_json(
|
headers={'Accept': 'application/json'})
|
||||||
r'<script[^>]*>[^<]*\bhydrationData\s*=', webpage,
|
|
||||||
'next data', video_id, end_pattern='</script>', fatal=False)
|
|
||||||
|
|
||||||
clip = traverse_obj(hydration_data, ('clips', ...), get_all=False)
|
|
||||||
if not clip:
|
|
||||||
raise ExtractorError(
|
|
||||||
'Could not find video information.', video_id=video_id)
|
|
||||||
|
|
||||||
title = clip['contentTitle']
|
|
||||||
|
|
||||||
source_width = int_or_none(clip.get('sourceWidth'))
|
|
||||||
source_height = int_or_none(clip.get('sourceHeight'))
|
|
||||||
|
|
||||||
aspect_ratio = source_width / source_height if source_width and source_height else 16 / 9
|
|
||||||
|
|
||||||
def add_item(container, item_url, height, id_key='format_id', item_id=None):
|
|
||||||
item_id = item_id or '%dp' % height
|
|
||||||
if item_id not in item_url:
|
|
||||||
return
|
|
||||||
container.append({
|
|
||||||
'url': item_url,
|
|
||||||
id_key: item_id,
|
|
||||||
'width': round(aspect_ratio * height),
|
|
||||||
'height': height,
|
|
||||||
})
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
thumbnails = []
|
if m3u8_url := url_or_none(content_data.get('contentUrlHls')):
|
||||||
for k, v in clip.items():
|
formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls'))
|
||||||
if not (v and isinstance(v, str)):
|
if http_url := url_or_none(content_data.get('contentUrl')):
|
||||||
continue
|
formats.append({
|
||||||
mobj = re.match(r'(contentUrl|thumbnail)(?:(\d+)p)?$', k)
|
'url': http_url,
|
||||||
if not mobj:
|
'format_id': 'http-source',
|
||||||
continue
|
'ext': 'mp4',
|
||||||
prefix = mobj.group(1)
|
'quality': 1,
|
||||||
height = int_or_none(mobj.group(2))
|
})
|
||||||
if prefix == 'contentUrl':
|
formats = [fmt for fmt in formats if 'video/privacy-protected-guest' not in fmt['url']]
|
||||||
add_item(
|
if not formats:
|
||||||
formats, v, height or source_height,
|
# Fallback, does not require auth
|
||||||
item_id=None if height else 'source')
|
self.report_warning('Video formats are not available through API, falling back to social video URL')
|
||||||
elif prefix == 'thumbnail':
|
urlh = self._request_webpage(
|
||||||
add_item(thumbnails, v, height, 'id')
|
f'https://medal.tv/api/content/{video_id}/socialVideoUrl', video_id,
|
||||||
|
note='Checking social video URL')
|
||||||
error = clip.get('error')
|
formats.append({
|
||||||
if not formats and error:
|
'url': urlh.url,
|
||||||
if error == 404:
|
'format_id': 'social-video',
|
||||||
self.raise_no_formats(
|
'ext': 'mp4',
|
||||||
'That clip does not exist.',
|
'quality': -1,
|
||||||
expected=True, video_id=video_id)
|
})
|
||||||
else:
|
|
||||||
self.raise_no_formats(
|
|
||||||
f'An unknown error occurred ({error}).',
|
|
||||||
video_id=video_id)
|
|
||||||
|
|
||||||
# Necessary because the id of the author is not known in advance.
|
|
||||||
# Won't raise an issue if no profile can be found as this is optional.
|
|
||||||
author = traverse_obj(hydration_data, ('profiles', ...), get_all=False) or {}
|
|
||||||
author_id = str_or_none(author.get('userId'))
|
|
||||||
author_url = format_field(author_id, None, 'https://medal.tv/users/%s')
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
'thumbnails': thumbnails,
|
**traverse_obj(content_data, {
|
||||||
'description': clip.get('contentDescription'),
|
'title': ('contentTitle', {str}),
|
||||||
'uploader': author.get('displayName'),
|
'description': ('contentDescription', {str}),
|
||||||
'timestamp': float_or_none(clip.get('created'), 1000),
|
'timestamp': ('created', {int_or_none(scale=1000)}),
|
||||||
'uploader_id': author_id,
|
'duration': ('videoLengthSeconds', {int_or_none}),
|
||||||
'uploader_url': author_url,
|
'view_count': ('views', {int_or_none}),
|
||||||
'duration': int_or_none(clip.get('videoLengthSeconds')),
|
'like_count': ('likes', {int_or_none}),
|
||||||
'view_count': int_or_none(clip.get('views')),
|
'comment_count': ('comments', {int_or_none}),
|
||||||
'like_count': int_or_none(clip.get('likes')),
|
'uploader': ('poster', 'displayName', {str}),
|
||||||
'comment_count': int_or_none(clip.get('comments')),
|
'uploader_id': ('poster', 'userId', {str}),
|
||||||
|
'uploader_url': ('poster', 'userId', {str}, filter, {lambda x: x and f'https://medal.tv/users/{x}'}),
|
||||||
|
'tags': ('tags', ..., {str}),
|
||||||
|
'thumbnail': ('thumbnailUrl', {url_or_none}),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
92
yt_dlp/extractor/mux.py
Normal file
92
yt_dlp/extractor/mux.py
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
extract_attributes,
|
||||||
|
filter_dict,
|
||||||
|
parse_qs,
|
||||||
|
smuggle_url,
|
||||||
|
unsmuggle_url,
|
||||||
|
update_url_query,
|
||||||
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
|
class MuxIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:stream\.new/v|player\.mux\.com)/(?P<id>[A-Za-z0-9-]+)'
|
||||||
|
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:stream\.new/v|player\.mux\.com)/(?P<id>[A-Za-z0-9-]+)[^"\']+)']
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://stream.new/v/OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j/embed',
|
||||||
|
'info_dict': {
|
||||||
|
'ext': 'mp4',
|
||||||
|
'id': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||||
|
'title': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://player.mux.com/OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||||
|
'info_dict': {
|
||||||
|
'ext': 'mp4',
|
||||||
|
'id': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||||
|
'title': 'OCtRWZiZqKvLbnZ32WSEYiGNvHdAmB01j',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
_WEBPAGE_TESTS = [{
|
||||||
|
# iframe embed
|
||||||
|
'url': 'https://www.redbrickai.com/blog/2025-07-14-FAST-brush',
|
||||||
|
'info_dict': {
|
||||||
|
'ext': 'mp4',
|
||||||
|
'id': 'cXhzAiW1AmsHY01eRbEYFcTEAn0102aGN8sbt8JprP6Dfw',
|
||||||
|
'title': 'cXhzAiW1AmsHY01eRbEYFcTEAn0102aGN8sbt8JprP6Dfw',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
# mux-player embed
|
||||||
|
'url': 'https://muxvideo.2coders.com/download/',
|
||||||
|
'info_dict': {
|
||||||
|
'ext': 'mp4',
|
||||||
|
'id': 'JBuasdg35Hw7tYmTe9k68QLPQKixL300YsWHDz5Flit8',
|
||||||
|
'title': 'JBuasdg35Hw7tYmTe9k68QLPQKixL300YsWHDz5Flit8',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
# mux-player with title metadata
|
||||||
|
'url': 'https://datastar-todomvc.cross.stream/',
|
||||||
|
'info_dict': {
|
||||||
|
'ext': 'mp4',
|
||||||
|
'id': 'KX01ZSZ8CXv5SVfVwMZKJTcuBcUQmo1ReS9U5JjoHm4k',
|
||||||
|
'title': 'TodoMVC with Datastar Tutorial',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _extract_embed_urls(cls, url, webpage):
|
||||||
|
yield from super()._extract_embed_urls(url, webpage)
|
||||||
|
for mux_player in re.findall(r'<mux-(?:player|video)\b[^>]*\bplayback-id=[^>]+>', webpage):
|
||||||
|
attrs = extract_attributes(mux_player)
|
||||||
|
playback_id = attrs.get('playback-id')
|
||||||
|
if not playback_id:
|
||||||
|
continue
|
||||||
|
token = attrs.get('playback-token') or traverse_obj(playback_id, ({parse_qs}, 'token', -1))
|
||||||
|
playback_id = playback_id.partition('?')[0]
|
||||||
|
|
||||||
|
embed_url = update_url_query(
|
||||||
|
f'https://player.mux.com/{playback_id}',
|
||||||
|
filter_dict({'playback-token': token}))
|
||||||
|
if title := attrs.get('metadata-video-title'):
|
||||||
|
embed_url = smuggle_url(embed_url, {'title': title})
|
||||||
|
yield embed_url
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
url, smuggled_data = unsmuggle_url(url, {})
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
token = traverse_obj(parse_qs(url), ('playback-token', -1))
|
||||||
|
|
||||||
|
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||||
|
f'https://stream.mux.com/{video_id}.m3u8', video_id, 'mp4',
|
||||||
|
query=filter_dict({'token': token}))
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': smuggled_data.get('title') or video_id,
|
||||||
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
}
|
||||||
79
yt_dlp/extractor/netapp.py
Normal file
79
yt_dlp/extractor/netapp.py
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
from .brightcove import BrightcoveNewIE
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import parse_iso8601
|
||||||
|
from ..utils.traversal import require, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
|
class NetAppBaseIE(InfoExtractor):
|
||||||
|
_BC_URL = 'https://players.brightcove.net/6255154784001/default_default/index.html?videoId={}'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_metadata(item):
|
||||||
|
return traverse_obj(item, {
|
||||||
|
'title': ('name', {str}),
|
||||||
|
'description': ('description', {str}),
|
||||||
|
'timestamp': ('createdAt', {parse_iso8601}),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
class NetAppVideoIE(NetAppBaseIE):
|
||||||
|
_VALID_URL = r'https?://media\.netapp\.com/video-detail/(?P<id>[0-9a-f-]+)'
|
||||||
|
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://media.netapp.com/video-detail/da25fc01-82ad-5284-95bc-26920200a222/seamless-storage-for-modern-kubernetes-deployments',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1843620950167202073',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Seamless storage for modern Kubernetes deployments',
|
||||||
|
'description': 'md5:1ee39e315243fe71fb90af2796037248',
|
||||||
|
'uploader_id': '6255154784001',
|
||||||
|
'duration': 2159.41,
|
||||||
|
'thumbnail': r're:https://house-fastly-signed-us-east-1-prod\.brightcovecdn\.com/image/.*\.jpg',
|
||||||
|
'tags': 'count:15',
|
||||||
|
'timestamp': 1758213949,
|
||||||
|
'upload_date': '20250918',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://media.netapp.com/video-detail/45593e5d-cf1c-5996-978c-c9081906e69f/unleash-ai-innovation-with-your-data-with-the-netapp-platform',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_uuid = self._match_id(url)
|
||||||
|
metadata = self._download_json(
|
||||||
|
f'https://api.media.netapp.com/client/detail/{video_uuid}', video_uuid)
|
||||||
|
|
||||||
|
brightcove_video_id = traverse_obj(metadata, (
|
||||||
|
'sections', lambda _, v: v['type'] == 'Player', 'video', {str}, any, {require('brightcove video id')}))
|
||||||
|
|
||||||
|
video_item = traverse_obj(metadata, ('sections', lambda _, v: v['type'] == 'VideoDetail', any))
|
||||||
|
|
||||||
|
return self.url_result(
|
||||||
|
self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id,
|
||||||
|
url_transparent=True, **self._parse_metadata(video_item))
|
||||||
|
|
||||||
|
|
||||||
|
class NetAppCollectionIE(NetAppBaseIE):
|
||||||
|
_VALID_URL = r'https?://media\.netapp\.com/collection/(?P<id>[0-9a-f-]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://media.netapp.com/collection/9820e190-f2a6-47ac-9c0a-98e5e64234a4',
|
||||||
|
'info_dict': {
|
||||||
|
'title': 'Featured sessions',
|
||||||
|
'id': '9820e190-f2a6-47ac-9c0a-98e5e64234a4',
|
||||||
|
},
|
||||||
|
'playlist_count': 4,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _entries(self, metadata):
|
||||||
|
for item in traverse_obj(metadata, ('items', lambda _, v: v['brightcoveVideoId'])):
|
||||||
|
brightcove_video_id = item['brightcoveVideoId']
|
||||||
|
yield self.url_result(
|
||||||
|
self._BC_URL.format(brightcove_video_id), BrightcoveNewIE, brightcove_video_id,
|
||||||
|
url_transparent=True, **self._parse_metadata(item))
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
collection_uuid = self._match_id(url)
|
||||||
|
metadata = self._download_json(
|
||||||
|
f'https://api.media.netapp.com/client/collection/{collection_uuid}', collection_uuid)
|
||||||
|
|
||||||
|
return self.playlist_result(self._entries(metadata), collection_uuid, playlist_title=metadata.get('name'))
|
||||||
37
yt_dlp/extractor/nowcanal.py
Normal file
37
yt_dlp/extractor/nowcanal.py
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
from .brightcove import BrightcoveNewIE
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class NowCanalIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?nowcanal\.pt(?:/[\w-]+)+/detalhe/(?P<id>[\w-]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.nowcanal.pt/ultimas/detalhe/pedro-sousa-hjulmand-pode-ter-uma-saida-limpa-do-sporting-daqui-a-um-ano',
|
||||||
|
'md5': '047f17cb783e66e467d703e704bbc95d',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '6376598467112',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Pedro Sousa «Hjulmand pode ter uma saída limpa do Sporting daqui a um ano»',
|
||||||
|
'description': '',
|
||||||
|
'uploader_id': '6108484330001',
|
||||||
|
'duration': 65.237,
|
||||||
|
'thumbnail': r're:^https://.+\.jpg',
|
||||||
|
'timestamp': 1754440620,
|
||||||
|
'upload_date': '20250806',
|
||||||
|
'tags': ['now'],
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.nowcanal.pt/programas/frente-a-frente/detalhe/frente-a-frente-eva-cruzeiro-ps-e-rita-matias-chega',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
_BC_URL_TMPL = 'https://players.brightcove.net/6108484330001/chhIqzukMq_default/index.html?videoId={}'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
display_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
video_id = self._search_json(
|
||||||
|
r'videoHandler\.addBrightcoveVideoWithJson\(\[',
|
||||||
|
webpage, 'video data', display_id)['brightcoveVideoId']
|
||||||
|
|
||||||
|
return self.url_result(self._BC_URL_TMPL.format(video_id), BrightcoveNewIE)
|
||||||
|
|
@ -1,17 +1,40 @@
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
strip_or_none,
|
parse_iso8601,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
|
url_or_none,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
)
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class NTVRuIE(InfoExtractor):
|
class NTVRuIE(InfoExtractor):
|
||||||
IE_NAME = 'ntv.ru'
|
IE_NAME = 'ntv.ru'
|
||||||
_VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
_VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/#?]+/)*(?P<id>[^/?#&]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
|
# JSON Api is geo restricted
|
||||||
|
'url': 'https://www.ntv.ru/peredacha/svoya_igra/m58980/o818800',
|
||||||
|
'md5': '818962a1b52747d446db7cd5be43e142',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '2520563',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Участники: Ирина Петрова, Сергей Коновалов, Кристина Кораблина',
|
||||||
|
'description': 'md5:fcbd21cd45238a940b95550f9e178e3e',
|
||||||
|
'thumbnail': r're:^http://.*\.jpg',
|
||||||
|
'duration': 2462,
|
||||||
|
'view_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'tags': ['игры и игрушки'],
|
||||||
|
'timestamp': 1761821096,
|
||||||
|
'upload_date': '20251030',
|
||||||
|
'release_timestamp': 1761821096,
|
||||||
|
'release_date': '20251030',
|
||||||
|
'modified_timestamp': 1761821096,
|
||||||
|
'modified_date': '20251030',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
'url': 'http://www.ntv.ru/novosti/863142/',
|
'url': 'http://www.ntv.ru/novosti/863142/',
|
||||||
'md5': 'ba7ea172a91cb83eb734cad18c10e723',
|
'md5': 'ba7ea172a91cb83eb734cad18c10e723',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -22,31 +45,35 @@ class NTVRuIE(InfoExtractor):
|
||||||
'thumbnail': r're:^http://.*\.jpg',
|
'thumbnail': r're:^http://.*\.jpg',
|
||||||
'duration': 136,
|
'duration': 136,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'tags': ['ВМС', 'захват', 'митинги', 'Севастополь', 'Украина'],
|
||||||
|
'timestamp': 1395222013,
|
||||||
|
'upload_date': '20140319',
|
||||||
|
'release_timestamp': 1395222013,
|
||||||
|
'release_date': '20140319',
|
||||||
|
'modified_timestamp': 1395222013,
|
||||||
|
'modified_date': '20140319',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.ntv.ru/video/novosti/750370/',
|
# Requires unescapeHTML
|
||||||
'md5': 'adecff79691b4d71e25220a191477124',
|
|
||||||
'info_dict': {
|
|
||||||
'id': '750370',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
|
|
||||||
'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
|
|
||||||
'thumbnail': r're:^http://.*\.jpg',
|
|
||||||
'duration': 172,
|
|
||||||
'view_count': int,
|
|
||||||
},
|
|
||||||
'skip': '404 Not Found',
|
|
||||||
}, {
|
|
||||||
'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
|
'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
|
||||||
'md5': '82dbd49b38e3af1d00df16acbeab260c',
|
'md5': '82dbd49b38e3af1d00df16acbeab260c',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '747480',
|
'id': '747480',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '«Сегодня». 21 марта 2014 года. 16:00',
|
'title': '"Сегодня". 21 марта 2014 года. 16:00 ',
|
||||||
'description': '«Сегодня». 21 марта 2014 года. 16:00',
|
'description': 'md5:bed80745ca72af557433195f51a02785',
|
||||||
'thumbnail': r're:^http://.*\.jpg',
|
'thumbnail': r're:^http://.*\.jpg',
|
||||||
'duration': 1496,
|
'duration': 1496,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'tags': ['Брюссель', 'гражданство', 'ЕС', 'Крым', 'ОСАГО', 'саммит', 'санкции', 'события', 'чиновники', 'рейтинг'],
|
||||||
|
'timestamp': 1395406951,
|
||||||
|
'upload_date': '20140321',
|
||||||
|
'release_timestamp': 1395406951,
|
||||||
|
'release_date': '20140321',
|
||||||
|
'modified_timestamp': 1395406951,
|
||||||
|
'modified_date': '20140321',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/',
|
'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/',
|
||||||
|
|
@ -54,11 +81,19 @@ class NTVRuIE(InfoExtractor):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '1126480',
|
'id': '1126480',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Остросюжетный фильм «Кома»',
|
'title': 'Остросюжетный фильм "Кома"',
|
||||||
'description': 'Остросюжетный фильм «Кома»',
|
'description': 'md5:e79ffd0887425a0f05a58885c408d7d8',
|
||||||
'thumbnail': r're:^http://.*\.jpg',
|
'thumbnail': r're:^http://.*\.jpg',
|
||||||
'duration': 5592,
|
'duration': 5608,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'tags': ['кино'],
|
||||||
|
'timestamp': 1432868572,
|
||||||
|
'upload_date': '20150529',
|
||||||
|
'release_timestamp': 1432868572,
|
||||||
|
'release_date': '20150529',
|
||||||
|
'modified_timestamp': 1432868572,
|
||||||
|
'modified_date': '20150529',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
|
'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
|
||||||
|
|
@ -66,11 +101,19 @@ class NTVRuIE(InfoExtractor):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '751482',
|
'id': '751482',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '«Дело врачей»: «Деревце жизни»',
|
'title': '"Дело врачей": "Деревце жизни"',
|
||||||
'description': '«Дело врачей»: «Деревце жизни»',
|
'description': 'md5:d6fbf9193f880f50d9cbfbcc954161c1',
|
||||||
'thumbnail': r're:^http://.*\.jpg',
|
'thumbnail': r're:^http://.*\.jpg',
|
||||||
'duration': 2590,
|
'duration': 2590,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'tags': ['врачи', 'больницы'],
|
||||||
|
'timestamp': 1395882300,
|
||||||
|
'upload_date': '20140327',
|
||||||
|
'release_timestamp': 1395882300,
|
||||||
|
'release_date': '20140327',
|
||||||
|
'modified_timestamp': 1395882300,
|
||||||
|
'modified_date': '20140327',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
# Schemeless file URL
|
# Schemeless file URL
|
||||||
|
|
@ -78,48 +121,26 @@ class NTVRuIE(InfoExtractor):
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
_VIDEO_ID_REGEXES = [
|
|
||||||
r'<meta property="og:url" content="https?://www\.ntv\.ru/video/(\d+)',
|
|
||||||
r'<meta property="og:video:(?:url|iframe)" content="https?://www\.ntv\.ru/embed/(\d+)',
|
|
||||||
r'<video embed=[^>]+><id>(\d+)</id>',
|
|
||||||
r'<video restriction[^>]+><key>(\d+)</key>',
|
|
||||||
]
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
video_id = self._html_search_regex(
|
||||||
|
r'<meta property="ya:ovs:feed_url" content="https?://www\.ntv\.ru/(?:exp/)?video/(\d+)', webpage, 'video id')
|
||||||
video_url = self._og_search_property(
|
|
||||||
('video', 'video:iframe'), webpage, default=None)
|
|
||||||
if video_url:
|
|
||||||
video_id = self._search_regex(
|
|
||||||
r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
|
|
||||||
video_url, 'video id', default=None)
|
|
||||||
|
|
||||||
if not video_id:
|
|
||||||
video_id = self._html_search_regex(
|
|
||||||
self._VIDEO_ID_REGEXES, webpage, 'video id')
|
|
||||||
|
|
||||||
player = self._download_xml(
|
player = self._download_xml(
|
||||||
f'http://www.ntv.ru/vi{video_id}/',
|
f'http://www.ntv.ru/vi{video_id}/',
|
||||||
video_id, 'Downloading video XML')
|
video_id, 'Downloading video XML')
|
||||||
|
|
||||||
title = strip_or_none(unescapeHTML(xpath_text(player, './data/title', 'title', fatal=True)))
|
|
||||||
|
|
||||||
video = player.find('./data/video')
|
video = player.find('./data/video')
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for format_id in ['', 'hi', 'webm']:
|
for format_id in ['', 'hi', 'webm']:
|
||||||
file_ = xpath_text(video, f'./{format_id}file')
|
video_url = url_or_none(xpath_text(video, f'./{format_id}file'))
|
||||||
if not file_:
|
if not video_url:
|
||||||
continue
|
continue
|
||||||
if file_.startswith('//'):
|
|
||||||
file_ = self._proto_relative_url(file_)
|
|
||||||
elif not file_.startswith('http'):
|
|
||||||
file_ = 'http://media.ntv.ru/vod/' + file_
|
|
||||||
formats.append({
|
formats.append({
|
||||||
'url': file_,
|
'url': video_url,
|
||||||
'filesize': int_or_none(xpath_text(video, f'./{format_id}size')),
|
'filesize': int_or_none(xpath_text(video, f'./{format_id}size')),
|
||||||
})
|
})
|
||||||
hls_manifest = xpath_text(video, './playback/hls')
|
hls_manifest = xpath_text(video, './playback/hls')
|
||||||
|
|
@ -131,12 +152,28 @@ class NTVRuIE(InfoExtractor):
|
||||||
formats.extend(self._extract_mpd_formats(
|
formats.extend(self._extract_mpd_formats(
|
||||||
dash_manifest, video_id, mpd_id='dash', fatal=False))
|
dash_manifest, video_id, mpd_id='dash', fatal=False))
|
||||||
|
|
||||||
|
metadata = self._download_xml(
|
||||||
|
f'https://www.ntv.ru/exp/video/{video_id}', video_id, 'Downloading XML metadata', fatal=False)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': xpath_text(video, './id'),
|
'id': video_id,
|
||||||
'title': title,
|
|
||||||
'description': strip_or_none(unescapeHTML(xpath_text(player, './data/description'))),
|
|
||||||
'thumbnail': xpath_text(video, './splash'),
|
|
||||||
'duration': int_or_none(xpath_text(video, './totaltime')),
|
|
||||||
'view_count': int_or_none(xpath_text(video, './views')),
|
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
|
**traverse_obj(player, {
|
||||||
|
'title': ('data/title/text()', ..., {str}, {unescapeHTML}, any),
|
||||||
|
'description': ('data/description/text()', ..., {str}, {unescapeHTML}, any),
|
||||||
|
'duration': ('data/video/totaltime/text()', ..., {int_or_none}, any),
|
||||||
|
'view_count': ('data/video/views/text()', ..., {int_or_none}, any),
|
||||||
|
'thumbnail': ('data/video/splash/text()', ..., {url_or_none}, any),
|
||||||
|
}),
|
||||||
|
**traverse_obj(metadata, {
|
||||||
|
'title': ('{*}title/text()', ..., {str}, {unescapeHTML}, any),
|
||||||
|
'description': ('{*}description/text()', ..., {str}, {unescapeHTML}, any),
|
||||||
|
'duration': ('{*}duration/text()', ..., {int_or_none}, any),
|
||||||
|
'timestamp': ('{*}create_date/text()', ..., {parse_iso8601}, any),
|
||||||
|
'release_timestamp': ('{*}upload_date/text()', ..., {parse_iso8601}, any),
|
||||||
|
'modified_timestamp': ('{*}modify_date/text()', ..., {parse_iso8601}, any),
|
||||||
|
'tags': ('{*}tag/text()', ..., {str}, {lambda x: x.split(',')}, ..., {str.strip}, filter),
|
||||||
|
'view_count': ('{*}stats/views_total/text()', ..., {int_or_none}, any),
|
||||||
|
'comment_count': ('{*}stats/comments/text()', ..., {int_or_none}, any),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,14 @@ from ..utils import (
|
||||||
MEDIA_EXTENSIONS,
|
MEDIA_EXTENSIONS,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
traverse_obj,
|
|
||||||
url_or_none,
|
url_or_none,
|
||||||
)
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class RinseFMBaseIE(InfoExtractor):
|
class RinseFMBaseIE(InfoExtractor):
|
||||||
|
_API_BASE = 'https://rinse.fm/api/query/v1'
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_entry(entry):
|
def _parse_entry(entry):
|
||||||
return {
|
return {
|
||||||
|
|
@ -45,8 +47,10 @@ class RinseFMIE(RinseFMBaseIE):
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(url, display_id)
|
|
||||||
entry = self._search_nextjs_data(webpage, display_id)['props']['pageProps']['entry']
|
entry = self._download_json(
|
||||||
|
f'{self._API_BASE}/episodes/{display_id}', display_id,
|
||||||
|
note='Downloading episode data from API')['entry']
|
||||||
|
|
||||||
return self._parse_entry(entry)
|
return self._parse_entry(entry)
|
||||||
|
|
||||||
|
|
@ -58,32 +62,35 @@ class RinseFMArtistPlaylistIE(RinseFMBaseIE):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'resources',
|
'id': 'resources',
|
||||||
'title': '[re]sources',
|
'title': '[re]sources',
|
||||||
'description': '[re]sources est un label parisien piloté par le DJ et producteur Tommy Kid.',
|
'description': 'md5:fd6a7254e8273510e6d49fbf50edf392',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 40,
|
'playlist_mincount': 40,
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://rinse.fm/shows/ivy/',
|
'url': 'https://www.rinse.fm/shows/esk',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'ivy',
|
'id': 'esk',
|
||||||
'title': '[IVY]',
|
'title': 'Esk',
|
||||||
'description': 'A dedicated space for DNB/Turbo House and 4x4.',
|
'description': 'md5:5893d7c1d411ae8dea7fba12f109aa98',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 7,
|
'playlist_mincount': 139,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _entries(self, data):
|
def _entries(self, data):
|
||||||
for episode in traverse_obj(data, (
|
for episode in traverse_obj(data, (
|
||||||
'props', 'pageProps', 'episodes', lambda _, v: determine_ext(v['fileUrl']) in MEDIA_EXTENSIONS.audio),
|
'episodes', lambda _, v: determine_ext(v['fileUrl']) in MEDIA_EXTENSIONS.audio),
|
||||||
):
|
):
|
||||||
yield self._parse_entry(episode)
|
yield self._parse_entry(episode)
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
playlist_id = self._match_id(url)
|
playlist_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(url, playlist_id)
|
|
||||||
title = self._og_search_title(webpage) or self._html_search_meta('title', webpage)
|
api_data = self._download_json(
|
||||||
description = self._og_search_description(webpage) or self._html_search_meta(
|
f'{self._API_BASE}/shows/{playlist_id}', playlist_id,
|
||||||
'description', webpage)
|
note='Downloading show data from API')
|
||||||
data = self._search_nextjs_data(webpage, playlist_id)
|
|
||||||
|
|
||||||
return self.playlist_result(
|
return self.playlist_result(
|
||||||
self._entries(data), playlist_id, title, description=description)
|
self._entries(api_data), playlist_id,
|
||||||
|
**traverse_obj(api_data, ('entry', {
|
||||||
|
'title': ('title', {str}),
|
||||||
|
'description': ('description', {str}),
|
||||||
|
})))
|
||||||
|
|
|
||||||
|
|
@ -1064,7 +1064,7 @@ class SoundcloudRelatedIE(SoundcloudPagedPlaylistBaseIE):
|
||||||
|
|
||||||
|
|
||||||
class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
|
class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
|
||||||
_VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
|
_VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?:soundcloud(?:%3A|:)playlists(?:%3A|:))?(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
|
||||||
IE_NAME = 'soundcloud:playlist'
|
IE_NAME = 'soundcloud:playlist'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://api.soundcloud.com/playlists/4110309',
|
'url': 'https://api.soundcloud.com/playlists/4110309',
|
||||||
|
|
@ -1079,6 +1079,12 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
|
||||||
'album': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
|
'album': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
|
||||||
},
|
},
|
||||||
'playlist_count': 6,
|
'playlist_count': 6,
|
||||||
|
}, {
|
||||||
|
'url': 'https://api.soundcloud.com/playlists/soundcloud%3Aplaylists%3A1759227795',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://api.soundcloud.com/playlists/soundcloud:playlists:2104769627?secret_token=s-wmpCLuExeYX',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
|
|
||||||
|
|
@ -101,8 +101,8 @@ class SproutVideoIE(InfoExtractor):
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'}))
|
url, video_id, headers=traverse_obj(smuggled_data, {'Referer': 'referer'}))
|
||||||
data = self._search_json(
|
data = self._search_json(
|
||||||
r'(?:var|const|let)\s+(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage, 'player info',
|
r'(?:window\.|(?:var|const|let)\s+)(?:dat|(?:player|video)Info|)\s*=\s*["\']', webpage,
|
||||||
video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];',
|
'player info', video_id, contains_pattern=r'[A-Za-z0-9+/=]+', end_pattern=r'["\'];',
|
||||||
transform_source=lambda x: base64.b64decode(x).decode())
|
transform_source=lambda x: base64.b64decode(x).decode())
|
||||||
|
|
||||||
# SproutVideo may send player info for 'SMPTE Color Monitor Test' [a791d7b71b12ecc52e]
|
# SproutVideo may send player info for 'SMPTE Color Monitor Test' [a791d7b71b12ecc52e]
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,17 @@
|
||||||
import json
|
import urllib.parse
|
||||||
|
|
||||||
from .brightcove import BrightcoveNewIE
|
from .brightcove import BrightcoveNewIE
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from .zype import ZypeIE
|
from .zype import ZypeIE
|
||||||
from ..networking import HEADRequest
|
from ..networking import HEADRequest
|
||||||
from ..networking.exceptions import HTTPError
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
filter_dict,
|
filter_dict,
|
||||||
parse_qs,
|
parse_qs,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
try_call,
|
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
)
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class ThisOldHouseIE(InfoExtractor):
|
class ThisOldHouseIE(InfoExtractor):
|
||||||
|
|
@ -77,46 +76,43 @@ class ThisOldHouseIE(InfoExtractor):
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
_LOGIN_URL = 'https://login.thisoldhouse.com/usernamepassword/login'
|
|
||||||
|
|
||||||
def _perform_login(self, username, password):
|
def _perform_login(self, username, password):
|
||||||
self._request_webpage(
|
login_page = self._download_webpage(
|
||||||
HEADRequest('https://www.thisoldhouse.com/insider'), None, 'Requesting session cookies')
|
'https://www.thisoldhouse.com/insider-login', None, 'Downloading login page')
|
||||||
urlh = self._request_webpage(
|
hidden_inputs = self._hidden_inputs(login_page)
|
||||||
'https://www.thisoldhouse.com/wp-login.php', None, 'Requesting login info',
|
response = self._download_json(
|
||||||
errnote='Unable to login', query={'redirect_to': 'https://www.thisoldhouse.com/insider'})
|
'https://www.thisoldhouse.com/wp-admin/admin-ajax.php', None, 'Logging in',
|
||||||
|
headers={
|
||||||
|
'Accept': 'application/json',
|
||||||
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
}, data=urlencode_postdata(filter_dict({
|
||||||
|
'action': 'onebill_subscriber_login',
|
||||||
|
'email': username,
|
||||||
|
'password': password,
|
||||||
|
'pricingPlanTerm': hidden_inputs['pricing_plan_term'],
|
||||||
|
'utm_parameters': hidden_inputs.get('utm_parameters'),
|
||||||
|
'nonce': hidden_inputs['mdcr_onebill_login_nonce'],
|
||||||
|
})))
|
||||||
|
|
||||||
try:
|
message = traverse_obj(response, ('data', 'message', {str}))
|
||||||
auth_form = self._download_webpage(
|
if not response['success']:
|
||||||
self._LOGIN_URL, None, 'Submitting credentials', headers={
|
if message and 'Something went wrong' in message:
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Referer': urlh.url,
|
|
||||||
}, data=json.dumps(filter_dict({
|
|
||||||
**{('client_id' if k == 'client' else k): v[0] for k, v in parse_qs(urlh.url).items()},
|
|
||||||
'tenant': 'thisoldhouse',
|
|
||||||
'username': username,
|
|
||||||
'password': password,
|
|
||||||
'popup_options': {},
|
|
||||||
'sso': True,
|
|
||||||
'_csrf': try_call(lambda: self._get_cookies(self._LOGIN_URL)['_csrf'].value),
|
|
||||||
'_intstate': 'deprecated',
|
|
||||||
}), separators=(',', ':')).encode())
|
|
||||||
except ExtractorError as e:
|
|
||||||
if isinstance(e.cause, HTTPError) and e.cause.status == 401:
|
|
||||||
raise ExtractorError('Invalid username or password', expected=True)
|
raise ExtractorError('Invalid username or password', expected=True)
|
||||||
raise
|
raise ExtractorError(message or 'Login was unsuccessful')
|
||||||
|
if message and 'Your subscription is not active' in message:
|
||||||
self._request_webpage(
|
self.report_warning(
|
||||||
'https://login.thisoldhouse.com/login/callback', None, 'Completing login',
|
f'{self.IE_NAME} said your subscription is not active. '
|
||||||
data=urlencode_postdata(self._hidden_inputs(auth_form)))
|
f'If your subscription is active, this could be caused by too many sign-ins, '
|
||||||
|
f'and you should instead try using {self._login_hint(method="cookies")[4:]}')
|
||||||
|
else:
|
||||||
|
self.write_debug(f'{self.IE_NAME} said: {message}')
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
webpage = self._download_webpage(url, display_id)
|
webpage, urlh = self._download_webpage_handle(url, display_id)
|
||||||
if 'To Unlock This content' in webpage:
|
# If login response says inactive subscription, site redirects to frontpage for Insider content
|
||||||
self.raise_login_required(
|
if 'To Unlock This content' in webpage or urllib.parse.urlparse(urlh.url).path in ('', '/'):
|
||||||
'This video is only available for subscribers. '
|
self.raise_login_required('This video is only available for subscribers')
|
||||||
'Note that --cookies-from-browser may not work due to this site using session cookies')
|
|
||||||
|
|
||||||
video_url, video_id = self._search_regex(
|
video_url, video_id = self._search_regex(
|
||||||
r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
|
r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
|
||||||
|
|
|
||||||
|
|
@ -136,8 +136,10 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||||
IE_DESC = 'tube.tugraz.at'
|
IE_DESC = 'tube.tugraz.at'
|
||||||
|
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)
|
||||||
https?://tube\.tugraz\.at/paella/ui/watch.html\?id=
|
https?://tube\.tugraz\.at/(?:
|
||||||
(?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
|
paella/ui/watch\.html\?(?:[^#]*&)?id=|
|
||||||
|
portal/watch/
|
||||||
|
)(?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
|
||||||
'''
|
'''
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
|
|
@ -149,9 +151,9 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||||
'title': '#6 (23.11.2017)',
|
'title': '#6 (23.11.2017)',
|
||||||
'episode': '#6 (23.11.2017)',
|
'episode': '#6 (23.11.2017)',
|
||||||
'series': '[INB03001UF] Einführung in die strukturierte Programmierung',
|
'series': '[INB03001UF] Einführung in die strukturierte Programmierung',
|
||||||
'creator': 'Safran C',
|
|
||||||
'duration': 3295818,
|
'duration': 3295818,
|
||||||
'series_id': 'b1192fff-2aa7-4bf0-a5cf-7b15c3bd3b34',
|
'series_id': 'b1192fff-2aa7-4bf0-a5cf-7b15c3bd3b34',
|
||||||
|
'creators': ['Safran C'],
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=2df6d787-e56a-428d-8ef4-d57f07eef238',
|
'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=2df6d787-e56a-428d-8ef4-d57f07eef238',
|
||||||
|
|
@ -162,6 +164,10 @@ class TubeTuGrazIE(TubeTuGrazBaseIE):
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
},
|
},
|
||||||
'expected_warnings': ['Extractor failed to obtain "title"'],
|
'expected_warnings': ['Extractor failed to obtain "title"'],
|
||||||
|
}, {
|
||||||
|
# Portal URL format
|
||||||
|
'url': 'https://tube.tugraz.at/portal/watch/ab28ec60-8cbe-4f1a-9b96-a95add56c612',
|
||||||
|
'only_matching': True,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,16 +41,16 @@ class TwitchBaseIE(InfoExtractor):
|
||||||
_NETRC_MACHINE = 'twitch'
|
_NETRC_MACHINE = 'twitch'
|
||||||
|
|
||||||
_OPERATION_HASHES = {
|
_OPERATION_HASHES = {
|
||||||
'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
|
'CollectionSideBar': '016e1e4ccee0eb4698eb3bf1a04dc1c077fb746c78c82bac9a8f0289658fbd1a',
|
||||||
'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
|
'FilterableVideoTower_Videos': '67004f7881e65c297936f32c75246470629557a393788fb5a69d6d9a25a8fd5f',
|
||||||
'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
|
'ClipsCards__User': '90c33f5e6465122fba8f9371e2a97076f9ed06c6fed3788d002ab9eba8f91d88',
|
||||||
'ShareClipRenderStatus': 'e0a46b287d760c6890a39d1ccd736af5ec9479a267d02c710e9ac33326b651d2',
|
'ShareClipRenderStatus': '1844261bb449fa51e6167040311da4a7a5f1c34fe71c71a3e0c4f551bc30c698',
|
||||||
'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
|
'ChannelCollectionsContent': '5247910a19b1cd2b760939bf4cba4dcbd3d13bdf8c266decd16956f6ef814077',
|
||||||
'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
|
'StreamMetadata': 'b57f9b910f8cd1a4659d894fe7550ccc81ec9052c01e438b290fd66a040b9b93',
|
||||||
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
|
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
|
||||||
'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
|
'VideoPreviewOverlay': '9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f',
|
||||||
'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
|
'VideoMetadata': '45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d',
|
||||||
'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
|
'VideoPlayer_ChapterSelectButtonVideo': '71835d5ef425e154bf282453a926d99b328cdc5e32f36d3a209d0f4778b41203',
|
||||||
'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
|
'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -621,15 +621,15 @@ def _make_video_result(node):
|
||||||
|
|
||||||
|
|
||||||
class TwitchCollectionIE(TwitchBaseIE):
|
class TwitchCollectionIE(TwitchBaseIE):
|
||||||
|
IE_NAME = 'twitch:collection'
|
||||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
|
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://www.twitch.tv/collections/wlDCoH0zEBZZbQ',
|
'url': 'https://www.twitch.tv/collections/o9zZer3IQBhTJw',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'wlDCoH0zEBZZbQ',
|
'id': 'o9zZer3IQBhTJw',
|
||||||
'title': 'Overthrow Nook, capitalism for children',
|
'title': 'Playthrough Archives',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 13,
|
'playlist_mincount': 21,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
_OPERATION_NAME = 'CollectionSideBar'
|
_OPERATION_NAME = 'CollectionSideBar'
|
||||||
|
|
@ -720,8 +720,8 @@ class TwitchVideosBaseIE(TwitchPlaylistBaseIE):
|
||||||
|
|
||||||
|
|
||||||
class TwitchVideosIE(TwitchVideosBaseIE):
|
class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
|
IE_NAME = 'twitch:videos'
|
||||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'
|
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
# All Videos sorted by Date
|
# All Videos sorted by Date
|
||||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
|
'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
|
||||||
|
|
@ -729,7 +729,7 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
'id': 'spamfish',
|
'id': 'spamfish',
|
||||||
'title': 'spamfish - All Videos sorted by Date',
|
'title': 'spamfish - All Videos sorted by Date',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 924,
|
'playlist_mincount': 751,
|
||||||
}, {
|
}, {
|
||||||
# All Videos sorted by Popular
|
# All Videos sorted by Popular
|
||||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
|
'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
|
||||||
|
|
@ -737,8 +737,9 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
'id': 'spamfish',
|
'id': 'spamfish',
|
||||||
'title': 'spamfish - All Videos sorted by Popular',
|
'title': 'spamfish - All Videos sorted by Popular',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 931,
|
'playlist_mincount': 754,
|
||||||
}, {
|
}, {
|
||||||
|
# TODO: Investigate why we get 0 entries
|
||||||
# Past Broadcasts sorted by Date
|
# Past Broadcasts sorted by Date
|
||||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
|
'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -753,8 +754,9 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
'id': 'spamfish',
|
'id': 'spamfish',
|
||||||
'title': 'spamfish - Highlights sorted by Date',
|
'title': 'spamfish - Highlights sorted by Date',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 901,
|
'playlist_mincount': 751,
|
||||||
}, {
|
}, {
|
||||||
|
# TODO: Investigate why we get 0 entries
|
||||||
# Uploads sorted by Date
|
# Uploads sorted by Date
|
||||||
'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
|
'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -763,6 +765,7 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
},
|
},
|
||||||
'playlist_mincount': 5,
|
'playlist_mincount': 5,
|
||||||
}, {
|
}, {
|
||||||
|
# TODO: Investigate why we get 0 entries
|
||||||
# Past Premieres sorted by Date
|
# Past Premieres sorted by Date
|
||||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
|
'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -825,8 +828,8 @@ class TwitchVideosIE(TwitchVideosBaseIE):
|
||||||
|
|
||||||
|
|
||||||
class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
||||||
|
IE_NAME = 'twitch:videos:clips'
|
||||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'
|
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
# Clips
|
# Clips
|
||||||
'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
|
'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
|
||||||
|
|
@ -898,8 +901,8 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
||||||
|
|
||||||
|
|
||||||
class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
|
class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
|
||||||
|
IE_NAME = 'twitch:videos:collections'
|
||||||
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'
|
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
# Collections
|
# Collections
|
||||||
'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
|
'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
|
||||||
|
|
@ -1050,7 +1053,10 @@ class TwitchStreamIE(TwitchVideosBaseIE):
|
||||||
gql = self._download_gql(
|
gql = self._download_gql(
|
||||||
channel_name, [{
|
channel_name, [{
|
||||||
'operationName': 'StreamMetadata',
|
'operationName': 'StreamMetadata',
|
||||||
'variables': {'channelLogin': channel_name},
|
'variables': {
|
||||||
|
'channelLogin': channel_name,
|
||||||
|
'includeIsDJ': True,
|
||||||
|
},
|
||||||
}, {
|
}, {
|
||||||
'operationName': 'ComscoreStreamingQuery',
|
'operationName': 'ComscoreStreamingQuery',
|
||||||
'variables': {
|
'variables': {
|
||||||
|
|
|
||||||
|
|
@ -7,15 +7,15 @@ from ..utils import (
|
||||||
parse_age_limit,
|
parse_age_limit,
|
||||||
try_get,
|
try_get,
|
||||||
unified_timestamp,
|
unified_timestamp,
|
||||||
|
url_or_none,
|
||||||
)
|
)
|
||||||
from ..utils.traversal import traverse_obj
|
from ..utils.traversal import require, traverse_obj
|
||||||
|
|
||||||
|
|
||||||
class URPlayIE(InfoExtractor):
|
class URPlayIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
|
'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
|
||||||
'md5': '5ba36643c77cc3d34ffeadad89937d1e',
|
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '203704',
|
'id': '203704',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
|
@ -31,6 +31,7 @@ class URPlayIE(InfoExtractor):
|
||||||
'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
|
'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
|
||||||
'age_limit': 15,
|
'age_limit': 15,
|
||||||
},
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
|
'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -49,6 +50,7 @@ class URPlayIE(InfoExtractor):
|
||||||
'tags': 'count:7',
|
'tags': 'count:7',
|
||||||
'episode': 'Mitt barn skadar sig själv',
|
'episode': 'Mitt barn skadar sig själv',
|
||||||
},
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
|
'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
|
|
@ -68,6 +70,27 @@ class URPlayIE(InfoExtractor):
|
||||||
'episode': 'Sovkudde',
|
'episode': 'Sovkudde',
|
||||||
'season': 'Säsong 1',
|
'season': 'Säsong 1',
|
||||||
},
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
|
}, {
|
||||||
|
# Only accessible through new media api
|
||||||
|
'url': 'https://urplay.se/program/242932-vulkanernas-krafter-fran-kraftfull-till-forgorande',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '242932',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Vulkanernas krafter : Från kraftfull till förgörande',
|
||||||
|
'description': 'md5:742bb87048e7d5a7f209d28f9bb70ab1',
|
||||||
|
'age_limit': 15,
|
||||||
|
'duration': 2613,
|
||||||
|
'thumbnail': 'https://assets.ur.se/id/242932/images/1_hd.jpg',
|
||||||
|
'categories': ['Vetenskap & teknik'],
|
||||||
|
'tags': ['Geofysik', 'Naturvetenskap', 'Vulkaner', 'Vulkanutbrott'],
|
||||||
|
'series': 'Vulkanernas krafter',
|
||||||
|
'episode': 'Från kraftfull till förgörande',
|
||||||
|
'episode_number': 2,
|
||||||
|
'timestamp': 1763514000,
|
||||||
|
'upload_date': '20251119',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': 'm3u8'},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
|
'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
|
@ -88,21 +111,12 @@ class URPlayIE(InfoExtractor):
|
||||||
webpage, 'urplayer data'), video_id)['accessibleEpisodes']
|
webpage, 'urplayer data'), video_id)['accessibleEpisodes']
|
||||||
urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id))
|
urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id))
|
||||||
episode = urplayer_data['title']
|
episode = urplayer_data['title']
|
||||||
|
sources = self._download_json(
|
||||||
host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']
|
f'https://media-api.urplay.se/config-streaming/v1/urplay/sources/{video_id}', video_id,
|
||||||
formats = []
|
note='Downloading streaming information')
|
||||||
urplayer_streams = urplayer_data.get('streamingInfo', {})
|
hls_url = traverse_obj(sources, ('sources', 'hls', {url_or_none}, {require('HLS URL')}))
|
||||||
|
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||||
for k, v in urplayer_streams.get('raw', {}).items():
|
hls_url, video_id, 'mp4', m3u8_id='hls')
|
||||||
if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)):
|
|
||||||
continue
|
|
||||||
file_http = v.get('location')
|
|
||||||
if file_http:
|
|
||||||
formats.extend(self._extract_wowza_formats(
|
|
||||||
f'http://{host}/{file_http}playlist.m3u8',
|
|
||||||
video_id, skip_protocols=['f4m', 'rtmp', 'rtsp']))
|
|
||||||
|
|
||||||
subtitles = {}
|
|
||||||
|
|
||||||
def parse_lang_code(code):
|
def parse_lang_code(code):
|
||||||
"3-character language code or None (utils candidate)"
|
"3-character language code or None (utils candidate)"
|
||||||
|
|
|
||||||
|
|
@ -60,6 +60,37 @@ class _ByteGenerator:
|
||||||
s = to_signed_32(s * to_signed_32(0xc2b2ae3d))
|
s = to_signed_32(s * to_signed_32(0xc2b2ae3d))
|
||||||
return to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 16))
|
return to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 16))
|
||||||
|
|
||||||
|
def _algo4(self, s):
|
||||||
|
# Custom scrambling function involving a left rotation (ROL)
|
||||||
|
s = self._s = to_signed_32(s + 0x6d2b79f5)
|
||||||
|
s = to_signed_32((s << 7) | ((s & 0xFFFFFFFF) >> 25)) # ROL 7
|
||||||
|
s = to_signed_32(s + 0x9e3779b9)
|
||||||
|
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 11))
|
||||||
|
return to_signed_32(s * 0x27d4eb2d)
|
||||||
|
|
||||||
|
def _algo5(self, s):
|
||||||
|
# xorshift variant with a final addition
|
||||||
|
s = to_signed_32(s ^ (s << 7))
|
||||||
|
s = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 9))
|
||||||
|
s = to_signed_32(s ^ (s << 8))
|
||||||
|
s = self._s = to_signed_32(s + 0xa5a5a5a5)
|
||||||
|
return s
|
||||||
|
|
||||||
|
def _algo6(self, s):
|
||||||
|
# LCG (a=0x2c9277b5, c=0xac564b05) with a variable right shift scrambler
|
||||||
|
s = self._s = to_signed_32(s * to_signed_32(0x2c9277b5) + to_signed_32(0xac564b05))
|
||||||
|
s2 = to_signed_32(s ^ ((s & 0xFFFFFFFF) >> 18))
|
||||||
|
shift = (s & 0xFFFFFFFF) >> 27 & 31
|
||||||
|
return to_signed_32((s2 & 0xFFFFFFFF) >> shift)
|
||||||
|
|
||||||
|
def _algo7(self, s):
|
||||||
|
# Weyl Sequence (k=0x9e3779b9) + custom multiply-xor-shift mixing function
|
||||||
|
s = self._s = to_signed_32(s + to_signed_32(0x9e3779b9))
|
||||||
|
e = to_signed_32(s ^ (s << 5))
|
||||||
|
e = to_signed_32(e * to_signed_32(0x7feb352d))
|
||||||
|
e = to_signed_32(e ^ ((e & 0xFFFFFFFF) >> 15))
|
||||||
|
return to_signed_32(e * to_signed_32(0x846ca68b))
|
||||||
|
|
||||||
def __next__(self):
|
def __next__(self):
|
||||||
return self._algorithm(self._s) & 0xFF
|
return self._algorithm(self._s) & 0xFF
|
||||||
|
|
||||||
|
|
|
||||||
67
yt_dlp/extractor/yfanefa.py
Normal file
67
yt_dlp/extractor/yfanefa.py
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
determine_ext,
|
||||||
|
int_or_none,
|
||||||
|
join_nonempty,
|
||||||
|
remove_end,
|
||||||
|
url_or_none,
|
||||||
|
)
|
||||||
|
from ..utils.traversal import traverse_obj
|
||||||
|
|
||||||
|
|
||||||
|
class YfanefaIE(InfoExtractor):
|
||||||
|
IE_NAME = 'yfanefa'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?yfanefa\.com/(?P<id>[^?#]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.yfanefa.com/record/2717',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'record-2717',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'THE HALLAMSHIRE RIFLES LEAVING SHEFFIELD, 1914',
|
||||||
|
'duration': 5239,
|
||||||
|
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.yfanefa.com/news/53',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'news-53',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Memory Bank: Bradford Launch',
|
||||||
|
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.yfanefa.com/evaluating_nature_matters',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'evaluating_nature_matters',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Evaluating Nature Matters',
|
||||||
|
'thumbnail': r're:https://media\.yfanefa\.com/storage/v1/file/',
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
player_data = self._search_json(
|
||||||
|
r'iwPlayer\.options\["[\w.]+"\]\s*=', webpage, 'player options', video_id)
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
video_url = join_nonempty(player_data['url'], player_data.get('signature'), delim='')
|
||||||
|
if determine_ext(video_url) == 'm3u8':
|
||||||
|
formats = self._extract_m3u8_formats(
|
||||||
|
video_url, video_id, 'mp4', m3u8_id='hls')
|
||||||
|
else:
|
||||||
|
formats = [{'url': video_url, 'ext': 'mp4'}]
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id.strip('/').replace('/', '-'),
|
||||||
|
'title':
|
||||||
|
self._og_search_title(webpage, default=None)
|
||||||
|
or remove_end(self._html_extract_title(webpage), ' | Yorkshire Film Archive'),
|
||||||
|
'formats': formats,
|
||||||
|
**traverse_obj(player_data, {
|
||||||
|
'thumbnail': ('preview', {url_or_none}),
|
||||||
|
'duration': ('duration', {int_or_none}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
@ -327,6 +327,17 @@ INNERTUBE_CLIENTS = {
|
||||||
# See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506
|
# See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506
|
||||||
'AUTHENTICATED_USER_AGENT': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)',
|
'AUTHENTICATED_USER_AGENT': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)',
|
||||||
},
|
},
|
||||||
|
'tv_downgraded': {
|
||||||
|
'INNERTUBE_CONTEXT': {
|
||||||
|
'client': {
|
||||||
|
'clientName': 'TVHTML5',
|
||||||
|
'clientVersion': '5.20251105',
|
||||||
|
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
|
||||||
|
'SUPPORTS_COOKIES': True,
|
||||||
|
},
|
||||||
'tv_simply': {
|
'tv_simply': {
|
||||||
'INNERTUBE_CONTEXT': {
|
'INNERTUBE_CONTEXT': {
|
||||||
'client': {
|
'client': {
|
||||||
|
|
|
||||||
|
|
@ -340,8 +340,9 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
|
||||||
thumbnails=self._extract_thumbnails(view_model, (
|
thumbnails=self._extract_thumbnails(view_model, (
|
||||||
'contentImage', *thumb_keys, 'thumbnailViewModel', 'image'), final_key='sources'),
|
'contentImage', *thumb_keys, 'thumbnailViewModel', 'image'), final_key='sources'),
|
||||||
duration=traverse_obj(view_model, (
|
duration=traverse_obj(view_model, (
|
||||||
'contentImage', 'thumbnailViewModel', 'overlays', ..., 'thumbnailOverlayBadgeViewModel',
|
'contentImage', 'thumbnailViewModel', 'overlays', ...,
|
||||||
'thumbnailBadges', ..., 'thumbnailBadgeViewModel', 'text', {parse_duration}, any)),
|
(('thumbnailBottomOverlayViewModel', 'badges'), ('thumbnailOverlayBadgeViewModel', 'thumbnailBadges')),
|
||||||
|
..., 'thumbnailBadgeViewModel', 'text', {parse_duration}, any)),
|
||||||
timestamp=(traverse_obj(view_model, (
|
timestamp=(traverse_obj(view_model, (
|
||||||
'metadata', 'lockupMetadataViewModel', 'metadata', 'contentMetadataViewModel', 'metadataRows',
|
'metadata', 'lockupMetadataViewModel', 'metadata', 'contentMetadataViewModel', 'metadataRows',
|
||||||
..., 'metadataParts', ..., 'text', 'content', {lambda t: self._parse_time_text(t, report_failure=False)}, any))
|
..., 'metadataParts', ..., 'text', 'content', {lambda t: self._parse_time_text(t, report_failure=False)}, any))
|
||||||
|
|
|
||||||
|
|
@ -147,9 +147,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
_SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'srt', 'vtt')
|
_SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'srt', 'vtt')
|
||||||
_DEFAULT_CLIENTS = ('tv', 'android_sdkless', 'web')
|
_DEFAULT_CLIENTS = ('tv', 'android_sdkless', 'web')
|
||||||
_DEFAULT_JSLESS_CLIENTS = ('android_sdkless', 'web_safari', 'web')
|
_DEFAULT_JSLESS_CLIENTS = ('android_sdkless', 'web_safari', 'web')
|
||||||
_DEFAULT_AUTHED_CLIENTS = ('tv', 'web_safari', 'web')
|
_DEFAULT_AUTHED_CLIENTS = ('tv_downgraded', 'web_safari', 'web')
|
||||||
# Premium does not require POT (except for subtitles)
|
# Premium does not require POT (except for subtitles)
|
||||||
_DEFAULT_PREMIUM_CLIENTS = ('tv', 'web_creator', 'web')
|
_DEFAULT_PREMIUM_CLIENTS = ('tv_downgraded', 'web_creator', 'web')
|
||||||
|
|
||||||
_GEO_BYPASS = False
|
_GEO_BYPASS = False
|
||||||
|
|
||||||
|
|
@ -1556,6 +1556,110 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
},
|
},
|
||||||
'params': {'skip_download': True},
|
'params': {'skip_download': True},
|
||||||
|
}, {
|
||||||
|
# Youtube Music Auto-generated description with dot in artist name
|
||||||
|
'url': 'https://music.youtube.com/watch?v=DbCvuSGfR3Y',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'DbCvuSGfR3Y',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Back Around',
|
||||||
|
'artists': ['half·alive'],
|
||||||
|
'track': 'Back Around',
|
||||||
|
'album': 'Conditions Of A Punk',
|
||||||
|
'release_date': '20221202',
|
||||||
|
'release_year': 2021,
|
||||||
|
'alt_title': 'Back Around',
|
||||||
|
'description': 'md5:bfc0e2b3cc903a608d8a85a13cb50f95',
|
||||||
|
'media_type': 'video',
|
||||||
|
'uploader': 'half•alive',
|
||||||
|
'channel': 'half•alive',
|
||||||
|
'channel_id': 'UCYQrYophdVI3nVDPOnXyIng',
|
||||||
|
'channel_url': 'https://www.youtube.com/channel/UCYQrYophdVI3nVDPOnXyIng',
|
||||||
|
'channel_is_verified': True,
|
||||||
|
'channel_follower_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'age_limit': 0,
|
||||||
|
'duration': 223,
|
||||||
|
'thumbnail': 'https://i.ytimg.com/vi_webp/DbCvuSGfR3Y/maxresdefault.webp',
|
||||||
|
'heatmap': 'count:100',
|
||||||
|
'categories': ['Music'],
|
||||||
|
'tags': ['half·alive', 'Conditions Of A Punk', 'Back Around'],
|
||||||
|
'creators': ['half·alive'],
|
||||||
|
'timestamp': 1669889281,
|
||||||
|
'upload_date': '20221201',
|
||||||
|
'playable_in_embed': True,
|
||||||
|
'availability': 'public',
|
||||||
|
'live_status': 'not_live',
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
# Video with two collaborators
|
||||||
|
'url': 'https://www.youtube.com/watch?v=brhfDfLdDZ8',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'brhfDfLdDZ8',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'This is the WORST Movie Science We\'ve Ever Seen',
|
||||||
|
'description': 'md5:8afd0a3cd69ec63438fc573580436f92',
|
||||||
|
'media_type': 'video',
|
||||||
|
'uploader': 'Open Sauce',
|
||||||
|
'uploader_id': '@opensaucelive',
|
||||||
|
'uploader_url': 'https://www.youtube.com/@opensaucelive',
|
||||||
|
'channel': 'Open Sauce',
|
||||||
|
'channel_id': 'UC2EiGVmCeD79l_vZ204DUSw',
|
||||||
|
'channel_url': 'https://www.youtube.com/channel/UC2EiGVmCeD79l_vZ204DUSw',
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'age_limit': 0,
|
||||||
|
'duration': 1664,
|
||||||
|
'thumbnail': 'https://i.ytimg.com/vi/brhfDfLdDZ8/hqdefault.jpg',
|
||||||
|
'categories': ['Entertainment'],
|
||||||
|
'tags': ['Moonfall', 'Bad Science', 'Open Sauce', 'Sauce+', 'The Backyard Scientist', 'William Osman', 'Allen Pan'],
|
||||||
|
'creators': ['Open Sauce', 'William Osman 2'],
|
||||||
|
'timestamp': 1759452918,
|
||||||
|
'upload_date': '20251003',
|
||||||
|
'playable_in_embed': True,
|
||||||
|
'availability': 'public',
|
||||||
|
'live_status': 'not_live',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': True},
|
||||||
|
}, {
|
||||||
|
# Video with five collaborators
|
||||||
|
'url': 'https://www.youtube.com/watch?v=_A9KsMbWh4E',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '_A9KsMbWh4E',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '【MV】薫習 - LIVE UNION【RK Music】',
|
||||||
|
'description': 'md5:9b3dc2b91103f303fcc0dac8617e7938',
|
||||||
|
'media_type': 'video',
|
||||||
|
'uploader': 'RK Music',
|
||||||
|
'uploader_id': '@RKMusic_inc',
|
||||||
|
'uploader_url': 'https://www.youtube.com/@RKMusic_inc',
|
||||||
|
'channel': 'RK Music',
|
||||||
|
'channel_id': 'UCiLhMk-gmE2zgF7KGVyqvFw',
|
||||||
|
'channel_url': 'https://www.youtube.com/channel/UCiLhMk-gmE2zgF7KGVyqvFw',
|
||||||
|
'comment_count': int,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'age_limit': 0,
|
||||||
|
'duration': 193,
|
||||||
|
'thumbnail': 'https://i.ytimg.com/vi_webp/_A9KsMbWh4E/maxresdefault.webp',
|
||||||
|
'categories': ['Music'],
|
||||||
|
'tags': [],
|
||||||
|
'creators': ['RK Music', 'HACHI', '焔魔るり CH. / Ruri Enma', '瀬戸乃とと', '水瀬 凪/MINASE Nagi'],
|
||||||
|
'timestamp': 1761908406,
|
||||||
|
'upload_date': '20251031',
|
||||||
|
'release_timestamp': 1761908406,
|
||||||
|
'release_date': '20251031',
|
||||||
|
'playable_in_embed': True,
|
||||||
|
'availability': 'public',
|
||||||
|
'live_status': 'not_live',
|
||||||
|
},
|
||||||
|
'params': {'skip_download': True},
|
||||||
}]
|
}]
|
||||||
_WEBPAGE_TESTS = [{
|
_WEBPAGE_TESTS = [{
|
||||||
# <object>
|
# <object>
|
||||||
|
|
@ -3023,8 +3127,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
|
|
||||||
def _extract_formats_and_subtitles(self, video_id, player_responses, player_url, live_status, duration):
|
def _extract_formats_and_subtitles(self, video_id, player_responses, player_url, live_status, duration):
|
||||||
CHUNK_SIZE = 10 << 20
|
CHUNK_SIZE = 10 << 20
|
||||||
PREFERRED_LANG_VALUE = 10
|
ORIGINAL_LANG_VALUE = 10
|
||||||
original_language = None
|
DEFAULT_LANG_VALUE = 5
|
||||||
|
language_map = {
|
||||||
|
ORIGINAL_LANG_VALUE: None,
|
||||||
|
DEFAULT_LANG_VALUE: None,
|
||||||
|
}
|
||||||
itags, stream_ids = collections.defaultdict(set), []
|
itags, stream_ids = collections.defaultdict(set), []
|
||||||
itag_qualities, res_qualities = {}, {0: None}
|
itag_qualities, res_qualities = {}, {0: None}
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
|
|
@ -3042,6 +3150,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
self._downloader.deprecated_feature('[youtube] include_duplicate_formats extractor argument is deprecated. '
|
self._downloader.deprecated_feature('[youtube] include_duplicate_formats extractor argument is deprecated. '
|
||||||
'Use formats=duplicate extractor argument instead')
|
'Use formats=duplicate extractor argument instead')
|
||||||
|
|
||||||
|
def is_super_resolution(f_url):
|
||||||
|
return '1' in traverse_obj(f_url, ({parse_qs}, 'xtags', ..., {urllib.parse.parse_qs}, 'sr', ...))
|
||||||
|
|
||||||
def solve_sig(s, spec):
|
def solve_sig(s, spec):
|
||||||
return ''.join(s[i] for i in spec)
|
return ''.join(s[i] for i in spec)
|
||||||
|
|
||||||
|
|
@ -3064,6 +3175,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
# For handling potential pre-playback required waiting period
|
# For handling potential pre-playback required waiting period
|
||||||
playback_wait = int_or_none(self._configuration_arg('playback_wait', [None])[0], default=6)
|
playback_wait = int_or_none(self._configuration_arg('playback_wait', [None])[0], default=6)
|
||||||
|
|
||||||
|
def get_language_code_and_preference(fmt_stream):
|
||||||
|
audio_track = fmt_stream.get('audioTrack') or {}
|
||||||
|
display_name = audio_track.get('displayName') or ''
|
||||||
|
language_code = audio_track.get('id', '').split('.')[0] or None
|
||||||
|
if 'descriptive' in display_name.lower():
|
||||||
|
return join_nonempty(language_code, 'desc'), -10
|
||||||
|
if 'original' in display_name.lower():
|
||||||
|
if language_code and not language_map.get(ORIGINAL_LANG_VALUE):
|
||||||
|
language_map[ORIGINAL_LANG_VALUE] = language_code
|
||||||
|
return language_code, ORIGINAL_LANG_VALUE
|
||||||
|
if audio_track.get('audioIsDefault'):
|
||||||
|
if language_code and not language_map.get(DEFAULT_LANG_VALUE):
|
||||||
|
language_map[DEFAULT_LANG_VALUE] = language_code
|
||||||
|
return language_code, DEFAULT_LANG_VALUE
|
||||||
|
return language_code, -1
|
||||||
|
|
||||||
for pr in player_responses:
|
for pr in player_responses:
|
||||||
streaming_data = traverse_obj(pr, 'streamingData')
|
streaming_data = traverse_obj(pr, 'streamingData')
|
||||||
if not streaming_data:
|
if not streaming_data:
|
||||||
|
|
@ -3078,8 +3205,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
def get_stream_id(fmt_stream):
|
def get_stream_id(fmt_stream):
|
||||||
return str_or_none(fmt_stream.get('itag')), traverse_obj(fmt_stream, 'audioTrack', 'id'), fmt_stream.get('isDrc')
|
return str_or_none(fmt_stream.get('itag')), traverse_obj(fmt_stream, 'audioTrack', 'id'), fmt_stream.get('isDrc')
|
||||||
|
|
||||||
def process_format_stream(fmt_stream, proto, missing_pot):
|
def process_format_stream(fmt_stream, proto, missing_pot, super_resolution=False):
|
||||||
nonlocal original_language
|
|
||||||
itag = str_or_none(fmt_stream.get('itag'))
|
itag = str_or_none(fmt_stream.get('itag'))
|
||||||
audio_track = fmt_stream.get('audioTrack') or {}
|
audio_track = fmt_stream.get('audioTrack') or {}
|
||||||
quality = fmt_stream.get('quality')
|
quality = fmt_stream.get('quality')
|
||||||
|
|
@ -3096,13 +3222,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
if height:
|
if height:
|
||||||
res_qualities[height] = quality
|
res_qualities[height] = quality
|
||||||
|
|
||||||
display_name = audio_track.get('displayName') or ''
|
language_code, language_preference = get_language_code_and_preference(fmt_stream)
|
||||||
is_original = 'original' in display_name.lower()
|
|
||||||
is_descriptive = 'descriptive' in display_name.lower()
|
|
||||||
is_default = audio_track.get('audioIsDefault')
|
|
||||||
language_code = audio_track.get('id', '').split('.')[0]
|
|
||||||
if language_code and (is_original or (is_default and not original_language)):
|
|
||||||
original_language = language_code
|
|
||||||
|
|
||||||
has_drm = bool(fmt_stream.get('drmFamilies'))
|
has_drm = bool(fmt_stream.get('drmFamilies'))
|
||||||
|
|
||||||
|
|
@ -3136,10 +3256,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
dct = {
|
dct = {
|
||||||
'asr': int_or_none(fmt_stream.get('audioSampleRate')),
|
'asr': int_or_none(fmt_stream.get('audioSampleRate')),
|
||||||
'filesize': int_or_none(fmt_stream.get('contentLength')),
|
'filesize': int_or_none(fmt_stream.get('contentLength')),
|
||||||
'format_id': f'{itag}{"-drc" if fmt_stream.get("isDrc") else ""}',
|
'format_id': join_nonempty(itag, (
|
||||||
|
'drc' if fmt_stream.get('isDrc')
|
||||||
|
else 'sr' if super_resolution
|
||||||
|
else None)),
|
||||||
'format_note': join_nonempty(
|
'format_note': join_nonempty(
|
||||||
join_nonempty(display_name, is_default and ' (default)', delim=''),
|
join_nonempty(audio_track.get('displayName'), audio_track.get('audioIsDefault') and '(default)', delim=' '),
|
||||||
name, fmt_stream.get('isDrc') and 'DRC',
|
name, fmt_stream.get('isDrc') and 'DRC', super_resolution and 'AI-upscaled',
|
||||||
try_get(fmt_stream, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
|
try_get(fmt_stream, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
|
||||||
try_get(fmt_stream, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
|
try_get(fmt_stream, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
|
||||||
is_damaged and 'DAMAGED', missing_pot and 'MISSING POT',
|
is_damaged and 'DAMAGED', missing_pot and 'MISSING POT',
|
||||||
|
|
@ -3155,8 +3278,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
'tbr': tbr,
|
'tbr': tbr,
|
||||||
'filesize_approx': filesize_from_tbr(tbr, format_duration),
|
'filesize_approx': filesize_from_tbr(tbr, format_duration),
|
||||||
'width': int_or_none(fmt_stream.get('width')),
|
'width': int_or_none(fmt_stream.get('width')),
|
||||||
'language': join_nonempty(language_code, 'desc' if is_descriptive else '') or None,
|
'language': language_code,
|
||||||
'language_preference': PREFERRED_LANG_VALUE if is_original else 5 if is_default else -10 if is_descriptive else -1,
|
'language_preference': language_preference,
|
||||||
# Strictly de-prioritize damaged and 3gp formats
|
# Strictly de-prioritize damaged and 3gp formats
|
||||||
'preference': -10 if is_damaged else -2 if itag == '17' else None,
|
'preference': -10 if is_damaged else -2 if itag == '17' else None,
|
||||||
}
|
}
|
||||||
|
|
@ -3206,6 +3329,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
fmt_url = fmt_stream.get('url')
|
fmt_url = fmt_stream.get('url')
|
||||||
encrypted_sig, sc = None, None
|
encrypted_sig, sc = None, None
|
||||||
if not fmt_url:
|
if not fmt_url:
|
||||||
|
# We still need to register original/default language information
|
||||||
|
# See: https://github.com/yt-dlp/yt-dlp/issues/14883
|
||||||
|
get_language_code_and_preference(fmt_stream)
|
||||||
sc = urllib.parse.parse_qs(fmt_stream.get('signatureCipher'))
|
sc = urllib.parse.parse_qs(fmt_stream.get('signatureCipher'))
|
||||||
fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
|
fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
|
||||||
encrypted_sig = try_get(sc, lambda x: x['s'][0])
|
encrypted_sig = try_get(sc, lambda x: x['s'][0])
|
||||||
|
|
@ -3222,7 +3348,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
self.report_warning(msg, video_id, only_once=True)
|
self.report_warning(msg, video_id, only_once=True)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
fmt = process_format_stream(fmt_stream, proto, missing_pot=require_po_token and not po_token)
|
fmt = process_format_stream(
|
||||||
|
fmt_stream, proto, missing_pot=require_po_token and not po_token,
|
||||||
|
super_resolution=is_super_resolution(fmt_url))
|
||||||
if not fmt:
|
if not fmt:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
@ -3391,9 +3519,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
elif itag:
|
elif itag:
|
||||||
f['format_id'] = itag
|
f['format_id'] = itag
|
||||||
|
|
||||||
if original_language and f.get('language') == original_language:
|
lang_code = f.get('language')
|
||||||
|
if lang_code and lang_code == language_map[ORIGINAL_LANG_VALUE]:
|
||||||
|
f['format_note'] = join_nonempty(f.get('format_note'), '(original)', delim=' ')
|
||||||
|
f['language_preference'] = ORIGINAL_LANG_VALUE
|
||||||
|
elif lang_code and lang_code == language_map[DEFAULT_LANG_VALUE]:
|
||||||
f['format_note'] = join_nonempty(f.get('format_note'), '(default)', delim=' ')
|
f['format_note'] = join_nonempty(f.get('format_note'), '(default)', delim=' ')
|
||||||
f['language_preference'] = PREFERRED_LANG_VALUE
|
f['language_preference'] = DEFAULT_LANG_VALUE
|
||||||
|
|
||||||
if itag in ('616', '235'):
|
if itag in ('616', '235'):
|
||||||
f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
|
f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
|
||||||
|
|
@ -3988,20 +4120,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
|
|
||||||
# Youtube Music Auto-generated description
|
# Youtube Music Auto-generated description
|
||||||
if (video_description or '').strip().endswith('\nAuto-generated by YouTube.'):
|
if (video_description or '').strip().endswith('\nAuto-generated by YouTube.'):
|
||||||
# XXX: Causes catastrophic backtracking if description has "·"
|
|
||||||
# E.g. https://www.youtube.com/watch?v=DoPaAxMQoiI
|
|
||||||
# Simulating atomic groups: (?P<a>[^xy]+)x => (?=(?P<a>[^xy]+))(?P=a)x
|
|
||||||
# reduces it, but does not fully fix it. https://regex101.com/r/8Ssf2h/2
|
|
||||||
mobj = re.search(
|
mobj = re.search(
|
||||||
r'''(?xs)
|
r'''(?xs)
|
||||||
(?=(?P<track>[^\n·]+))(?P=track)·
|
(?:\n|^)(?P<track>[^\n·]+)\ ·\ (?P<artist>[^\n]+)\n+
|
||||||
(?=(?P<artist>[^\n]+))(?P=artist)\n+
|
(?P<album>[^\n]+)\n+
|
||||||
(?=(?P<album>[^\n]+))(?P=album)\n
|
(?:℗\s*(?P<release_year>\d{4}))?
|
||||||
(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
|
(?:.+?\nReleased\ on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
|
||||||
(?:.+?Released\ on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
|
(?:.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+)\n)?
|
||||||
(.+?\nArtist\s*:\s*
|
.+\nAuto-generated\ by\ YouTube\.\s*$
|
||||||
(?=(?P<clean_artist>[^\n]+))(?P=clean_artist)\n
|
|
||||||
)?.+\nAuto-generated\ by\ YouTube\.\s*$
|
|
||||||
''', video_description)
|
''', video_description)
|
||||||
if mobj:
|
if mobj:
|
||||||
release_year = mobj.group('release_year')
|
release_year = mobj.group('release_year')
|
||||||
|
|
@ -4013,7 +4139,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
info.update({
|
info.update({
|
||||||
'album': mobj.group('album'.strip()),
|
'album': mobj.group('album'.strip()),
|
||||||
'artists': ([a] if (a := mobj.group('clean_artist'))
|
'artists': ([a] if (a := mobj.group('clean_artist'))
|
||||||
else [a.strip() for a in mobj.group('artist').split('·')]),
|
else [a.strip() for a in mobj.group('artist').split(' · ')]),
|
||||||
'track': mobj.group('track').strip(),
|
'track': mobj.group('track').strip(),
|
||||||
'release_date': release_date,
|
'release_date': release_date,
|
||||||
'release_year': int_or_none(release_year),
|
'release_year': int_or_none(release_year),
|
||||||
|
|
@ -4112,9 +4238,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||||
vsir = get_first(contents, 'videoSecondaryInfoRenderer')
|
vsir = get_first(contents, 'videoSecondaryInfoRenderer')
|
||||||
if vsir:
|
if vsir:
|
||||||
vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
|
vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
|
||||||
|
collaborators = traverse_obj(vor, (
|
||||||
|
'attributedTitle', 'commandRuns', ..., 'onTap', 'innertubeCommand', 'showDialogCommand',
|
||||||
|
'panelLoadingStrategy', 'inlineContent', 'dialogViewModel', 'customContent', 'listViewModel',
|
||||||
|
'listItems', ..., 'listItemViewModel', 'title', 'content', {str}))
|
||||||
info.update({
|
info.update({
|
||||||
'channel': self._get_text(vor, 'title'),
|
'channel': self._get_text(vor, 'title') or (collaborators[0] if collaborators else None),
|
||||||
'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
|
'channel_follower_count': self._get_count(vor, 'subscriberCountText'),
|
||||||
|
'creators': collaborators if collaborators else None,
|
||||||
|
})
|
||||||
|
|
||||||
if not channel_handle:
|
if not channel_handle:
|
||||||
channel_handle = self.handle_from_url(
|
channel_handle = self.handle_from_url(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
||||||
|
|
||||||
VERSION = '0.3.0'
|
VERSION = '0.3.1'
|
||||||
HASHES = {
|
HASHES = {
|
||||||
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
|
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
|
||||||
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
|
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,10 @@ class CurlCFFIResponseAdapter(Response):
|
||||||
|
|
||||||
def read(self, amt=None):
|
def read(self, amt=None):
|
||||||
try:
|
try:
|
||||||
return self.fp.read(amt)
|
res = self.fp.read(amt)
|
||||||
|
if self.fp.closed:
|
||||||
|
self.close()
|
||||||
|
return res
|
||||||
except curl_cffi.requests.errors.RequestsError as e:
|
except curl_cffi.requests.errors.RequestsError as e:
|
||||||
if e.code == CurlECode.PARTIAL_FILE:
|
if e.code == CurlECode.PARTIAL_FILE:
|
||||||
content_length = e.response and int_or_none(e.response.headers.get('Content-Length'))
|
content_length = e.response and int_or_none(e.response.headers.get('Content-Length'))
|
||||||
|
|
|
||||||
|
|
@ -119,17 +119,22 @@ class RequestsResponseAdapter(Response):
|
||||||
|
|
||||||
self._requests_response = res
|
self._requests_response = res
|
||||||
|
|
||||||
|
def _real_read(self, amt: int | None = None) -> bytes:
|
||||||
|
# Work around issue with `.read(amt)` then `.read()`
|
||||||
|
# See: https://github.com/urllib3/urllib3/issues/3636
|
||||||
|
if amt is None:
|
||||||
|
# compat: py3.9: Python 3.9 preallocates the whole read buffer, read in chunks
|
||||||
|
read_chunk = functools.partial(self.fp.read, 1 << 20, decode_content=True)
|
||||||
|
return b''.join(iter(read_chunk, b''))
|
||||||
|
# Interact with urllib3 response directly.
|
||||||
|
return self.fp.read(amt, decode_content=True)
|
||||||
|
|
||||||
def read(self, amt: int | None = None):
|
def read(self, amt: int | None = None):
|
||||||
try:
|
try:
|
||||||
# Work around issue with `.read(amt)` then `.read()`
|
data = self._real_read(amt)
|
||||||
# See: https://github.com/urllib3/urllib3/issues/3636
|
if self.fp.closed:
|
||||||
if amt is None:
|
self.close()
|
||||||
# compat: py3.9: Python 3.9 preallocates the whole read buffer, read in chunks
|
return data
|
||||||
read_chunk = functools.partial(self.fp.read, 1 << 20, decode_content=True)
|
|
||||||
return b''.join(iter(read_chunk, b''))
|
|
||||||
# Interact with urllib3 response directly.
|
|
||||||
return self.fp.read(amt, decode_content=True)
|
|
||||||
|
|
||||||
# See urllib3.response.HTTPResponse.read() for exceptions raised on read
|
# See urllib3.response.HTTPResponse.read() for exceptions raised on read
|
||||||
except urllib3.exceptions.SSLError as e:
|
except urllib3.exceptions.SSLError as e:
|
||||||
raise SSLError(cause=e) from e
|
raise SSLError(cause=e) from e
|
||||||
|
|
|
||||||
|
|
@ -305,8 +305,28 @@ class UrllibResponseAdapter(Response):
|
||||||
status=getattr(res, 'status', None) or res.getcode(), reason=getattr(res, 'reason', None))
|
status=getattr(res, 'status', None) or res.getcode(), reason=getattr(res, 'reason', None))
|
||||||
|
|
||||||
def read(self, amt=None):
|
def read(self, amt=None):
|
||||||
|
if self.closed:
|
||||||
|
return b''
|
||||||
try:
|
try:
|
||||||
return self.fp.read(amt)
|
data = self.fp.read(amt)
|
||||||
|
underlying = getattr(self.fp, 'fp', None)
|
||||||
|
if isinstance(self.fp, http.client.HTTPResponse) and underlying is None:
|
||||||
|
# http.client.HTTPResponse automatically closes itself when fully read
|
||||||
|
self.close()
|
||||||
|
elif isinstance(self.fp, urllib.response.addinfourl) and underlying is not None:
|
||||||
|
# urllib's addinfourl does not close the underlying fp automatically when fully read
|
||||||
|
if isinstance(underlying, io.BytesIO):
|
||||||
|
# data URLs or in-memory responses (e.g. gzip/deflate/brotli decoded)
|
||||||
|
if underlying.tell() >= len(underlying.getbuffer()):
|
||||||
|
self.close()
|
||||||
|
elif isinstance(underlying, io.BufferedReader) and amt is None:
|
||||||
|
# file URLs.
|
||||||
|
# XXX: this will not mark the response as closed if it was fully read with amt.
|
||||||
|
self.close()
|
||||||
|
elif underlying is not None and underlying.closed:
|
||||||
|
# Catch-all for any cases where underlying file is closed
|
||||||
|
self.close()
|
||||||
|
return data
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
handle_response_read_exceptions(e)
|
handle_response_read_exceptions(e)
|
||||||
raise e
|
raise e
|
||||||
|
|
|
||||||
|
|
@ -554,12 +554,16 @@ class Response(io.IOBase):
|
||||||
# Expected errors raised here should be of type RequestError or subclasses.
|
# Expected errors raised here should be of type RequestError or subclasses.
|
||||||
# Subclasses should redefine this method with more precise error handling.
|
# Subclasses should redefine this method with more precise error handling.
|
||||||
try:
|
try:
|
||||||
return self.fp.read(amt)
|
res = self.fp.read(amt)
|
||||||
|
if self.fp.closed:
|
||||||
|
self.close()
|
||||||
|
return res
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise TransportError(cause=e) from e
|
raise TransportError(cause=e) from e
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
self.fp.close()
|
if not self.fp.closed:
|
||||||
|
self.fp.close()
|
||||||
return super().close()
|
return super().close()
|
||||||
|
|
||||||
def get_header(self, name, default=None):
|
def get_header(self, name, default=None):
|
||||||
|
|
|
||||||
|
|
@ -441,7 +441,7 @@ def create_parser():
|
||||||
'("-" for stdin). Can be used multiple times and inside other configuration files'))
|
'("-" for stdin). Can be used multiple times and inside other configuration files'))
|
||||||
general.add_option(
|
general.add_option(
|
||||||
'--plugin-dirs',
|
'--plugin-dirs',
|
||||||
metavar='PATH',
|
metavar='DIR',
|
||||||
dest='plugin_dirs',
|
dest='plugin_dirs',
|
||||||
action='callback',
|
action='callback',
|
||||||
callback=_list_from_options_callback,
|
callback=_list_from_options_callback,
|
||||||
|
|
@ -466,9 +466,13 @@ def create_parser():
|
||||||
callback_kwargs={'delim': None},
|
callback_kwargs={'delim': None},
|
||||||
default=['deno'],
|
default=['deno'],
|
||||||
help=(
|
help=(
|
||||||
'Additional JavaScript runtime to enable, with an optional path to the runtime location. '
|
'Additional JavaScript runtime to enable, with an optional location for the runtime '
|
||||||
|
'(either the path to the binary or its containing directory). '
|
||||||
'This option can be used multiple times to enable multiple runtimes. '
|
'This option can be used multiple times to enable multiple runtimes. '
|
||||||
'Supported runtimes: deno, node, bun, quickjs. By default, only "deno" runtime is enabled.'))
|
'Supported runtimes are (in order of priority, from highest to lowest): deno, node, quickjs, bun. '
|
||||||
|
'Only "deno" is enabled by default. The highest priority runtime that is both enabled and '
|
||||||
|
'available will be used. In order to use a lower priority runtime when "deno" is available, '
|
||||||
|
'--no-js-runtimes needs to be passed before enabling other runtimes'))
|
||||||
general.add_option(
|
general.add_option(
|
||||||
'--no-js-runtimes',
|
'--no-js-runtimes',
|
||||||
dest='js_runtimes', action='store_const', const=[],
|
dest='js_runtimes', action='store_const', const=[],
|
||||||
|
|
@ -484,9 +488,12 @@ def create_parser():
|
||||||
default=[],
|
default=[],
|
||||||
help=(
|
help=(
|
||||||
'Remote components to allow yt-dlp to fetch when required. '
|
'Remote components to allow yt-dlp to fetch when required. '
|
||||||
|
'This option is currently not needed if you are using an official executable '
|
||||||
|
'or have the requisite version of the yt-dlp-ejs package installed. '
|
||||||
'You can use this option multiple times to allow multiple components. '
|
'You can use this option multiple times to allow multiple components. '
|
||||||
'Supported values: ejs:npm (external JavaScript components from npm), ejs:github (external JavaScript components from yt-dlp-ejs GitHub). '
|
'Supported values: ejs:npm (external JavaScript components from npm), '
|
||||||
'By default, no remote components are allowed.'))
|
'ejs:github (external JavaScript components from yt-dlp-ejs GitHub). '
|
||||||
|
'By default, no remote components are allowed'))
|
||||||
general.add_option(
|
general.add_option(
|
||||||
'--no-remote-components',
|
'--no-remote-components',
|
||||||
dest='remote_components', action='store_const', const=[],
|
dest='remote_components', action='store_const', const=[],
|
||||||
|
|
|
||||||
|
|
@ -192,7 +192,10 @@ class FFmpegPostProcessor(PostProcessor):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def available(self):
|
def available(self):
|
||||||
return bool(self._ffmpeg_location.get()) or self.basename is not None
|
# If we return that ffmpeg is available, then the basename property *must* be run
|
||||||
|
# (as doing so has side effects), and its value can never be None
|
||||||
|
# See: https://github.com/yt-dlp/yt-dlp/issues/12829
|
||||||
|
return self.basename is not None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def executable(self):
|
def executable(self):
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
import abc
|
import abc
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import functools
|
import functools
|
||||||
|
import os.path
|
||||||
|
|
||||||
from ._utils import _get_exe_version_output, detect_exe_version, int_or_none
|
from ._utils import _get_exe_version_output, detect_exe_version, int_or_none
|
||||||
|
|
||||||
|
|
@ -12,6 +13,14 @@ def runtime_version_tuple(v):
|
||||||
return tuple(int_or_none(x, default=0) for x in v.split('.'))
|
return tuple(int_or_none(x, default=0) for x in v.split('.'))
|
||||||
|
|
||||||
|
|
||||||
|
def _determine_runtime_path(path, basename):
|
||||||
|
if not path:
|
||||||
|
return basename
|
||||||
|
if os.path.isdir(path):
|
||||||
|
return os.path.join(path, basename)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
@dataclasses.dataclass(frozen=True)
|
||||||
class JsRuntimeInfo:
|
class JsRuntimeInfo:
|
||||||
name: str
|
name: str
|
||||||
|
|
@ -38,7 +47,7 @@ class DenoJsRuntime(JsRuntime):
|
||||||
MIN_SUPPORTED_VERSION = (2, 0, 0)
|
MIN_SUPPORTED_VERSION = (2, 0, 0)
|
||||||
|
|
||||||
def _info(self):
|
def _info(self):
|
||||||
path = self._path or 'deno'
|
path = _determine_runtime_path(self._path, 'deno')
|
||||||
out = _get_exe_version_output(path, ['--version'])
|
out = _get_exe_version_output(path, ['--version'])
|
||||||
if not out:
|
if not out:
|
||||||
return None
|
return None
|
||||||
|
|
@ -53,7 +62,7 @@ class BunJsRuntime(JsRuntime):
|
||||||
MIN_SUPPORTED_VERSION = (1, 0, 31)
|
MIN_SUPPORTED_VERSION = (1, 0, 31)
|
||||||
|
|
||||||
def _info(self):
|
def _info(self):
|
||||||
path = self._path or 'bun'
|
path = _determine_runtime_path(self._path, 'bun')
|
||||||
out = _get_exe_version_output(path, ['--version'])
|
out = _get_exe_version_output(path, ['--version'])
|
||||||
if not out:
|
if not out:
|
||||||
return None
|
return None
|
||||||
|
|
@ -68,7 +77,7 @@ class NodeJsRuntime(JsRuntime):
|
||||||
MIN_SUPPORTED_VERSION = (20, 0, 0)
|
MIN_SUPPORTED_VERSION = (20, 0, 0)
|
||||||
|
|
||||||
def _info(self):
|
def _info(self):
|
||||||
path = self._path or 'node'
|
path = _determine_runtime_path(self._path, 'node')
|
||||||
out = _get_exe_version_output(path, ['--version'])
|
out = _get_exe_version_output(path, ['--version'])
|
||||||
if not out:
|
if not out:
|
||||||
return None
|
return None
|
||||||
|
|
@ -83,7 +92,7 @@ class QuickJsRuntime(JsRuntime):
|
||||||
MIN_SUPPORTED_VERSION = (2023, 12, 9)
|
MIN_SUPPORTED_VERSION = (2023, 12, 9)
|
||||||
|
|
||||||
def _info(self):
|
def _info(self):
|
||||||
path = self._path or 'qjs'
|
path = _determine_runtime_path(self._path, 'qjs')
|
||||||
# quickjs does not have --version and --help returns a status code of 1
|
# quickjs does not have --version and --help returns a status code of 1
|
||||||
out = _get_exe_version_output(path, ['--help'], ignore_return_code=True)
|
out = _get_exe_version_output(path, ['--help'], ignore_return_code=True)
|
||||||
if not out:
|
if not out:
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
# Autogenerated by devscripts/update-version.py
|
# Autogenerated by devscripts/update-version.py
|
||||||
|
|
||||||
__version__ = '2025.10.22'
|
__version__ = '2025.11.12'
|
||||||
|
|
||||||
RELEASE_GIT_HEAD = 'c9356f308dd3c5f9f494cb40ed14c5df017b4fe0'
|
RELEASE_GIT_HEAD = '335653be82d5ef999cfc2879d005397402eebec1'
|
||||||
|
|
||||||
VARIANT = None
|
VARIANT = None
|
||||||
|
|
||||||
|
|
@ -12,4 +12,4 @@ CHANNEL = 'stable'
|
||||||
|
|
||||||
ORIGIN = 'yt-dlp/yt-dlp'
|
ORIGIN = 'yt-dlp/yt-dlp'
|
||||||
|
|
||||||
_pkg_version = '2025.10.22'
|
_pkg_version = '2025.11.12'
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue