mirror of https://github.com/yt-dlp/yt-dlp.git
4278 lines
201 KiB
Python
4278 lines
201 KiB
Python
import collections
|
|
import contextlib
|
|
import copy
|
|
import datetime
|
|
import errno
|
|
import fileinput
|
|
import http.cookiejar
|
|
import io
|
|
import itertools
|
|
import json
|
|
import locale
|
|
import operator
|
|
import os
|
|
import random
|
|
import re
|
|
import shutil
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import tokenize
|
|
import traceback
|
|
import unicodedata
|
|
|
|
from .cache import Cache
|
|
from .compat import functools, urllib # isort: split
|
|
from .compat import compat_os_name, compat_shlex_quote, urllib_req_to_req
|
|
from .cookies import LenientSimpleCookie, load_cookies
|
|
from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
|
|
from .downloader.rtmp import rtmpdump_version
|
|
from .extractor import gen_extractor_classes, get_info_extractor
|
|
from .extractor.common import UnsupportedURLIE
|
|
from .extractor.openload import PhantomJSwrapper
|
|
from .minicurses import format_text
|
|
from .networking import HEADRequest, Request, RequestDirector
|
|
from .networking.common import _REQUEST_HANDLERS, _RH_PREFERENCES
|
|
from .networking.exceptions import (
|
|
HTTPError,
|
|
NoSupportingHandlers,
|
|
RequestError,
|
|
SSLError,
|
|
_CompatHTTPError,
|
|
network_exceptions,
|
|
)
|
|
from .plugins import directories as plugin_directories
|
|
from .postprocessor import _PLUGIN_CLASSES as plugin_pps
|
|
from .postprocessor import (
|
|
EmbedThumbnailPP,
|
|
FFmpegFixupDuplicateMoovPP,
|
|
FFmpegFixupDurationPP,
|
|
FFmpegFixupM3u8PP,
|
|
FFmpegFixupM4aPP,
|
|
FFmpegFixupStretchedPP,
|
|
FFmpegFixupTimestampPP,
|
|
FFmpegMergerPP,
|
|
FFmpegPostProcessor,
|
|
FFmpegVideoConvertorPP,
|
|
MoveFilesAfterDownloadPP,
|
|
get_postprocessor,
|
|
)
|
|
from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
|
|
from .update import REPOSITORY, current_git_head, detect_variant
|
|
from .utils import (
|
|
DEFAULT_OUTTMPL,
|
|
IDENTITY,
|
|
LINK_TEMPLATES,
|
|
MEDIA_EXTENSIONS,
|
|
NO_DEFAULT,
|
|
NUMBER_RE,
|
|
OUTTMPL_TYPES,
|
|
POSTPROCESS_WHEN,
|
|
STR_FORMAT_RE_TMPL,
|
|
STR_FORMAT_TYPES,
|
|
ContentTooShortError,
|
|
DateRange,
|
|
DownloadCancelled,
|
|
DownloadError,
|
|
EntryNotInPlaylist,
|
|
ExistingVideoReached,
|
|
ExtractorError,
|
|
FormatSorter,
|
|
GeoRestrictedError,
|
|
ISO3166Utils,
|
|
LazyList,
|
|
MaxDownloadsReached,
|
|
Namespace,
|
|
PagedList,
|
|
PlaylistEntries,
|
|
Popen,
|
|
PostProcessingError,
|
|
ReExtractInfo,
|
|
RejectedVideoReached,
|
|
SameFileError,
|
|
UnavailableVideoError,
|
|
UserNotLive,
|
|
age_restricted,
|
|
args_to_str,
|
|
bug_reports_message,
|
|
date_from_str,
|
|
deprecation_warning,
|
|
determine_ext,
|
|
determine_protocol,
|
|
encode_compat_str,
|
|
encodeFilename,
|
|
error_to_compat_str,
|
|
escapeHTML,
|
|
expand_path,
|
|
extract_basic_auth,
|
|
filter_dict,
|
|
float_or_none,
|
|
format_bytes,
|
|
format_decimal_suffix,
|
|
format_field,
|
|
formatSeconds,
|
|
get_compatible_ext,
|
|
get_domain,
|
|
int_or_none,
|
|
iri_to_uri,
|
|
is_path_like,
|
|
join_nonempty,
|
|
locked_file,
|
|
make_archive_id,
|
|
make_dir,
|
|
number_of_digits,
|
|
orderedSet,
|
|
orderedSet_from_options,
|
|
parse_filesize,
|
|
preferredencoding,
|
|
prepend_extension,
|
|
remove_terminal_sequences,
|
|
render_table,
|
|
replace_extension,
|
|
sanitize_filename,
|
|
sanitize_path,
|
|
sanitize_url,
|
|
str_or_none,
|
|
strftime_or_none,
|
|
subtitles_filename,
|
|
supports_terminal_sequences,
|
|
system_identifier,
|
|
timetuple_from_msec,
|
|
to_high_limit_path,
|
|
traverse_obj,
|
|
try_call,
|
|
try_get,
|
|
url_basename,
|
|
variadic,
|
|
version_tuple,
|
|
windows_enable_vt_mode,
|
|
write_json_file,
|
|
write_string,
|
|
)
|
|
from .utils._utils import _YDLLogger
|
|
from .utils.networking import (
|
|
HTTPHeaderDict,
|
|
clean_headers,
|
|
clean_proxies,
|
|
std_headers,
|
|
)
|
|
from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
|
|
|
|
if compat_os_name == 'nt':
|
|
import ctypes
|
|
|
|
|
|
class YoutubeDL:
|
|
"""YoutubeDL class.
|
|
|
|
YoutubeDL objects are the ones responsible of downloading the
|
|
actual video file and writing it to disk if the user has requested
|
|
it, among some other tasks. In most cases there should be one per
|
|
program. As, given a video URL, the downloader doesn't know how to
|
|
extract all the needed information, task that InfoExtractors do, it
|
|
has to pass the URL to one of them.
|
|
|
|
For this, YoutubeDL objects have a method that allows
|
|
InfoExtractors to be registered in a given order. When it is passed
|
|
a URL, the YoutubeDL object handles it to the first InfoExtractor it
|
|
finds that reports being able to handle it. The InfoExtractor extracts
|
|
all the information about the video or videos the URL refers to, and
|
|
YoutubeDL process the extracted information, possibly using a File
|
|
Downloader to download the video.
|
|
|
|
YoutubeDL objects accept a lot of parameters. In order not to saturate
|
|
the object constructor with arguments, it receives a dictionary of
|
|
options instead. These options are available through the params
|
|
attribute for the InfoExtractors to use. The YoutubeDL also
|
|
registers itself as the downloader in charge for the InfoExtractors
|
|
that are added to it, so this is a "mutual registration".
|
|
|
|
Available options:
|
|
|
|
username: Username for authentication purposes.
|
|
password: Password for authentication purposes.
|
|
videopassword: Password for accessing a video.
|
|
ap_mso: Adobe Pass multiple-system operator identifier.
|
|
ap_username: Multiple-system operator account username.
|
|
ap_password: Multiple-system operator account password.
|
|
usenetrc: Use netrc for authentication instead.
|
|
netrc_location: Location of the netrc file. Defaults to ~/.netrc.
|
|
netrc_cmd: Use a shell command to get credentials
|
|
verbose: Print additional info to stdout.
|
|
quiet: Do not print messages to stdout.
|
|
no_warnings: Do not print out anything for warnings.
|
|
forceprint: A dict with keys WHEN mapped to a list of templates to
|
|
print to stdout. The allowed keys are video or any of the
|
|
items in utils.POSTPROCESS_WHEN.
|
|
For compatibility, a single list is also accepted
|
|
print_to_file: A dict with keys WHEN (same as forceprint) mapped to
|
|
a list of tuples with (template, filename)
|
|
forcejson: Force printing info_dict as JSON.
|
|
dump_single_json: Force printing the info_dict of the whole playlist
|
|
(or video) as a single JSON line.
|
|
force_write_download_archive: Force writing download archive regardless
|
|
of 'skip_download' or 'simulate'.
|
|
simulate: Do not download the video files. If unset (or None),
|
|
simulate only if listsubtitles, listformats or list_thumbnails is used
|
|
format: Video format code. see "FORMAT SELECTION" for more details.
|
|
You can also pass a function. The function takes 'ctx' as
|
|
argument and returns the formats to download.
|
|
See "build_format_selector" for an implementation
|
|
allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
|
|
ignore_no_formats_error: Ignore "No video formats" error. Usefull for
|
|
extracting metadata even if the video is not actually
|
|
available for download (experimental)
|
|
format_sort: A list of fields by which to sort the video formats.
|
|
See "Sorting Formats" for more details.
|
|
format_sort_force: Force the given format_sort. see "Sorting Formats"
|
|
for more details.
|
|
prefer_free_formats: Whether to prefer video formats with free containers
|
|
over non-free ones of same quality.
|
|
allow_multiple_video_streams: Allow multiple video streams to be merged
|
|
into a single file
|
|
allow_multiple_audio_streams: Allow multiple audio streams to be merged
|
|
into a single file
|
|
check_formats Whether to test if the formats are downloadable.
|
|
Can be True (check all), False (check none),
|
|
'selected' (check selected formats),
|
|
or None (check only if requested by extractor)
|
|
paths: Dictionary of output paths. The allowed keys are 'home'
|
|
'temp' and the keys of OUTTMPL_TYPES (in utils.py)
|
|
outtmpl: Dictionary of templates for output names. Allowed keys
|
|
are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
|
|
For compatibility with youtube-dl, a single string can also be used
|
|
outtmpl_na_placeholder: Placeholder for unavailable meta fields.
|
|
restrictfilenames: Do not allow "&" and spaces in file names
|
|
trim_file_name: Limit length of filename (extension excluded)
|
|
windowsfilenames: Force the filenames to be windows compatible
|
|
ignoreerrors: Do not stop on download/postprocessing errors.
|
|
Can be 'only_download' to ignore only download errors.
|
|
Default is 'only_download' for CLI, but False for API
|
|
skip_playlist_after_errors: Number of allowed failures until the rest of
|
|
the playlist is skipped
|
|
allowed_extractors: List of regexes to match against extractor names that are allowed
|
|
overwrites: Overwrite all video and metadata files if True,
|
|
overwrite only non-video files if None
|
|
and don't overwrite any file if False
|
|
playlist_items: Specific indices of playlist to download.
|
|
playlistrandom: Download playlist items in random order.
|
|
lazy_playlist: Process playlist entries as they are received.
|
|
matchtitle: Download only matching titles.
|
|
rejecttitle: Reject downloads for matching titles.
|
|
logger: Log messages to a logging.Logger instance.
|
|
logtostderr: Print everything to stderr instead of stdout.
|
|
consoletitle: Display progress in console window's titlebar.
|
|
writedescription: Write the video description to a .description file
|
|
writeinfojson: Write the video description to a .info.json file
|
|
clean_infojson: Remove internal metadata from the infojson
|
|
getcomments: Extract video comments. This will not be written to disk
|
|
unless writeinfojson is also given
|
|
writeannotations: Write the video annotations to a .annotations.xml file
|
|
writethumbnail: Write the thumbnail image to a file
|
|
allow_playlist_files: Whether to write playlists' description, infojson etc
|
|
also to disk when using the 'write*' options
|
|
write_all_thumbnails: Write all thumbnail formats to files
|
|
writelink: Write an internet shortcut file, depending on the
|
|
current platform (.url/.webloc/.desktop)
|
|
writeurllink: Write a Windows internet shortcut file (.url)
|
|
writewebloclink: Write a macOS internet shortcut file (.webloc)
|
|
writedesktoplink: Write a Linux internet shortcut file (.desktop)
|
|
writesubtitles: Write the video subtitles to a file
|
|
writeautomaticsub: Write the automatically generated subtitles to a file
|
|
listsubtitles: Lists all available subtitles for the video
|
|
subtitlesformat: The format code for subtitles
|
|
subtitleslangs: List of languages of the subtitles to download (can be regex).
|
|
The list may contain "all" to refer to all the available
|
|
subtitles. The language can be prefixed with a "-" to
|
|
exclude it from the requested languages, e.g. ['all', '-live_chat']
|
|
keepvideo: Keep the video file after post-processing
|
|
daterange: A utils.DateRange object, download only if the upload_date is in the range.
|
|
skip_download: Skip the actual download of the video file
|
|
cachedir: Location of the cache files in the filesystem.
|
|
False to disable filesystem cache.
|
|
noplaylist: Download single video instead of a playlist if in doubt.
|
|
age_limit: An integer representing the user's age in years.
|
|
Unsuitable videos for the given age are skipped.
|
|
min_views: An integer representing the minimum view count the video
|
|
must have in order to not be skipped.
|
|
Videos without view count information are always
|
|
downloaded. None for no limit.
|
|
max_views: An integer representing the maximum view count.
|
|
Videos that are more popular than that are not
|
|
downloaded.
|
|
Videos without view count information are always
|
|
downloaded. None for no limit.
|
|
download_archive: A set, or the name of a file where all downloads are recorded.
|
|
Videos already present in the file are not downloaded again.
|
|
break_on_existing: Stop the download process after attempting to download a
|
|
file that is in the archive.
|
|
break_per_url: Whether break_on_reject and break_on_existing
|
|
should act on each input URL as opposed to for the entire queue
|
|
cookiefile: File name or text stream from where cookies should be read and dumped to
|
|
cookiesfrombrowser: A tuple containing the name of the browser, the profile
|
|
name/path from where cookies are loaded, the name of the keyring,
|
|
and the container name, e.g. ('chrome', ) or
|
|
('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta')
|
|
legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
|
|
support RFC 5746 secure renegotiation
|
|
nocheckcertificate: Do not verify SSL certificates
|
|
client_certificate: Path to client certificate file in PEM format. May include the private key
|
|
client_certificate_key: Path to private key file for client certificate
|
|
client_certificate_password: Password for client certificate private key, if encrypted.
|
|
If not provided and the key is encrypted, yt-dlp will ask interactively
|
|
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
|
|
(Only supported by some extractors)
|
|
enable_file_urls: Enable file:// URLs. This is disabled by default for security reasons.
|
|
http_headers: A dictionary of custom headers to be used for all requests
|
|
proxy: URL of the proxy server to use
|
|
geo_verification_proxy: URL of the proxy to use for IP address verification
|
|
on geo-restricted sites.
|
|
socket_timeout: Time to wait for unresponsive hosts, in seconds
|
|
bidi_workaround: Work around buggy terminals without bidirectional text
|
|
support, using fridibi
|
|
debug_printtraffic:Print out sent and received HTTP traffic
|
|
default_search: Prepend this string if an input url is not valid.
|
|
'auto' for elaborate guessing
|
|
encoding: Use this encoding instead of the system-specified.
|
|
extract_flat: Whether to resolve and process url_results further
|
|
* False: Always process. Default for API
|
|
* True: Never process
|
|
* 'in_playlist': Do not process inside playlist/multi_video
|
|
* 'discard': Always process, but don't return the result
|
|
from inside playlist/multi_video
|
|
* 'discard_in_playlist': Same as "discard", but only for
|
|
playlists (not multi_video). Default for CLI
|
|
wait_for_video: If given, wait for scheduled streams to become available.
|
|
The value should be a tuple containing the range
|
|
(min_secs, max_secs) to wait between retries
|
|
postprocessors: A list of dictionaries, each with an entry
|
|
* key: The name of the postprocessor. See
|
|
yt_dlp/postprocessor/__init__.py for a list.
|
|
* when: When to run the postprocessor. Allowed values are
|
|
the entries of utils.POSTPROCESS_WHEN
|
|
Assumed to be 'post_process' if not given
|
|
progress_hooks: A list of functions that get called on download
|
|
progress, with a dictionary with the entries
|
|
* status: One of "downloading", "error", or "finished".
|
|
Check this first and ignore unknown values.
|
|
* info_dict: The extracted info_dict
|
|
|
|
If status is one of "downloading", or "finished", the
|
|
following properties may also be present:
|
|
* filename: The final filename (always present)
|
|
* tmpfilename: The filename we're currently writing to
|
|
* downloaded_bytes: Bytes on disk
|
|
* total_bytes: Size of the whole file, None if unknown
|
|
* total_bytes_estimate: Guess of the eventual file size,
|
|
None if unavailable.
|
|
* elapsed: The number of seconds since download started.
|
|
* eta: The estimated time in seconds, None if unknown
|
|
* speed: The download speed in bytes/second, None if
|
|
unknown
|
|
* fragment_index: The counter of the currently
|
|
downloaded video fragment.
|
|
* fragment_count: The number of fragments (= individual
|
|
files that will be merged)
|
|
|
|
Progress hooks are guaranteed to be called at least once
|
|
(with status "finished") if the download is successful.
|
|
postprocessor_hooks: A list of functions that get called on postprocessing
|
|
progress, with a dictionary with the entries
|
|
* status: One of "started", "processing", or "finished".
|
|
Check this first and ignore unknown values.
|
|
* postprocessor: Name of the postprocessor
|
|
* info_dict: The extracted info_dict
|
|
|
|
Progress hooks are guaranteed to be called at least twice
|
|
(with status "started" and "finished") if the processing is successful.
|
|
merge_output_format: "/" separated list of extensions to use when merging formats.
|
|
final_ext: Expected final extension; used to detect when the file was
|
|
already downloaded and converted
|
|
fixup: Automatically correct known faults of the file.
|
|
One of:
|
|
- "never": do nothing
|
|
- "warn": only emit a warning
|
|
- "detect_or_warn": check whether we can do anything
|
|
about it, warn otherwise (default)
|
|
source_address: Client-side IP address to bind to.
|
|
sleep_interval_requests: Number of seconds to sleep between requests
|
|
during extraction
|
|
sleep_interval: Number of seconds to sleep before each download when
|
|
used alone or a lower bound of a range for randomized
|
|
sleep before each download (minimum possible number
|
|
of seconds to sleep) when used along with
|
|
max_sleep_interval.
|
|
max_sleep_interval:Upper bound of a range for randomized sleep before each
|
|
download (maximum possible number of seconds to sleep).
|
|
Must only be used along with sleep_interval.
|
|
Actual sleep time will be a random float from range
|
|
[sleep_interval; max_sleep_interval].
|
|
sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
|
|
listformats: Print an overview of available video formats and exit.
|
|
list_thumbnails: Print a table of all thumbnails and exit.
|
|
match_filter: A function that gets called for every video with the signature
|
|
(info_dict, *, incomplete: bool) -> Optional[str]
|
|
For backward compatibility with youtube-dl, the signature
|
|
(info_dict) -> Optional[str] is also allowed.
|
|
- If it returns a message, the video is ignored.
|
|
- If it returns None, the video is downloaded.
|
|
- If it returns utils.NO_DEFAULT, the user is interactively
|
|
asked whether to download the video.
|
|
- Raise utils.DownloadCancelled(msg) to abort remaining
|
|
downloads when a video is rejected.
|
|
match_filter_func in utils.py is one example for this.
|
|
color: A Dictionary with output stream names as keys
|
|
and their respective color policy as values.
|
|
Can also just be a single color policy,
|
|
in which case it applies to all outputs.
|
|
Valid stream names are 'stdout' and 'stderr'.
|
|
Valid color policies are one of 'always', 'auto', 'no_color' or 'never'.
|
|
geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
|
|
HTTP header
|
|
geo_bypass_country:
|
|
Two-letter ISO 3166-2 country code that will be used for
|
|
explicit geographic restriction bypassing via faking
|
|
X-Forwarded-For HTTP header
|
|
geo_bypass_ip_block:
|
|
IP range in CIDR notation that will be used similarly to
|
|
geo_bypass_country
|
|
external_downloader: A dictionary of protocol keys and the executable of the
|
|
external downloader to use for it. The allowed protocols
|
|
are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
|
|
Set the value to 'native' to use the native downloader
|
|
compat_opts: Compatibility options. See "Differences in default behavior".
|
|
The following options do not work when used through the API:
|
|
filename, abort-on-error, multistreams, no-live-chat, format-sort
|
|
no-clean-infojson, no-playlist-metafiles, no-keep-subs, no-attach-info-json.
|
|
Refer __init__.py for their implementation
|
|
progress_template: Dictionary of templates for progress outputs.
|
|
Allowed keys are 'download', 'postprocess',
|
|
'download-title' (console title) and 'postprocess-title'.
|
|
The template is mapped on a dictionary with keys 'progress' and 'info'
|
|
retry_sleep_functions: Dictionary of functions that takes the number of attempts
|
|
as argument and returns the time to sleep in seconds.
|
|
Allowed keys are 'http', 'fragment', 'file_access'
|
|
download_ranges: A callback function that gets called for every video with
|
|
the signature (info_dict, ydl) -> Iterable[Section].
|
|
Only the returned sections will be downloaded.
|
|
Each Section is a dict with the following keys:
|
|
* start_time: Start time of the section in seconds
|
|
* end_time: End time of the section in seconds
|
|
* title: Section title (Optional)
|
|
* index: Section number (Optional)
|
|
force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
|
|
noprogress: Do not print the progress bar
|
|
live_from_start: Whether to download livestreams videos from the start
|
|
|
|
The following parameters are not used by YoutubeDL itself, they are used by
|
|
the downloader (see yt_dlp/downloader/common.py):
|
|
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
|
|
max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
|
|
continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
|
|
external_downloader_args, concurrent_fragment_downloads.
|
|
|
|
The following options are used by the post processors:
|
|
ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
|
|
to the binary or its containing directory.
|
|
postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
|
|
and a list of additional command-line arguments for the
|
|
postprocessor/executable. The dict can also have "PP+EXE" keys
|
|
which are used when the given exe is used by the given PP.
|
|
Use 'default' as the name for arguments to passed to all PP
|
|
For compatibility with youtube-dl, a single list of args
|
|
can also be used
|
|
|
|
The following options are used by the extractors:
|
|
extractor_retries: Number of times to retry for known errors (default: 3)
|
|
dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
|
|
hls_split_discontinuity: Split HLS playlists to different formats at
|
|
discontinuities such as ad breaks (default: False)
|
|
extractor_args: A dictionary of arguments to be passed to the extractors.
|
|
See "EXTRACTOR ARGUMENTS" for details.
|
|
E.g. {'youtube': {'skip': ['dash', 'hls']}}
|
|
mark_watched: Mark videos watched (even with --simulate). Only for YouTube
|
|
|
|
The following options are deprecated and may be removed in the future:
|
|
|
|
break_on_reject: Stop the download process when encountering a video that
|
|
has been filtered out.
|
|
- `raise DownloadCancelled(msg)` in match_filter instead
|
|
force_generic_extractor: Force downloader to use the generic extractor
|
|
- Use allowed_extractors = ['generic', 'default']
|
|
playliststart: - Use playlist_items
|
|
Playlist item to start at.
|
|
playlistend: - Use playlist_items
|
|
Playlist item to end at.
|
|
playlistreverse: - Use playlist_items
|
|
Download playlist items in reverse order.
|
|
forceurl: - Use forceprint
|
|
Force printing final URL.
|
|
forcetitle: - Use forceprint
|
|
Force printing title.
|
|
forceid: - Use forceprint
|
|
Force printing ID.
|
|
forcethumbnail: - Use forceprint
|
|
Force printing thumbnail URL.
|
|
forcedescription: - Use forceprint
|
|
Force printing description.
|
|
forcefilename: - Use forceprint
|
|
Force printing final filename.
|
|
forceduration: - Use forceprint
|
|
Force printing duration.
|
|
allsubtitles: - Use subtitleslangs = ['all']
|
|
Downloads all the subtitles of the video
|
|
(requires writesubtitles or writeautomaticsub)
|
|
include_ads: - Doesn't work
|
|
Download ads as well
|
|
call_home: - Not implemented
|
|
Boolean, true iff we are allowed to contact the
|
|
yt-dlp servers for debugging.
|
|
post_hooks: - Register a custom postprocessor
|
|
A list of functions that get called as the final step
|
|
for each video file, after all postprocessors have been
|
|
called. The filename will be passed as the only argument.
|
|
hls_prefer_native: - Use external_downloader = {'m3u8': 'native'} or {'m3u8': 'ffmpeg'}.
|
|
Use the native HLS downloader instead of ffmpeg/avconv
|
|
if True, otherwise use ffmpeg/avconv if False, otherwise
|
|
use downloader suggested by extractor if None.
|
|
prefer_ffmpeg: - avconv support is deprecated
|
|
If False, use avconv instead of ffmpeg if both are available,
|
|
otherwise prefer ffmpeg.
|
|
youtube_include_dash_manifest: - Use extractor_args
|
|
If True (default), DASH manifests and related
|
|
data will be downloaded and processed by extractor.
|
|
You can reduce network I/O by disabling it if you don't
|
|
care about DASH. (only for youtube)
|
|
youtube_include_hls_manifest: - Use extractor_args
|
|
If True (default), HLS manifests and related
|
|
data will be downloaded and processed by extractor.
|
|
You can reduce network I/O by disabling it if you don't
|
|
care about HLS. (only for youtube)
|
|
no_color: Same as `color='no_color'`
|
|
no_overwrites: Same as `overwrites=False`
|
|
"""
|
|
|
|
_NUMERIC_FIELDS = {
|
|
'width', 'height', 'asr', 'audio_channels', 'fps',
|
|
'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
|
|
'timestamp', 'release_timestamp',
|
|
'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
|
|
'average_rating', 'comment_count', 'age_limit',
|
|
'start_time', 'end_time',
|
|
'chapter_number', 'season_number', 'episode_number',
|
|
'track_number', 'disc_number', 'release_year',
|
|
}
|
|
|
|
_format_fields = {
|
|
# NB: Keep in sync with the docstring of extractor/common.py
|
|
'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
|
|
'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
|
|
'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
|
|
'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
|
|
'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
|
|
'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'hls_aes', 'downloader_options',
|
|
'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
|
|
}
|
|
_format_selection_exts = {
|
|
'audio': set(MEDIA_EXTENSIONS.common_audio),
|
|
'video': set(MEDIA_EXTENSIONS.common_video + ('3gp', )),
|
|
'storyboards': set(MEDIA_EXTENSIONS.storyboards),
|
|
}
|
|
|
|
def __init__(self, params=None, auto_init=True):
|
|
"""Create a FileDownloader object with the given options.
|
|
@param auto_init Whether to load the default extractors and print header (if verbose).
|
|
Set to 'no_verbose_header' to not print the header
|
|
"""
|
|
if params is None:
|
|
params = {}
|
|
self.params = params
|
|
self._ies = {}
|
|
self._ies_instances = {}
|
|
self._pps = {k: [] for k in POSTPROCESS_WHEN}
|
|
self._printed_messages = set()
|
|
self._first_webpage_request = True
|
|
self._post_hooks = []
|
|
self._progress_hooks = []
|
|
self._postprocessor_hooks = []
|
|
self._download_retcode = 0
|
|
self._num_downloads = 0
|
|
self._num_videos = 0
|
|
self._playlist_level = 0
|
|
self._playlist_urls = set()
|
|
self.cache = Cache(self)
|
|
self.__header_cookies = []
|
|
|
|
stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout
|
|
self._out_files = Namespace(
|
|
out=stdout,
|
|
error=sys.stderr,
|
|
screen=sys.stderr if self.params.get('quiet') else stdout,
|
|
console=None if compat_os_name == 'nt' else next(
|
|
filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None)
|
|
)
|
|
|
|
try:
|
|
windows_enable_vt_mode()
|
|
except Exception as e:
|
|
self.write_debug(f'Failed to enable VT mode: {e}')
|
|
|
|
if self.params.get('no_color'):
|
|
if self.params.get('color') is not None:
|
|
self.params.setdefault('_warnings', []).append(
|
|
'Overwriting params from "color" with "no_color"')
|
|
self.params['color'] = 'no_color'
|
|
|
|
term_allow_color = os.environ.get('TERM', '').lower() != 'dumb'
|
|
|
|
def process_color_policy(stream):
|
|
stream_name = {sys.stdout: 'stdout', sys.stderr: 'stderr'}[stream]
|
|
policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False)
|
|
if policy in ('auto', None):
|
|
return term_allow_color and supports_terminal_sequences(stream)
|
|
assert policy in ('always', 'never', 'no_color'), policy
|
|
return {'always': True, 'never': False}.get(policy, policy)
|
|
|
|
self._allow_colors = Namespace(**{
|
|
name: process_color_policy(stream)
|
|
for name, stream in self._out_files.items_ if name != 'console'
|
|
})
|
|
|
|
# The code is left like this to be reused for future deprecations
|
|
MIN_SUPPORTED, MIN_RECOMMENDED = (3, 7), (3, 7)
|
|
current_version = sys.version_info[:2]
|
|
if current_version < MIN_RECOMMENDED:
|
|
msg = ('Support for Python version %d.%d has been deprecated. '
|
|
'See https://github.com/yt-dlp/yt-dlp/issues/3764 for more details.'
|
|
'\n You will no longer receive updates on this version')
|
|
if current_version < MIN_SUPPORTED:
|
|
msg = 'Python version %d.%d is no longer supported'
|
|
self.deprecated_feature(
|
|
f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED))
|
|
|
|
if self.params.get('allow_unplayable_formats'):
|
|
self.report_warning(
|
|
f'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
|
|
'This is a developer option intended for debugging. \n'
|
|
' If you experience any issues while using this option, '
|
|
f'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
|
|
|
|
if self.params.get('bidi_workaround', False):
|
|
try:
|
|
import pty
|
|
master, slave = pty.openpty()
|
|
width = shutil.get_terminal_size().columns
|
|
width_args = [] if width is None else ['-w', str(width)]
|
|
sp_kwargs = {'stdin': subprocess.PIPE, 'stdout': slave, 'stderr': self._out_files.error}
|
|
try:
|
|
self._output_process = Popen(['bidiv'] + width_args, **sp_kwargs)
|
|
except OSError:
|
|
self._output_process = Popen(['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
|
|
self._output_channel = os.fdopen(master, 'rb')
|
|
except OSError as ose:
|
|
if ose.errno == errno.ENOENT:
|
|
self.report_warning(
|
|
'Could not find fribidi executable, ignoring --bidi-workaround. '
|
|
'Make sure that fribidi is an executable file in one of the directories in your $PATH.')
|
|
else:
|
|
raise
|
|
|
|
self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
|
|
self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers'))
|
|
self._load_cookies(self.params['http_headers'].get('Cookie')) # compat
|
|
self.params['http_headers'].pop('Cookie', None)
|
|
self._request_director = self.build_request_director(_REQUEST_HANDLERS.values(), _RH_PREFERENCES)
|
|
|
|
if auto_init and auto_init != 'no_verbose_header':
|
|
self.print_debug_header()
|
|
|
|
def check_deprecated(param, option, suggestion):
|
|
if self.params.get(param) is not None:
|
|
self.report_warning(f'{option} is deprecated. Use {suggestion} instead')
|
|
return True
|
|
return False
|
|
|
|
if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
|
|
if self.params.get('geo_verification_proxy') is None:
|
|
self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
|
|
|
|
check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
|
|
check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
|
|
check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
|
|
|
|
for msg in self.params.get('_warnings', []):
|
|
self.report_warning(msg)
|
|
for msg in self.params.get('_deprecation_warnings', []):
|
|
self.deprecated_feature(msg)
|
|
|
|
if 'list-formats' in self.params['compat_opts']:
|
|
self.params['listformats_table'] = False
|
|
|
|
if 'overwrites' not in self.params and self.params.get('nooverwrites') is not None:
|
|
# nooverwrites was unnecessarily changed to overwrites
|
|
# in 0c3d0f51778b153f65c21906031c2e091fcfb641
|
|
# This ensures compatibility with both keys
|
|
self.params['overwrites'] = not self.params['nooverwrites']
|
|
elif self.params.get('overwrites') is None:
|
|
self.params.pop('overwrites', None)
|
|
else:
|
|
self.params['nooverwrites'] = not self.params['overwrites']
|
|
|
|
if self.params.get('simulate') is None and any((
|
|
self.params.get('list_thumbnails'),
|
|
self.params.get('listformats'),
|
|
self.params.get('listsubtitles'),
|
|
)):
|
|
self.params['simulate'] = 'list_only'
|
|
|
|
self.params.setdefault('forceprint', {})
|
|
self.params.setdefault('print_to_file', {})
|
|
|
|
# Compatibility with older syntax
|
|
if not isinstance(params['forceprint'], dict):
|
|
self.params['forceprint'] = {'video': params['forceprint']}
|
|
|
|
if auto_init:
|
|
self.add_default_info_extractors()
|
|
|
|
if (sys.platform != 'win32'
|
|
and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
|
|
and not self.params.get('restrictfilenames', False)):
|
|
# Unicode filesystem API will throw errors (#1474, #13027)
|
|
self.report_warning(
|
|
'Assuming --restrict-filenames since file system encoding '
|
|
'cannot encode all characters. '
|
|
'Set the LC_ALL environment variable to fix this.')
|
|
self.params['restrictfilenames'] = True
|
|
|
|
self._parse_outtmpl()
|
|
|
|
# Creating format selector here allows us to catch syntax errors before the extraction
|
|
self.format_selector = (
|
|
self.params.get('format') if self.params.get('format') in (None, '-')
|
|
else self.params['format'] if callable(self.params['format'])
|
|
else self.build_format_selector(self.params['format']))
|
|
|
|
hooks = {
|
|
'post_hooks': self.add_post_hook,
|
|
'progress_hooks': self.add_progress_hook,
|
|
'postprocessor_hooks': self.add_postprocessor_hook,
|
|
}
|
|
for opt, fn in hooks.items():
|
|
for ph in self.params.get(opt, []):
|
|
fn(ph)
|
|
|
|
for pp_def_raw in self.params.get('postprocessors', []):
|
|
pp_def = dict(pp_def_raw)
|
|
when = pp_def.pop('when', 'post_process')
|
|
self.add_post_processor(
|
|
get_postprocessor(pp_def.pop('key'))(self, **pp_def),
|
|
when=when)
|
|
|
|
def preload_download_archive(fn):
|
|
"""Preload the archive, if any is specified"""
|
|
archive = set()
|
|
if fn is None:
|
|
return archive
|
|
elif not is_path_like(fn):
|
|
return fn
|
|
|
|
self.write_debug(f'Loading archive file {fn!r}')
|
|
try:
|
|
with locked_file(fn, 'r', encoding='utf-8') as archive_file:
|
|
for line in archive_file:
|
|
archive.add(line.strip())
|
|
except OSError as ioe:
|
|
if ioe.errno != errno.ENOENT:
|
|
raise
|
|
return archive
|
|
|
|
self.archive = preload_download_archive(self.params.get('download_archive'))
|
|
|
|
def warn_if_short_id(self, argv):
|
|
# short YouTube ID starting with dash?
|
|
idxs = [
|
|
i for i, a in enumerate(argv)
|
|
if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
|
|
if idxs:
|
|
correct_argv = (
|
|
['yt-dlp']
|
|
+ [a for i, a in enumerate(argv) if i not in idxs]
|
|
+ ['--'] + [argv[i] for i in idxs]
|
|
)
|
|
self.report_warning(
|
|
'Long argument string detected. '
|
|
'Use -- to separate parameters and URLs, like this:\n%s' %
|
|
args_to_str(correct_argv))
|
|
|
|
def add_info_extractor(self, ie):
|
|
"""Add an InfoExtractor object to the end of the list."""
|
|
ie_key = ie.ie_key()
|
|
self._ies[ie_key] = ie
|
|
if not isinstance(ie, type):
|
|
self._ies_instances[ie_key] = ie
|
|
ie.set_downloader(self)
|
|
|
|
def get_info_extractor(self, ie_key):
|
|
"""
|
|
Get an instance of an IE with name ie_key, it will try to get one from
|
|
the _ies list, if there's no instance it will create a new one and add
|
|
it to the extractor list.
|
|
"""
|
|
ie = self._ies_instances.get(ie_key)
|
|
if ie is None:
|
|
ie = get_info_extractor(ie_key)()
|
|
self.add_info_extractor(ie)
|
|
return ie
|
|
|
|
def add_default_info_extractors(self):
|
|
"""
|
|
Add the InfoExtractors returned by gen_extractors to the end of the list
|
|
"""
|
|
all_ies = {ie.IE_NAME.lower(): ie for ie in gen_extractor_classes()}
|
|
all_ies['end'] = UnsupportedURLIE()
|
|
try:
|
|
ie_names = orderedSet_from_options(
|
|
self.params.get('allowed_extractors', ['default']), {
|
|
'all': list(all_ies),
|
|
'default': [name for name, ie in all_ies.items() if ie._ENABLED],
|
|
}, use_regex=True)
|
|
except re.error as e:
|
|
raise ValueError(f'Wrong regex for allowed_extractors: {e.pattern}')
|
|
for name in ie_names:
|
|
self.add_info_extractor(all_ies[name])
|
|
self.write_debug(f'Loaded {len(ie_names)} extractors')
|
|
|
|
def add_post_processor(self, pp, when='post_process'):
|
|
"""Add a PostProcessor object to the end of the chain."""
|
|
assert when in POSTPROCESS_WHEN, f'Invalid when={when}'
|
|
self._pps[when].append(pp)
|
|
pp.set_downloader(self)
|
|
|
|
def add_post_hook(self, ph):
|
|
"""Add the post hook"""
|
|
self._post_hooks.append(ph)
|
|
|
|
def add_progress_hook(self, ph):
|
|
"""Add the download progress hook"""
|
|
self._progress_hooks.append(ph)
|
|
|
|
def add_postprocessor_hook(self, ph):
|
|
"""Add the postprocessing progress hook"""
|
|
self._postprocessor_hooks.append(ph)
|
|
for pps in self._pps.values():
|
|
for pp in pps:
|
|
pp.add_progress_hook(ph)
|
|
|
|
def _bidi_workaround(self, message):
|
|
if not hasattr(self, '_output_channel'):
|
|
return message
|
|
|
|
assert hasattr(self, '_output_process')
|
|
assert isinstance(message, str)
|
|
line_count = message.count('\n') + 1
|
|
self._output_process.stdin.write((message + '\n').encode())
|
|
self._output_process.stdin.flush()
|
|
res = ''.join(self._output_channel.readline().decode()
|
|
for _ in range(line_count))
|
|
return res[:-len('\n')]
|
|
|
|
def _write_string(self, message, out=None, only_once=False):
|
|
if only_once:
|
|
if message in self._printed_messages:
|
|
return
|
|
self._printed_messages.add(message)
|
|
write_string(message, out=out, encoding=self.params.get('encoding'))
|
|
|
|
def to_stdout(self, message, skip_eol=False, quiet=None):
|
|
"""Print message to stdout"""
|
|
if quiet is not None:
|
|
self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. '
|
|
'Use "YoutubeDL.to_screen" instead')
|
|
if skip_eol is not False:
|
|
self.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. '
|
|
'Use "YoutubeDL.to_screen" instead')
|
|
self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.out)
|
|
|
|
def to_screen(self, message, skip_eol=False, quiet=None, only_once=False):
|
|
"""Print message to screen if not in quiet mode"""
|
|
if self.params.get('logger'):
|
|
self.params['logger'].debug(message)
|
|
return
|
|
if (self.params.get('quiet') if quiet is None else quiet) and not self.params.get('verbose'):
|
|
return
|
|
self._write_string(
|
|
'%s%s' % (self._bidi_workaround(message), ('' if skip_eol else '\n')),
|
|
self._out_files.screen, only_once=only_once)
|
|
|
|
def to_stderr(self, message, only_once=False):
|
|
"""Print message to stderr"""
|
|
assert isinstance(message, str)
|
|
if self.params.get('logger'):
|
|
self.params['logger'].error(message)
|
|
else:
|
|
self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.error, only_once=only_once)
|
|
|
|
def _send_console_code(self, code):
|
|
if compat_os_name == 'nt' or not self._out_files.console:
|
|
return
|
|
self._write_string(code, self._out_files.console)
|
|
|
|
def to_console_title(self, message):
|
|
if not self.params.get('consoletitle', False):
|
|
return
|
|
message = remove_terminal_sequences(message)
|
|
if compat_os_name == 'nt':
|
|
if ctypes.windll.kernel32.GetConsoleWindow():
|
|
# c_wchar_p() might not be necessary if `message` is
|
|
# already of type unicode()
|
|
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
|
|
else:
|
|
self._send_console_code(f'\033]0;{message}\007')
|
|
|
|
def save_console_title(self):
|
|
if not self.params.get('consoletitle') or self.params.get('simulate'):
|
|
return
|
|
self._send_console_code('\033[22;0t') # Save the title on stack
|
|
|
|
def restore_console_title(self):
|
|
if not self.params.get('consoletitle') or self.params.get('simulate'):
|
|
return
|
|
self._send_console_code('\033[23;0t') # Restore the title from stack
|
|
|
|
def __enter__(self):
|
|
self.save_console_title()
|
|
return self
|
|
|
|
def save_cookies(self):
|
|
if self.params.get('cookiefile') is not None:
|
|
self.cookiejar.save()
|
|
|
|
def __exit__(self, *args):
|
|
self.restore_console_title()
|
|
self.close()
|
|
|
|
def close(self):
|
|
self.save_cookies()
|
|
self._request_director.close()
|
|
|
|
def trouble(self, message=None, tb=None, is_error=True):
|
|
"""Determine action to take when a download problem appears.
|
|
|
|
Depending on if the downloader has been configured to ignore
|
|
download errors or not, this method may throw an exception or
|
|
not when errors are found, after printing the message.
|
|
|
|
@param tb If given, is additional traceback information
|
|
@param is_error Whether to raise error according to ignorerrors
|
|
"""
|
|
if message is not None:
|
|
self.to_stderr(message)
|
|
if self.params.get('verbose'):
|
|
if tb is None:
|
|
if sys.exc_info()[0]: # if .trouble has been called from an except block
|
|
tb = ''
|
|
if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
|
|
tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
|
|
tb += encode_compat_str(traceback.format_exc())
|
|
else:
|
|
tb_data = traceback.format_list(traceback.extract_stack())
|
|
tb = ''.join(tb_data)
|
|
if tb:
|
|
self.to_stderr(tb)
|
|
if not is_error:
|
|
return
|
|
if not self.params.get('ignoreerrors'):
|
|
if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
|
|
exc_info = sys.exc_info()[1].exc_info
|
|
else:
|
|
exc_info = sys.exc_info()
|
|
raise DownloadError(message, exc_info)
|
|
self._download_retcode = 1
|
|
|
|
Styles = Namespace(
|
|
HEADERS='yellow',
|
|
EMPHASIS='light blue',
|
|
FILENAME='green',
|
|
ID='green',
|
|
DELIM='blue',
|
|
ERROR='red',
|
|
BAD_FORMAT='light red',
|
|
WARNING='yellow',
|
|
SUPPRESS='light black',
|
|
)
|
|
|
|
def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_encoding=False):
|
|
text = str(text)
|
|
if test_encoding:
|
|
original_text = text
|
|
# handle.encoding can be None. See https://github.com/yt-dlp/yt-dlp/issues/2711
|
|
encoding = self.params.get('encoding') or getattr(handle, 'encoding', None) or 'ascii'
|
|
text = text.encode(encoding, 'ignore').decode(encoding)
|
|
if fallback is not None and text != original_text:
|
|
text = fallback
|
|
return format_text(text, f) if allow_colors is True else text if fallback is None else fallback
|
|
|
|
def _format_out(self, *args, **kwargs):
|
|
return self._format_text(self._out_files.out, self._allow_colors.out, *args, **kwargs)
|
|
|
|
def _format_screen(self, *args, **kwargs):
|
|
return self._format_text(self._out_files.screen, self._allow_colors.screen, *args, **kwargs)
|
|
|
|
def _format_err(self, *args, **kwargs):
|
|
return self._format_text(self._out_files.error, self._allow_colors.error, *args, **kwargs)
|
|
|
|
def report_warning(self, message, only_once=False):
|
|
'''
|
|
Print the message to stderr, it will be prefixed with 'WARNING:'
|
|
If stderr is a tty file the 'WARNING:' will be colored
|
|
'''
|
|
if self.params.get('logger') is not None:
|
|
self.params['logger'].warning(message)
|
|
else:
|
|
if self.params.get('no_warnings'):
|
|
return
|
|
self.to_stderr(f'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once)
|
|
|
|
def deprecation_warning(self, message, *, stacklevel=0):
|
|
deprecation_warning(
|
|
message, stacklevel=stacklevel + 1, printer=self.report_error, is_error=False)
|
|
|
|
def deprecated_feature(self, message):
|
|
if self.params.get('logger') is not None:
|
|
self.params['logger'].warning(f'Deprecated Feature: {message}')
|
|
self.to_stderr(f'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True)
|
|
|
|
def report_error(self, message, *args, **kwargs):
|
|
'''
|
|
Do the same as trouble, but prefixes the message with 'ERROR:', colored
|
|
in red if stderr is a tty file.
|
|
'''
|
|
self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args, **kwargs)
|
|
|
|
def write_debug(self, message, only_once=False):
|
|
'''Log debug message or Print message to stderr'''
|
|
if not self.params.get('verbose', False):
|
|
return
|
|
message = f'[debug] {message}'
|
|
if self.params.get('logger'):
|
|
self.params['logger'].debug(message)
|
|
else:
|
|
self.to_stderr(message, only_once)
|
|
|
|
def report_file_already_downloaded(self, file_name):
|
|
"""Report file has already been fully downloaded."""
|
|
try:
|
|
self.to_screen('[download] %s has already been downloaded' % file_name)
|
|
except UnicodeEncodeError:
|
|
self.to_screen('[download] The file has already been downloaded')
|
|
|
|
def report_file_delete(self, file_name):
|
|
"""Report that existing file will be deleted."""
|
|
try:
|
|
self.to_screen('Deleting existing file %s' % file_name)
|
|
except UnicodeEncodeError:
|
|
self.to_screen('Deleting existing file')
|
|
|
|
def raise_no_formats(self, info, forced=False, *, msg=None):
|
|
has_drm = info.get('_has_drm')
|
|
ignored, expected = self.params.get('ignore_no_formats_error'), bool(msg)
|
|
msg = msg or has_drm and 'This video is DRM protected' or 'No video formats found!'
|
|
if forced or not ignored:
|
|
raise ExtractorError(msg, video_id=info['id'], ie=info['extractor'],
|
|
expected=has_drm or ignored or expected)
|
|
else:
|
|
self.report_warning(msg)
|
|
|
|
def parse_outtmpl(self):
|
|
self.deprecation_warning('"YoutubeDL.parse_outtmpl" is deprecated and may be removed in a future version')
|
|
self._parse_outtmpl()
|
|
return self.params['outtmpl']
|
|
|
|
def _parse_outtmpl(self):
|
|
sanitize = IDENTITY
|
|
if self.params.get('restrictfilenames'): # Remove spaces in the default template
|
|
sanitize = lambda x: x.replace(' - ', ' ').replace(' ', '-')
|
|
|
|
outtmpl = self.params.setdefault('outtmpl', {})
|
|
if not isinstance(outtmpl, dict):
|
|
self.params['outtmpl'] = outtmpl = {'default': outtmpl}
|
|
outtmpl.update({k: sanitize(v) for k, v in DEFAULT_OUTTMPL.items() if outtmpl.get(k) is None})
|
|
|
|
def get_output_path(self, dir_type='', filename=None):
|
|
paths = self.params.get('paths', {})
|
|
assert isinstance(paths, dict), '"paths" parameter must be a dictionary'
|
|
path = os.path.join(
|
|
expand_path(paths.get('home', '').strip()),
|
|
expand_path(paths.get(dir_type, '').strip()) if dir_type else '',
|
|
filename or '')
|
|
return sanitize_path(path, force=self.params.get('windowsfilenames'))
|
|
|
|
@staticmethod
|
|
def _outtmpl_expandpath(outtmpl):
|
|
# expand_path translates '%%' into '%' and '$$' into '$'
|
|
# correspondingly that is not what we want since we need to keep
|
|
# '%%' intact for template dict substitution step. Working around
|
|
# with boundary-alike separator hack.
|
|
sep = ''.join(random.choices(string.ascii_letters, k=32))
|
|
outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$')
|
|
|
|
# outtmpl should be expand_path'ed before template dict substitution
|
|
# because meta fields may contain env variables we don't want to
|
|
# be expanded. E.g. for outtmpl "%(title)s.%(ext)s" and
|
|
# title "Hello $PATH", we don't want `$PATH` to be expanded.
|
|
return expand_path(outtmpl).replace(sep, '')
|
|
|
|
@staticmethod
|
|
def escape_outtmpl(outtmpl):
|
|
''' Escape any remaining strings like %s, %abc% etc. '''
|
|
return re.sub(
|
|
STR_FORMAT_RE_TMPL.format('', '(?![%(\0])'),
|
|
lambda mobj: ('' if mobj.group('has_key') else '%') + mobj.group(0),
|
|
outtmpl)
|
|
|
|
@classmethod
|
|
def validate_outtmpl(cls, outtmpl):
|
|
''' @return None or Exception object '''
|
|
outtmpl = re.sub(
|
|
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljhqBUDS]'),
|
|
lambda mobj: f'{mobj.group(0)[:-1]}s',
|
|
cls._outtmpl_expandpath(outtmpl))
|
|
try:
|
|
cls.escape_outtmpl(outtmpl) % collections.defaultdict(int)
|
|
return None
|
|
except ValueError as err:
|
|
return err
|
|
|
|
@staticmethod
|
|
def _copy_infodict(info_dict):
|
|
info_dict = dict(info_dict)
|
|
info_dict.pop('__postprocessors', None)
|
|
info_dict.pop('__pending_error', None)
|
|
return info_dict
|
|
|
|
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
|
|
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
|
|
@param sanitize Whether to sanitize the output as a filename.
|
|
For backward compatibility, a function can also be passed
|
|
"""
|
|
|
|
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
|
|
|
|
info_dict = self._copy_infodict(info_dict)
|
|
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
|
|
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
|
|
if info_dict.get('duration', None) is not None
|
|
else None)
|
|
info_dict['autonumber'] = int(self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
|
|
info_dict['video_autonumber'] = self._num_videos
|
|
if info_dict.get('resolution') is None:
|
|
info_dict['resolution'] = self.format_resolution(info_dict, default=None)
|
|
|
|
# For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
|
|
# of %(field)s to %(field)0Nd for backward compatibility
|
|
field_size_compat_map = {
|
|
'playlist_index': number_of_digits(info_dict.get('__last_playlist_index') or 0),
|
|
'playlist_autonumber': number_of_digits(info_dict.get('n_entries') or 0),
|
|
'autonumber': self.params.get('autonumber_size') or 5,
|
|
}
|
|
|
|
TMPL_DICT = {}
|
|
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljhqBUDS]'))
|
|
MATH_FUNCTIONS = {
|
|
'+': float.__add__,
|
|
'-': float.__sub__,
|
|
}
|
|
# Field is of the form key1.key2...
|
|
# where keys (except first) can be string, int, slice or "{field, ...}"
|
|
FIELD_INNER_RE = r'(?:\w+|%(num)s|%(num)s?(?::%(num)s?){1,2})' % {'num': r'(?:-?\d+)'}
|
|
FIELD_RE = r'\w*(?:\.(?:%(inner)s|{%(field)s(?:,%(field)s)*}))*' % {
|
|
'inner': FIELD_INNER_RE,
|
|
'field': rf'\w*(?:\.{FIELD_INNER_RE})*'
|
|
}
|
|
MATH_FIELD_RE = rf'(?:{FIELD_RE}|-?{NUMBER_RE})'
|
|
MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
|
|
INTERNAL_FORMAT_RE = re.compile(rf'''(?xs)
|
|
(?P<negate>-)?
|
|
(?P<fields>{FIELD_RE})
|
|
(?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
|
|
(?:>(?P<strf_format>.+?))?
|
|
(?P<remaining>
|
|
(?P<alternate>(?<!\\),[^|&)]+)?
|
|
(?:&(?P<replacement>.*?))?
|
|
(?:\|(?P<default>.*?))?
|
|
)$''')
|
|
|
|
def _traverse_infodict(fields):
|
|
fields = [f for x in re.split(r'\.({.+?})\.?', fields)
|
|
for f in ([x] if x.startswith('{') else x.split('.'))]
|
|
for i in (0, -1):
|
|
if fields and not fields[i]:
|
|
fields.pop(i)
|
|
|
|
for i, f in enumerate(fields):
|
|
if not f.startswith('{'):
|
|
continue
|
|
assert f.endswith('}'), f'No closing brace for {f} in {fields}'
|
|
fields[i] = {k: k.split('.') for k in f[1:-1].split(',')}
|
|
|
|
return traverse_obj(info_dict, fields, is_user_input=True, traverse_string=True)
|
|
|
|
def get_value(mdict):
|
|
# Object traversal
|
|
value = _traverse_infodict(mdict['fields'])
|
|
# Negative
|
|
if mdict['negate']:
|
|
value = float_or_none(value)
|
|
if value is not None:
|
|
value *= -1
|
|
# Do maths
|
|
offset_key = mdict['maths']
|
|
if offset_key:
|
|
value = float_or_none(value)
|
|
operator = None
|
|
while offset_key:
|
|
item = re.match(
|
|
MATH_FIELD_RE if operator else MATH_OPERATORS_RE,
|
|
offset_key).group(0)
|
|
offset_key = offset_key[len(item):]
|
|
if operator is None:
|
|
operator = MATH_FUNCTIONS[item]
|
|
continue
|
|
item, multiplier = (item[1:], -1) if item[0] == '-' else (item, 1)
|
|
offset = float_or_none(item)
|
|
if offset is None:
|
|
offset = float_or_none(_traverse_infodict(item))
|
|
try:
|
|
value = operator(value, multiplier * offset)
|
|
except (TypeError, ZeroDivisionError):
|
|
return None
|
|
operator = None
|
|
# Datetime formatting
|
|
if mdict['strf_format']:
|
|
value = strftime_or_none(value, mdict['strf_format'].replace('\\,', ','))
|
|
|
|
# XXX: Workaround for https://github.com/yt-dlp/yt-dlp/issues/4485
|
|
if sanitize and value == '':
|
|
value = None
|
|
return value
|
|
|
|
na = self.params.get('outtmpl_na_placeholder', 'NA')
|
|
|
|
def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')):
|
|
return sanitize_filename(str(value), restricted=restricted, is_id=(
|
|
bool(re.search(r'(^|[_.])id(\.|$)', key))
|
|
if 'filename-sanitization' in self.params['compat_opts']
|
|
else NO_DEFAULT))
|
|
|
|
sanitizer = sanitize if callable(sanitize) else filename_sanitizer
|
|
sanitize = bool(sanitize)
|
|
|
|
def _dumpjson_default(obj):
|
|
if isinstance(obj, (set, LazyList)):
|
|
return list(obj)
|
|
return repr(obj)
|
|
|
|
class _ReplacementFormatter(string.Formatter):
|
|
def get_field(self, field_name, args, kwargs):
|
|
if field_name.isdigit():
|
|
return args[0], -1
|
|
raise ValueError('Unsupported field')
|
|
|
|
replacement_formatter = _ReplacementFormatter()
|
|
|
|
def create_key(outer_mobj):
|
|
if not outer_mobj.group('has_key'):
|
|
return outer_mobj.group(0)
|
|
key = outer_mobj.group('key')
|
|
mobj = re.match(INTERNAL_FORMAT_RE, key)
|
|
value, replacement, default, last_field = None, None, na, ''
|
|
while mobj:
|
|
mobj = mobj.groupdict()
|
|
default = mobj['default'] if mobj['default'] is not None else default
|
|
value = get_value(mobj)
|
|
last_field, replacement = mobj['fields'], mobj['replacement']
|
|
if value is None and mobj['alternate']:
|
|
mobj = re.match(INTERNAL_FORMAT_RE, mobj['remaining'][1:])
|
|
else:
|
|
break
|
|
|
|
if None not in (value, replacement):
|
|
try:
|
|
value = replacement_formatter.format(replacement, value)
|
|
except ValueError:
|
|
value, default = None, na
|
|
|
|
fmt = outer_mobj.group('format')
|
|
if fmt == 's' and last_field in field_size_compat_map.keys() and isinstance(value, int):
|
|
fmt = f'0{field_size_compat_map[last_field]:d}d'
|
|
|
|
flags = outer_mobj.group('conversion') or ''
|
|
str_fmt = f'{fmt[:-1]}s'
|
|
if value is None:
|
|
value, fmt = default, 's'
|
|
elif fmt[-1] == 'l': # list
|
|
delim = '\n' if '#' in flags else ', '
|
|
value, fmt = delim.join(map(str, variadic(value, allowed_types=(str, bytes)))), str_fmt
|
|
elif fmt[-1] == 'j': # json
|
|
value, fmt = json.dumps(
|
|
value, default=_dumpjson_default,
|
|
indent=4 if '#' in flags else None, ensure_ascii='+' not in flags), str_fmt
|
|
elif fmt[-1] == 'h': # html
|
|
value, fmt = escapeHTML(str(value)), str_fmt
|
|
elif fmt[-1] == 'q': # quoted
|
|
value = map(str, variadic(value) if '#' in flags else [value])
|
|
value, fmt = ' '.join(map(compat_shlex_quote, value)), str_fmt
|
|
elif fmt[-1] == 'B': # bytes
|
|
value = f'%{str_fmt}'.encode() % str(value).encode()
|
|
value, fmt = value.decode('utf-8', 'ignore'), 's'
|
|
elif fmt[-1] == 'U': # unicode normalized
|
|
value, fmt = unicodedata.normalize(
|
|
# "+" = compatibility equivalence, "#" = NFD
|
|
'NF%s%s' % ('K' if '+' in flags else '', 'D' if '#' in flags else 'C'),
|
|
value), str_fmt
|
|
elif fmt[-1] == 'D': # decimal suffix
|
|
num_fmt, fmt = fmt[:-1].replace('#', ''), 's'
|
|
value = format_decimal_suffix(value, f'%{num_fmt}f%s' if num_fmt else '%d%s',
|
|
factor=1024 if '#' in flags else 1000)
|
|
elif fmt[-1] == 'S': # filename sanitization
|
|
value, fmt = filename_sanitizer(last_field, value, restricted='#' in flags), str_fmt
|
|
elif fmt[-1] == 'c':
|
|
if value:
|
|
value = str(value)[0]
|
|
else:
|
|
fmt = str_fmt
|
|
elif fmt[-1] not in 'rsa': # numeric
|
|
value = float_or_none(value)
|
|
if value is None:
|
|
value, fmt = default, 's'
|
|
|
|
if sanitize:
|
|
# If value is an object, sanitize might convert it to a string
|
|
# So we convert it to repr first
|
|
if fmt[-1] == 'r':
|
|
value, fmt = repr(value), str_fmt
|
|
elif fmt[-1] == 'a':
|
|
value, fmt = ascii(value), str_fmt
|
|
if fmt[-1] in 'csra':
|
|
value = sanitizer(last_field, value)
|
|
|
|
key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format'))
|
|
TMPL_DICT[key] = value
|
|
return '{prefix}%({key}){fmt}'.format(key=key, fmt=fmt, prefix=outer_mobj.group('prefix'))
|
|
|
|
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
|
|
|
|
def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs):
|
|
outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs)
|
|
return self.escape_outtmpl(outtmpl) % info_dict
|
|
|
|
def _prepare_filename(self, info_dict, *, outtmpl=None, tmpl_type=None):
|
|
assert None in (outtmpl, tmpl_type), 'outtmpl and tmpl_type are mutually exclusive'
|
|
if outtmpl is None:
|
|
outtmpl = self.params['outtmpl'].get(tmpl_type or 'default', self.params['outtmpl']['default'])
|
|
try:
|
|
outtmpl = self._outtmpl_expandpath(outtmpl)
|
|
filename = self.evaluate_outtmpl(outtmpl, info_dict, True)
|
|
if not filename:
|
|
return None
|
|
|
|
if tmpl_type in ('', 'temp'):
|
|
final_ext, ext = self.params.get('final_ext'), info_dict.get('ext')
|
|
if final_ext and ext and final_ext != ext and filename.endswith(f'.{final_ext}'):
|
|
filename = replace_extension(filename, ext, final_ext)
|
|
elif tmpl_type:
|
|
force_ext = OUTTMPL_TYPES[tmpl_type]
|
|
if force_ext:
|
|
filename = replace_extension(filename, force_ext, info_dict.get('ext'))
|
|
|
|
# https://github.com/blackjack4494/youtube-dlc/issues/85
|
|
trim_file_name = self.params.get('trim_file_name', False)
|
|
if trim_file_name:
|
|
no_ext, *ext = filename.rsplit('.', 2)
|
|
filename = join_nonempty(no_ext[:trim_file_name], *ext, delim='.')
|
|
|
|
return filename
|
|
except ValueError as err:
|
|
self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
|
|
return None
|
|
|
|
def prepare_filename(self, info_dict, dir_type='', *, outtmpl=None, warn=False):
|
|
"""Generate the output filename"""
|
|
if outtmpl:
|
|
assert not dir_type, 'outtmpl and dir_type are mutually exclusive'
|
|
dir_type = None
|
|
filename = self._prepare_filename(info_dict, tmpl_type=dir_type, outtmpl=outtmpl)
|
|
if not filename and dir_type not in ('', 'temp'):
|
|
return ''
|
|
|
|
if warn:
|
|
if not self.params.get('paths'):
|
|
pass
|
|
elif filename == '-':
|
|
self.report_warning('--paths is ignored when an outputting to stdout', only_once=True)
|
|
elif os.path.isabs(filename):
|
|
self.report_warning('--paths is ignored since an absolute path is given in output template', only_once=True)
|
|
if filename == '-' or not filename:
|
|
return filename
|
|
|
|
return self.get_output_path(dir_type, filename)
|
|
|
|
def _match_entry(self, info_dict, incomplete=False, silent=False):
|
|
"""Returns None if the file should be downloaded"""
|
|
_type = 'video' if 'playlist-match-filter' in self.params['compat_opts'] else info_dict.get('_type', 'video')
|
|
assert incomplete or _type == 'video', 'Only video result can be considered complete'
|
|
|
|
video_title = info_dict.get('title', info_dict.get('id', 'entry'))
|
|
|
|
def check_filter():
|
|
if _type in ('playlist', 'multi_video'):
|
|
return
|
|
elif _type in ('url', 'url_transparent') and not try_call(
|
|
lambda: self.get_info_extractor(info_dict['ie_key']).is_single_video(info_dict['url'])):
|
|
return
|
|
|
|
if 'title' in info_dict:
|
|
# This can happen when we're just evaluating the playlist
|
|
title = info_dict['title']
|
|
matchtitle = self.params.get('matchtitle', False)
|
|
if matchtitle:
|
|
if not re.search(matchtitle, title, re.IGNORECASE):
|
|
return '"' + title + '" title did not match pattern "' + matchtitle + '"'
|
|
rejecttitle = self.params.get('rejecttitle', False)
|
|
if rejecttitle:
|
|
if re.search(rejecttitle, title, re.IGNORECASE):
|
|
return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
|
|
|
|
date = info_dict.get('upload_date')
|
|
if date is not None:
|
|
dateRange = self.params.get('daterange', DateRange())
|
|
if date not in dateRange:
|
|
return f'{date_from_str(date).isoformat()} upload date is not in range {dateRange}'
|
|
view_count = info_dict.get('view_count')
|
|
if view_count is not None:
|
|
min_views = self.params.get('min_views')
|
|
if min_views is not None and view_count < min_views:
|
|
return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
|
|
max_views = self.params.get('max_views')
|
|
if max_views is not None and view_count > max_views:
|
|
return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
|
|
if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
|
|
return 'Skipping "%s" because it is age restricted' % video_title
|
|
|
|
match_filter = self.params.get('match_filter')
|
|
if match_filter is None:
|
|
return None
|
|
|
|
cancelled = None
|
|
try:
|
|
try:
|
|
ret = match_filter(info_dict, incomplete=incomplete)
|
|
except TypeError:
|
|
# For backward compatibility
|
|
ret = None if incomplete else match_filter(info_dict)
|
|
except DownloadCancelled as err:
|
|
if err.msg is not NO_DEFAULT:
|
|
raise
|
|
ret, cancelled = err.msg, err
|
|
|
|
if ret is NO_DEFAULT:
|
|
while True:
|
|
filename = self._format_screen(self.prepare_filename(info_dict), self.Styles.FILENAME)
|
|
reply = input(self._format_screen(
|
|
f'Download "{filename}"? (Y/n): ', self.Styles.EMPHASIS)).lower().strip()
|
|
if reply in {'y', ''}:
|
|
return None
|
|
elif reply == 'n':
|
|
if cancelled:
|
|
raise type(cancelled)(f'Skipping {video_title}')
|
|
return f'Skipping {video_title}'
|
|
return ret
|
|
|
|
if self.in_download_archive(info_dict):
|
|
reason = ''.join((
|
|
format_field(info_dict, 'id', f'{self._format_screen("%s", self.Styles.ID)}: '),
|
|
format_field(info_dict, 'title', f'{self._format_screen("%s", self.Styles.EMPHASIS)} '),
|
|
'has already been recorded in the archive'))
|
|
break_opt, break_err = 'break_on_existing', ExistingVideoReached
|
|
else:
|
|
try:
|
|
reason = check_filter()
|
|
except DownloadCancelled as e:
|
|
reason, break_opt, break_err = e.msg, 'match_filter', type(e)
|
|
else:
|
|
break_opt, break_err = 'break_on_reject', RejectedVideoReached
|
|
if reason is not None:
|
|
if not silent:
|
|
self.to_screen('[download] ' + reason)
|
|
if self.params.get(break_opt, False):
|
|
raise break_err()
|
|
return reason
|
|
|
|
@staticmethod
|
|
def add_extra_info(info_dict, extra_info):
|
|
'''Set the keys from extra_info in info dict if they are missing'''
|
|
for key, value in extra_info.items():
|
|
info_dict.setdefault(key, value)
|
|
|
|
def extract_info(self, url, download=True, ie_key=None, extra_info=None,
|
|
process=True, force_generic_extractor=False):
|
|
"""
|
|
Extract and return the information dictionary of the URL
|
|
|
|
Arguments:
|
|
@param url URL to extract
|
|
|
|
Keyword arguments:
|
|
@param download Whether to download videos
|
|
@param process Whether to resolve all unresolved references (URLs, playlist items).
|
|
Must be True for download to work
|
|
@param ie_key Use only the extractor with this key
|
|
|
|
@param extra_info Dictionary containing the extra values to add to the info (For internal use only)
|
|
@force_generic_extractor Force using the generic extractor (Deprecated; use ie_key='Generic')
|
|
"""
|
|
|
|
if extra_info is None:
|
|
extra_info = {}
|
|
|
|
if not ie_key and force_generic_extractor:
|
|
ie_key = 'Generic'
|
|
|
|
if ie_key:
|
|
ies = {ie_key: self._ies[ie_key]} if ie_key in self._ies else {}
|
|
else:
|
|
ies = self._ies
|
|
|
|
for key, ie in ies.items():
|
|
if not ie.suitable(url):
|
|
continue
|
|
|
|
if not ie.working():
|
|
self.report_warning('The program functionality for this site has been marked as broken, '
|
|
'and will probably not work.')
|
|
|
|
temp_id = ie.get_temp_id(url)
|
|
if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': key}):
|
|
self.to_screen(f'[download] {self._format_screen(temp_id, self.Styles.ID)}: '
|
|
'has already been recorded in the archive')
|
|
if self.params.get('break_on_existing', False):
|
|
raise ExistingVideoReached()
|
|
break
|
|
return self.__extract_info(url, self.get_info_extractor(key), download, extra_info, process)
|
|
else:
|
|
extractors_restricted = self.params.get('allowed_extractors') not in (None, ['default'])
|
|
self.report_error(f'No suitable extractor{format_field(ie_key, None, " (%s)")} found for URL {url}',
|
|
tb=False if extractors_restricted else None)
|
|
|
|
def _handle_extraction_exceptions(func):
|
|
@functools.wraps(func)
|
|
def wrapper(self, *args, **kwargs):
|
|
while True:
|
|
try:
|
|
return func(self, *args, **kwargs)
|
|
except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError):
|
|
raise
|
|
except ReExtractInfo as e:
|
|
if e.expected:
|
|
self.to_screen(f'{e}; Re-extracting data')
|
|
else:
|
|
self.to_stderr('\r')
|
|
self.report_warning(f'{e}; Re-extracting data')
|
|
continue
|
|
except GeoRestrictedError as e:
|
|
msg = e.msg
|
|
if e.countries:
|
|
msg += '\nThis video is available in %s.' % ', '.join(
|
|
map(ISO3166Utils.short2full, e.countries))
|
|
msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
|
|
self.report_error(msg)
|
|
except ExtractorError as e: # An error we somewhat expected
|
|
self.report_error(str(e), e.format_traceback())
|
|
except Exception as e:
|
|
if self.params.get('ignoreerrors'):
|
|
self.report_error(str(e), tb=encode_compat_str(traceback.format_exc()))
|
|
else:
|
|
raise
|
|
break
|
|
return wrapper
|
|
|
|
def _wait_for_video(self, ie_result={}):
|
|
if (not self.params.get('wait_for_video')
|
|
or ie_result.get('_type', 'video') != 'video'
|
|
or ie_result.get('formats') or ie_result.get('url')):
|
|
return
|
|
|
|
format_dur = lambda dur: '%02d:%02d:%02d' % timetuple_from_msec(dur * 1000)[:-1]
|
|
last_msg = ''
|
|
|
|
def progress(msg):
|
|
nonlocal last_msg
|
|
full_msg = f'{msg}\n'
|
|
if not self.params.get('noprogress'):
|
|
full_msg = msg + ' ' * (len(last_msg) - len(msg)) + '\r'
|
|
elif last_msg:
|
|
return
|
|
self.to_screen(full_msg, skip_eol=True)
|
|
last_msg = msg
|
|
|
|
min_wait, max_wait = self.params.get('wait_for_video')
|
|
diff = try_get(ie_result, lambda x: x['release_timestamp'] - time.time())
|
|
if diff is None and ie_result.get('live_status') == 'is_upcoming':
|
|
diff = round(random.uniform(min_wait, max_wait) if (max_wait and min_wait) else (max_wait or min_wait), 0)
|
|
self.report_warning('Release time of video is not known')
|
|
elif ie_result and (diff or 0) <= 0:
|
|
self.report_warning('Video should already be available according to extracted info')
|
|
diff = min(max(diff or 0, min_wait or 0), max_wait or float('inf'))
|
|
self.to_screen(f'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
|
|
|
|
wait_till = time.time() + diff
|
|
try:
|
|
while True:
|
|
diff = wait_till - time.time()
|
|
if diff <= 0:
|
|
progress('')
|
|
raise ReExtractInfo('[wait] Wait period ended', expected=True)
|
|
progress(f'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}')
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
progress('')
|
|
raise ReExtractInfo('[wait] Interrupted by user', expected=True)
|
|
except BaseException as e:
|
|
if not isinstance(e, ReExtractInfo):
|
|
self.to_screen('')
|
|
raise
|
|
|
|
def _load_cookies(self, data, *, autoscope=True):
|
|
"""Loads cookies from a `Cookie` header
|
|
|
|
This tries to work around the security vulnerability of passing cookies to every domain.
|
|
See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
|
|
|
|
@param data The Cookie header as string to load the cookies from
|
|
@param autoscope If `False`, scope cookies using Set-Cookie syntax and error for cookie without domains
|
|
If `True`, save cookies for later to be stored in the jar with a limited scope
|
|
If a URL, save cookies in the jar with the domain of the URL
|
|
"""
|
|
for cookie in LenientSimpleCookie(data).values():
|
|
if autoscope and any(cookie.values()):
|
|
raise ValueError('Invalid syntax in Cookie Header')
|
|
|
|
domain = cookie.get('domain') or ''
|
|
expiry = cookie.get('expires')
|
|
if expiry == '': # 0 is valid
|
|
expiry = None
|
|
prepared_cookie = http.cookiejar.Cookie(
|
|
cookie.get('version') or 0, cookie.key, cookie.value, None, False,
|
|
domain, True, True, cookie.get('path') or '', bool(cookie.get('path')),
|
|
cookie.get('secure') or False, expiry, False, None, None, {})
|
|
|
|
if domain:
|
|
self.cookiejar.set_cookie(prepared_cookie)
|
|
elif autoscope is True:
|
|
self.deprecated_feature(
|
|
'Passing cookies as a header is a potential security risk; '
|
|
'they will be scoped to the domain of the downloaded urls. '
|
|
'Please consider loading cookies from a file or browser instead.')
|
|
self.__header_cookies.append(prepared_cookie)
|
|
elif autoscope:
|
|
self.report_warning(
|
|
'The extractor result contains an unscoped cookie as an HTTP header. '
|
|
f'If you are using yt-dlp with an input URL{bug_reports_message(before=",")}',
|
|
only_once=True)
|
|
self._apply_header_cookies(autoscope, [prepared_cookie])
|
|
else:
|
|
self.report_error('Unscoped cookies are not allowed; please specify some sort of scoping',
|
|
tb=False, is_error=False)
|
|
|
|
def _apply_header_cookies(self, url, cookies=None):
|
|
"""Applies stray header cookies to the provided url
|
|
|
|
This loads header cookies and scopes them to the domain provided in `url`.
|
|
While this is not ideal, it helps reduce the risk of them being sent
|
|
to an unintended destination while mostly maintaining compatibility.
|
|
"""
|
|
parsed = urllib.parse.urlparse(url)
|
|
if not parsed.hostname:
|
|
return
|
|
|
|
for cookie in map(copy.copy, cookies or self.__header_cookies):
|
|
cookie.domain = f'.{parsed.hostname}'
|
|
self.cookiejar.set_cookie(cookie)
|
|
|
|
@_handle_extraction_exceptions
|
|
def __extract_info(self, url, ie, download, extra_info, process):
|
|
self._apply_header_cookies(url)
|
|
|
|
try:
|
|
ie_result = ie.extract(url)
|
|
except UserNotLive as e:
|
|
if process:
|
|
if self.params.get('wait_for_video'):
|
|
self.report_warning(e)
|
|
self._wait_for_video()
|
|
raise
|
|
if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
|
|
self.report_warning(f'Extractor {ie.IE_NAME} returned nothing{bug_reports_message()}')
|
|
return
|
|
if isinstance(ie_result, list):
|
|
# Backwards compatibility: old IE result format
|
|
ie_result = {
|
|
'_type': 'compat_list',
|
|
'entries': ie_result,
|
|
}
|
|
if extra_info.get('original_url'):
|
|
ie_result.setdefault('original_url', extra_info['original_url'])
|
|
self.add_default_extra_info(ie_result, ie, url)
|
|
if process:
|
|
self._wait_for_video(ie_result)
|
|
return self.process_ie_result(ie_result, download, extra_info)
|
|
else:
|
|
return ie_result
|
|
|
|
def add_default_extra_info(self, ie_result, ie, url):
|
|
if url is not None:
|
|
self.add_extra_info(ie_result, {
|
|
'webpage_url': url,
|
|
'original_url': url,
|
|
})
|
|
webpage_url = ie_result.get('webpage_url')
|
|
if webpage_url:
|
|
self.add_extra_info(ie_result, {
|
|
'webpage_url_basename': url_basename(webpage_url),
|
|
'webpage_url_domain': get_domain(webpage_url),
|
|
})
|
|
if ie is not None:
|
|
self.add_extra_info(ie_result, {
|
|
'extractor': ie.IE_NAME,
|
|
'extractor_key': ie.ie_key(),
|
|
})
|
|
|
|
def process_ie_result(self, ie_result, download=True, extra_info=None):
|
|
"""
|
|
Take the result of the ie(may be modified) and resolve all unresolved
|
|
references (URLs, playlist items).
|
|
|
|
It will also download the videos if 'download'.
|
|
Returns the resolved ie_result.
|
|
"""
|
|
if extra_info is None:
|
|
extra_info = {}
|
|
result_type = ie_result.get('_type', 'video')
|
|
|
|
if result_type in ('url', 'url_transparent'):
|
|
ie_result['url'] = sanitize_url(
|
|
ie_result['url'], scheme='http' if self.params.get('prefer_insecure') else 'https')
|
|
if ie_result.get('original_url') and not extra_info.get('original_url'):
|
|
extra_info = {'original_url': ie_result['original_url'], **extra_info}
|
|
|
|
extract_flat = self.params.get('extract_flat', False)
|
|
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
|
|
or extract_flat is True):
|
|
info_copy = ie_result.copy()
|
|
ie = try_get(ie_result.get('ie_key'), self.get_info_extractor)
|
|
if ie and not ie_result.get('id'):
|
|
info_copy['id'] = ie.get_temp_id(ie_result['url'])
|
|
self.add_default_extra_info(info_copy, ie, ie_result['url'])
|
|
self.add_extra_info(info_copy, extra_info)
|
|
info_copy, _ = self.pre_process(info_copy)
|
|
self._fill_common_fields(info_copy, False)
|
|
self.__forced_printings(info_copy)
|
|
self._raise_pending_errors(info_copy)
|
|
if self.params.get('force_write_download_archive', False):
|
|
self.record_download_archive(info_copy)
|
|
return ie_result
|
|
|
|
if result_type == 'video':
|
|
self.add_extra_info(ie_result, extra_info)
|
|
ie_result = self.process_video_result(ie_result, download=download)
|
|
self._raise_pending_errors(ie_result)
|
|
additional_urls = (ie_result or {}).get('additional_urls')
|
|
if additional_urls:
|
|
# TODO: Improve MetadataParserPP to allow setting a list
|
|
if isinstance(additional_urls, str):
|
|
additional_urls = [additional_urls]
|
|
self.to_screen(
|
|
'[info] %s: %d additional URL(s) requested' % (ie_result['id'], len(additional_urls)))
|
|
self.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls))
|
|
ie_result['additional_entries'] = [
|
|
self.extract_info(
|
|
url, download, extra_info=extra_info,
|
|
force_generic_extractor=self.params.get('force_generic_extractor'))
|
|
for url in additional_urls
|
|
]
|
|
return ie_result
|
|
elif result_type == 'url':
|
|
# We have to add extra_info to the results because it may be
|
|
# contained in a playlist
|
|
return self.extract_info(
|
|
ie_result['url'], download,
|
|
ie_key=ie_result.get('ie_key'),
|
|
extra_info=extra_info)
|
|
elif result_type == 'url_transparent':
|
|
# Use the information from the embedding page
|
|
info = self.extract_info(
|
|
ie_result['url'], ie_key=ie_result.get('ie_key'),
|
|
extra_info=extra_info, download=False, process=False)
|
|
|
|
# extract_info may return None when ignoreerrors is enabled and
|
|
# extraction failed with an error, don't crash and return early
|
|
# in this case
|
|
if not info:
|
|
return info
|
|
|
|
exempted_fields = {'_type', 'url', 'ie_key'}
|
|
if not ie_result.get('section_end') and ie_result.get('section_start') is None:
|
|
# For video clips, the id etc of the clip extractor should be used
|
|
exempted_fields |= {'id', 'extractor', 'extractor_key'}
|
|
|
|
new_result = info.copy()
|
|
new_result.update(filter_dict(ie_result, lambda k, v: v is not None and k not in exempted_fields))
|
|
|
|
# Extracted info may not be a video result (i.e.
|
|
# info.get('_type', 'video') != video) but rather an url or
|
|
# url_transparent. In such cases outer metadata (from ie_result)
|
|
# should be propagated to inner one (info). For this to happen
|
|
# _type of info should be overridden with url_transparent. This
|
|
# fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
|
|
if new_result.get('_type') == 'url':
|
|
new_result['_type'] = 'url_transparent'
|
|
|
|
return self.process_ie_result(
|
|
new_result, download=download, extra_info=extra_info)
|
|
elif result_type in ('playlist', 'multi_video'):
|
|
# Protect from infinite recursion due to recursively nested playlists
|
|
# (see https://github.com/ytdl-org/youtube-dl/issues/27833)
|
|
webpage_url = ie_result.get('webpage_url') # Playlists maynot have webpage_url
|
|
if webpage_url and webpage_url in self._playlist_urls:
|
|
self.to_screen(
|
|
'[download] Skipping already downloaded playlist: %s'
|
|
% ie_result.get('title') or ie_result.get('id'))
|
|
return
|
|
|
|
self._playlist_level += 1
|
|
self._playlist_urls.add(webpage_url)
|
|
self._fill_common_fields(ie_result, False)
|
|
self._sanitize_thumbnails(ie_result)
|
|
try:
|
|
return self.__process_playlist(ie_result, download)
|
|
finally:
|
|
self._playlist_level -= 1
|
|
if not self._playlist_level:
|
|
self._playlist_urls.clear()
|
|
elif result_type == 'compat_list':
|
|
self.report_warning(
|
|
'Extractor %s returned a compat_list result. '
|
|
'It needs to be updated.' % ie_result.get('extractor'))
|
|
|
|
def _fixup(r):
|
|
self.add_extra_info(r, {
|
|
'extractor': ie_result['extractor'],
|
|
'webpage_url': ie_result['webpage_url'],
|
|
'webpage_url_basename': url_basename(ie_result['webpage_url']),
|
|
'webpage_url_domain': get_domain(ie_result['webpage_url']),
|
|
'extractor_key': ie_result['extractor_key'],
|
|
})
|
|
return r
|
|
ie_result['entries'] = [
|
|
self.process_ie_result(_fixup(r), download, extra_info)
|
|
for r in ie_result['entries']
|
|
]
|
|
return ie_result
|
|
else:
|
|
raise Exception('Invalid result type: %s' % result_type)
|
|
|
|
def _ensure_dir_exists(self, path):
|
|
return make_dir(path, self.report_error)
|
|
|
|
@staticmethod
|
|
def _playlist_infodict(ie_result, strict=False, **kwargs):
|
|
info = {
|
|
'playlist_count': ie_result.get('playlist_count'),
|
|
'playlist': ie_result.get('title') or ie_result.get('id'),
|
|
'playlist_id': ie_result.get('id'),
|
|
'playlist_title': ie_result.get('title'),
|
|
'playlist_uploader': ie_result.get('uploader'),
|
|
'playlist_uploader_id': ie_result.get('uploader_id'),
|
|
**kwargs,
|
|
}
|
|
if strict:
|
|
return info
|
|
if ie_result.get('webpage_url'):
|
|
info.update({
|
|
'webpage_url': ie_result['webpage_url'],
|
|
'webpage_url_basename': url_basename(ie_result['webpage_url']),
|
|
'webpage_url_domain': get_domain(ie_result['webpage_url']),
|
|
})
|
|
return {
|
|
**info,
|
|
'playlist_index': 0,
|
|
'__last_playlist_index': max(ie_result.get('requested_entries') or (0, 0)),
|
|
'extractor': ie_result['extractor'],
|
|
'extractor_key': ie_result['extractor_key'],
|
|
}
|
|
|
|
def __process_playlist(self, ie_result, download):
|
|
"""Process each entry in the playlist"""
|
|
assert ie_result['_type'] in ('playlist', 'multi_video')
|
|
|
|
common_info = self._playlist_infodict(ie_result, strict=True)
|
|
title = common_info.get('playlist') or '<Untitled>'
|
|
if self._match_entry(common_info, incomplete=True) is not None:
|
|
return
|
|
self.to_screen(f'[download] Downloading {ie_result["_type"]}: {title}')
|
|
|
|
all_entries = PlaylistEntries(self, ie_result)
|
|
entries = orderedSet(all_entries.get_requested_items(), lazy=True)
|
|
|
|
lazy = self.params.get('lazy_playlist')
|
|
if lazy:
|
|
resolved_entries, n_entries = [], 'N/A'
|
|
ie_result['requested_entries'], ie_result['entries'] = None, None
|
|
else:
|
|
entries = resolved_entries = list(entries)
|
|
n_entries = len(resolved_entries)
|
|
ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries)) or ([], [])
|
|
if not ie_result.get('playlist_count'):
|
|
# Better to do this after potentially exhausting entries
|
|
ie_result['playlist_count'] = all_entries.get_full_count()
|
|
|
|
extra = self._playlist_infodict(ie_result, n_entries=int_or_none(n_entries))
|
|
ie_copy = collections.ChainMap(ie_result, extra)
|
|
|
|
_infojson_written = False
|
|
write_playlist_files = self.params.get('allow_playlist_files', True)
|
|
if write_playlist_files and self.params.get('list_thumbnails'):
|
|
self.list_thumbnails(ie_result)
|
|
if write_playlist_files and not self.params.get('simulate'):
|
|
_infojson_written = self._write_info_json(
|
|
'playlist', ie_result, self.prepare_filename(ie_copy, 'pl_infojson'))
|
|
if _infojson_written is None:
|
|
return
|
|
if self._write_description('playlist', ie_result,
|
|
self.prepare_filename(ie_copy, 'pl_description')) is None:
|
|
return
|
|
# TODO: This should be passed to ThumbnailsConvertor if necessary
|
|
self._write_thumbnails('playlist', ie_result, self.prepare_filename(ie_copy, 'pl_thumbnail'))
|
|
|
|
if lazy:
|
|
if self.params.get('playlistreverse') or self.params.get('playlistrandom'):
|
|
self.report_warning('playlistreverse and playlistrandom are not supported with lazy_playlist', only_once=True)
|
|
elif self.params.get('playlistreverse'):
|
|
entries.reverse()
|
|
elif self.params.get('playlistrandom'):
|
|
random.shuffle(entries)
|
|
|
|
self.to_screen(f'[{ie_result["extractor"]}] Playlist {title}: Downloading {n_entries} items'
|
|
f'{format_field(ie_result, "playlist_count", " of %s")}')
|
|
|
|
keep_resolved_entries = self.params.get('extract_flat') != 'discard'
|
|
if self.params.get('extract_flat') == 'discard_in_playlist':
|
|
keep_resolved_entries = ie_result['_type'] != 'playlist'
|
|
if keep_resolved_entries:
|
|
self.write_debug('The information of all playlist entries will be held in memory')
|
|
|
|
failures = 0
|
|
max_failures = self.params.get('skip_playlist_after_errors') or float('inf')
|
|
for i, (playlist_index, entry) in enumerate(entries):
|
|
if lazy:
|
|
resolved_entries.append((playlist_index, entry))
|
|
if not entry:
|
|
continue
|
|
|
|
entry['__x_forwarded_for_ip'] = ie_result.get('__x_forwarded_for_ip')
|
|
if not lazy and 'playlist-index' in self.params['compat_opts']:
|
|
playlist_index = ie_result['requested_entries'][i]
|
|
|
|
entry_copy = collections.ChainMap(entry, {
|
|
**common_info,
|
|
'n_entries': int_or_none(n_entries),
|
|
'playlist_index': playlist_index,
|
|
'playlist_autonumber': i + 1,
|
|
})
|
|
|
|
if self._match_entry(entry_copy, incomplete=True) is not None:
|
|
# For compatabilty with youtube-dl. See https://github.com/yt-dlp/yt-dlp/issues/4369
|
|
resolved_entries[i] = (playlist_index, NO_DEFAULT)
|
|
continue
|
|
|
|
self.to_screen('[download] Downloading item %s of %s' % (
|
|
self._format_screen(i + 1, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
|
|
|
|
entry_result = self.__process_iterable_entry(entry, download, collections.ChainMap({
|
|
'playlist_index': playlist_index,
|
|
'playlist_autonumber': i + 1,
|
|
}, extra))
|
|
if not entry_result:
|
|
failures += 1
|
|
if failures >= max_failures:
|
|
self.report_error(
|
|
f'Skipping the remaining entries in playlist "{title}" since {failures} items failed extraction')
|
|
break
|
|
if keep_resolved_entries:
|
|
resolved_entries[i] = (playlist_index, entry_result)
|
|
|
|
# Update with processed data
|
|
ie_result['entries'] = [e for _, e in resolved_entries if e is not NO_DEFAULT]
|
|
ie_result['requested_entries'] = [i for i, e in resolved_entries if e is not NO_DEFAULT]
|
|
if ie_result['requested_entries'] == try_call(lambda: list(range(1, ie_result['playlist_count'] + 1))):
|
|
# Do not set for full playlist
|
|
ie_result.pop('requested_entries')
|
|
|
|
# Write the updated info to json
|
|
if _infojson_written is True and self._write_info_json(
|
|
'updated playlist', ie_result,
|
|
self.prepare_filename(ie_copy, 'pl_infojson'), overwrite=True) is None:
|
|
return
|
|
|
|
ie_result = self.run_all_pps('playlist', ie_result)
|
|
self.to_screen(f'[download] Finished downloading playlist: {title}')
|
|
return ie_result
|
|
|
|
@_handle_extraction_exceptions
|
|
def __process_iterable_entry(self, entry, download, extra_info):
|
|
return self.process_ie_result(
|
|
entry, download=download, extra_info=extra_info)
|
|
|
|
def _build_format_filter(self, filter_spec):
|
|
" Returns a function to filter the formats according to the filter_spec "
|
|
|
|
OPERATORS = {
|
|
'<': operator.lt,
|
|
'<=': operator.le,
|
|
'>': operator.gt,
|
|
'>=': operator.ge,
|
|
'=': operator.eq,
|
|
'!=': operator.ne,
|
|
}
|
|
operator_rex = re.compile(r'''(?x)\s*
|
|
(?P<key>[\w.-]+)\s*
|
|
(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
|
|
(?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
|
|
''' % '|'.join(map(re.escape, OPERATORS.keys())))
|
|
m = operator_rex.fullmatch(filter_spec)
|
|
if m:
|
|
try:
|
|
comparison_value = int(m.group('value'))
|
|
except ValueError:
|
|
comparison_value = parse_filesize(m.group('value'))
|
|
if comparison_value is None:
|
|
comparison_value = parse_filesize(m.group('value') + 'B')
|
|
if comparison_value is None:
|
|
raise ValueError(
|
|
'Invalid value %r in format specification %r' % (
|
|
m.group('value'), filter_spec))
|
|
op = OPERATORS[m.group('op')]
|
|
|
|
if not m:
|
|
STR_OPERATORS = {
|
|
'=': operator.eq,
|
|
'^=': lambda attr, value: attr.startswith(value),
|
|
'$=': lambda attr, value: attr.endswith(value),
|
|
'*=': lambda attr, value: value in attr,
|
|
'~=': lambda attr, value: value.search(attr) is not None
|
|
}
|
|
str_operator_rex = re.compile(r'''(?x)\s*
|
|
(?P<key>[a-zA-Z0-9._-]+)\s*
|
|
(?P<negation>!\s*)?(?P<op>%s)\s*(?P<none_inclusive>\?\s*)?
|
|
(?P<quote>["'])?
|
|
(?P<value>(?(quote)(?:(?!(?P=quote))[^\\]|\\.)+|[\w.-]+))
|
|
(?(quote)(?P=quote))\s*
|
|
''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
|
|
m = str_operator_rex.fullmatch(filter_spec)
|
|
if m:
|
|
if m.group('op') == '~=':
|
|
comparison_value = re.compile(m.group('value'))
|
|
else:
|
|
comparison_value = re.sub(r'''\\([\\"'])''', r'\1', m.group('value'))
|
|
str_op = STR_OPERATORS[m.group('op')]
|
|
if m.group('negation'):
|
|
op = lambda attr, value: not str_op(attr, value)
|
|
else:
|
|
op = str_op
|
|
|
|
if not m:
|
|
raise SyntaxError('Invalid filter specification %r' % filter_spec)
|
|
|
|
def _filter(f):
|
|
actual_value = f.get(m.group('key'))
|
|
if actual_value is None:
|
|
return m.group('none_inclusive')
|
|
return op(actual_value, comparison_value)
|
|
return _filter
|
|
|
|
def _check_formats(self, formats):
|
|
for f in formats:
|
|
self.to_screen('[info] Testing format %s' % f['format_id'])
|
|
path = self.get_output_path('temp')
|
|
if not self._ensure_dir_exists(f'{path}/'):
|
|
continue
|
|
temp_file = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False, dir=path or None)
|
|
temp_file.close()
|
|
try:
|
|
success, _ = self.dl(temp_file.name, f, test=True)
|
|
except (DownloadError, OSError, ValueError) + network_exceptions:
|
|
success = False
|
|
finally:
|
|
if os.path.exists(temp_file.name):
|
|
try:
|
|
os.remove(temp_file.name)
|
|
except OSError:
|
|
self.report_warning('Unable to delete temporary file "%s"' % temp_file.name)
|
|
if success:
|
|
yield f
|
|
else:
|
|
self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
|
|
|
|
def _default_format_spec(self, info_dict, download=True):
|
|
|
|
def can_merge():
|
|
merger = FFmpegMergerPP(self)
|
|
return merger.available and merger.can_merge()
|
|
|
|
prefer_best = (
|
|
not self.params.get('simulate')
|
|
and download
|
|
and (
|
|
not can_merge()
|
|
or info_dict.get('is_live') and not self.params.get('live_from_start')
|
|
or self.params['outtmpl']['default'] == '-'))
|
|
compat = (
|
|
prefer_best
|
|
or self.params.get('allow_multiple_audio_streams', False)
|
|
or 'format-spec' in self.params['compat_opts'])
|
|
|
|
return (
|
|
'best/bestvideo+bestaudio' if prefer_best
|
|
else 'bestvideo*+bestaudio/best' if not compat
|
|
else 'bestvideo+bestaudio/best')
|
|
|
|
def build_format_selector(self, format_spec):
|
|
def syntax_error(note, start):
|
|
message = (
|
|
'Invalid format specification: '
|
|
'{}\n\t{}\n\t{}^'.format(note, format_spec, ' ' * start[1]))
|
|
return SyntaxError(message)
|
|
|
|
PICKFIRST = 'PICKFIRST'
|
|
MERGE = 'MERGE'
|
|
SINGLE = 'SINGLE'
|
|
GROUP = 'GROUP'
|
|
FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
|
|
|
|
allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False),
|
|
'video': self.params.get('allow_multiple_video_streams', False)}
|
|
|
|
def _parse_filter(tokens):
|
|
filter_parts = []
|
|
for type, string_, start, _, _ in tokens:
|
|
if type == tokenize.OP and string_ == ']':
|
|
return ''.join(filter_parts)
|
|
else:
|
|
filter_parts.append(string_)
|
|
|
|
def _remove_unused_ops(tokens):
|
|
# Remove operators that we don't use and join them with the surrounding strings.
|
|
# E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
|
|
ALLOWED_OPS = ('/', '+', ',', '(', ')')
|
|
last_string, last_start, last_end, last_line = None, None, None, None
|
|
for type, string_, start, end, line in tokens:
|
|
if type == tokenize.OP and string_ == '[':
|
|
if last_string:
|
|
yield tokenize.NAME, last_string, last_start, last_end, last_line
|
|
last_string = None
|
|
yield type, string_, start, end, line
|
|
# everything inside brackets will be handled by _parse_filter
|
|
for type, string_, start, end, line in tokens:
|
|
yield type, string_, start, end, line
|
|
if type == tokenize.OP and string_ == ']':
|
|
break
|
|
elif type == tokenize.OP and string_ in ALLOWED_OPS:
|
|
if last_string:
|
|
yield tokenize.NAME, last_string, last_start, last_end, last_line
|
|
last_string = None
|
|
yield type, string_, start, end, line
|
|
elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
|
|
if not last_string:
|
|
last_string = string_
|
|
last_start = start
|
|
last_end = end
|
|
else:
|
|
last_string += string_
|
|
if last_string:
|
|
yield tokenize.NAME, last_string, last_start, last_end, last_line
|
|
|
|
def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
|
|
selectors = []
|
|
current_selector = None
|
|
for type, string_, start, _, _ in tokens:
|
|
# ENCODING is only defined in python 3.x
|
|
if type == getattr(tokenize, 'ENCODING', None):
|
|
continue
|
|
elif type in [tokenize.NAME, tokenize.NUMBER]:
|
|
current_selector = FormatSelector(SINGLE, string_, [])
|
|
elif type == tokenize.OP:
|
|
if string_ == ')':
|
|
if not inside_group:
|
|
# ')' will be handled by the parentheses group
|
|
tokens.restore_last_token()
|
|
break
|
|
elif inside_merge and string_ in ['/', ',']:
|
|
tokens.restore_last_token()
|
|
break
|
|
elif inside_choice and string_ == ',':
|
|
tokens.restore_last_token()
|
|
break
|
|
elif string_ == ',':
|
|
if not current_selector:
|
|
raise syntax_error('"," must follow a format selector', start)
|
|
selectors.append(current_selector)
|
|
current_selector = None
|
|
elif string_ == '/':
|
|
if not current_selector:
|
|
raise syntax_error('"/" must follow a format selector', start)
|
|
first_choice = current_selector
|
|
second_choice = _parse_format_selection(tokens, inside_choice=True)
|
|
current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
|
|
elif string_ == '[':
|
|
if not current_selector:
|
|
current_selector = FormatSelector(SINGLE, 'best', [])
|
|
format_filter = _parse_filter(tokens)
|
|
current_selector.filters.append(format_filter)
|
|
elif string_ == '(':
|
|
if current_selector:
|
|
raise syntax_error('Unexpected "("', start)
|
|
group = _parse_format_selection(tokens, inside_group=True)
|
|
current_selector = FormatSelector(GROUP, group, [])
|
|
elif string_ == '+':
|
|
if not current_selector:
|
|
raise syntax_error('Unexpected "+"', start)
|
|
selector_1 = current_selector
|
|
selector_2 = _parse_format_selection(tokens, inside_merge=True)
|
|
if not selector_2:
|
|
raise syntax_error('Expected a selector', start)
|
|
current_selector = FormatSelector(MERGE, (selector_1, selector_2), [])
|
|
else:
|
|
raise syntax_error(f'Operator not recognized: "{string_}"', start)
|
|
elif type == tokenize.ENDMARKER:
|
|
break
|
|
if current_selector:
|
|
selectors.append(current_selector)
|
|
return selectors
|
|
|
|
def _merge(formats_pair):
|
|
format_1, format_2 = formats_pair
|
|
|
|
formats_info = []
|
|
formats_info.extend(format_1.get('requested_formats', (format_1,)))
|
|
formats_info.extend(format_2.get('requested_formats', (format_2,)))
|
|
|
|
if not allow_multiple_streams['video'] or not allow_multiple_streams['audio']:
|
|
get_no_more = {'video': False, 'audio': False}
|
|
for (i, fmt_info) in enumerate(formats_info):
|
|
if fmt_info.get('acodec') == fmt_info.get('vcodec') == 'none':
|
|
formats_info.pop(i)
|
|
continue
|
|
for aud_vid in ['audio', 'video']:
|
|
if not allow_multiple_streams[aud_vid] and fmt_info.get(aud_vid[0] + 'codec') != 'none':
|
|
if get_no_more[aud_vid]:
|
|
formats_info.pop(i)
|
|
break
|
|
get_no_more[aud_vid] = True
|
|
|
|
if len(formats_info) == 1:
|
|
return formats_info[0]
|
|
|
|
video_fmts = [fmt_info for fmt_info in formats_info if fmt_info.get('vcodec') != 'none']
|
|
audio_fmts = [fmt_info for fmt_info in formats_info if fmt_info.get('acodec') != 'none']
|
|
|
|
the_only_video = video_fmts[0] if len(video_fmts) == 1 else None
|
|
the_only_audio = audio_fmts[0] if len(audio_fmts) == 1 else None
|
|
|
|
output_ext = get_compatible_ext(
|
|
vcodecs=[f.get('vcodec') for |