[yandexdisk] extract info from webpage

the public API does not return metadata when download limit is reached
This commit is contained in:
Remita Amine 2020-12-30 16:45:53 +01:00
parent bd18824c2a
commit a6f75e6e89

View file

@ -2,24 +2,23 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import json import json
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
ExtractorError,
float_or_none, float_or_none,
int_or_none, int_or_none,
mimetype2ext, mimetype2ext,
parse_iso8601, try_get,
urljoin, urljoin,
) )
class YandexDiskIE(InfoExtractor): class YandexDiskIE(InfoExtractor):
_VALID_URL = r'''(?x)https?:// _VALID_URL = r'''(?x)https?://
(?: (?P<domain>
(?:www\.)?yadi\.sk| yadi\.sk|
disk\.yandex\. disk\.yandex\.
(?: (?:
az| az|
@ -38,7 +37,7 @@ class YandexDiskIE(InfoExtractor):
_TESTS = [{ _TESTS = [{
'url': 'https://yadi.sk/i/VdOeDou8eZs6Y', 'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
'md5': '33955d7ae052f15853dc41f35f17581c', 'md5': 'a4a8d52958c8fddcf9845935070402ae',
'info_dict': { 'info_dict': {
'id': 'VdOeDou8eZs6Y', 'id': 'VdOeDou8eZs6Y',
'ext': 'mp4', 'ext': 'mp4',
@ -46,10 +45,9 @@ class YandexDiskIE(InfoExtractor):
'duration': 168.6, 'duration': 168.6,
'uploader': 'y.botova', 'uploader': 'y.botova',
'uploader_id': '300043621', 'uploader_id': '300043621',
'timestamp': 1421396809,
'upload_date': '20150116',
'view_count': int, 'view_count': int,
}, },
'expected_warnings': ['Unable to download JSON metadata'],
}, { }, {
'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce', 'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
'only_matching': True, 'only_matching': True,
@ -59,51 +57,58 @@ class YandexDiskIE(InfoExtractor):
}] }]
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) domain, video_id = re.match(self._VALID_URL, url).groups()
try: webpage = self._download_webpage(url, video_id)
resource = self._download_json( store = self._parse_json(self._search_regex(
'https://cloud-api.yandex.net/v1/disk/public/resources', r'<script[^>]+id="store-prefetch"[^>]*>\s*({.+?})\s*</script>',
video_id, query={'public_key': url}) webpage, 'store'), video_id)
except ExtractorError as e: resource = store['resources'][store['rootResourceId']]
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
error_description = self._parse_json(
e.cause.read().decode(), video_id)['description']
raise ExtractorError(error_description, expected=True)
raise
title = resource['name'] title = resource['name']
public_url = resource.get('public_url') meta = resource.get('meta') or {}
public_url = meta.get('short_url')
if public_url: if public_url:
video_id = self._match_id(public_url) video_id = self._match_id(public_url)
self._set_cookie('yadi.sk', 'yandexuid', '0') source_url = (self._download_json(
'https://cloud-api.yandex.net/v1/disk/public/resources/download',
video_id, query={'public_key': url}, fatal=False) or {}).get('href')
video_streams = resource.get('videoStreams') or {}
video_hash = resource.get('hash') or url
environment = store.get('environment') or {}
sk = environment.get('sk')
yandexuid = environment.get('yandexuid')
if sk and yandexuid and not (source_url and video_streams):
self._set_cookie(domain, 'yandexuid', yandexuid)
def call_api(action): def call_api(action):
return (self._download_json( return (self._download_json(
urljoin(url, '/public/api/') + action, video_id, data=json.dumps({ urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
'hash': url, 'hash': video_hash,
# obtain sk if needed from call_api('check-auth') while 'sk': sk,
# the yandexuid cookie is set and sending an empty JSON object }).encode(), headers={
'sk': 'ya6b52f8c6b12abe91a66d22d3a31084b' 'Content-Type': 'text/plain',
}).encode(), headers={ }, fatal=False) or {}).get('data') or {}
'Content-Type': 'text/plain', if not source_url:
}, fatal=False) or {}).get('data') or {} # TODO: figure out how to detect if download limit has
# been reached and then avoid unnecessary source format
# extraction requests
source_url = call_api('download-url').get('url')
if not video_streams:
video_streams = call_api('get-video-streams')
formats = [] formats = []
source_url = resource.get('file')
if not source_url:
source_url = call_api('download-url').get('url')
if source_url: if source_url:
formats.append({ formats.append({
'url': source_url, 'url': source_url,
'format_id': 'source', 'format_id': 'source',
'ext': determine_ext(title, mimetype2ext(resource.get('mime_type')) or 'mp4'), 'ext': determine_ext(title, meta.get('ext') or mimetype2ext(meta.get('mime_type')) or 'mp4'),
'quality': 1, 'quality': 1,
'filesize': int_or_none(resource.get('size')) 'filesize': int_or_none(meta.get('size'))
}) })
video_streams = call_api('get-video-streams')
for video in (video_streams.get('videos') or []): for video in (video_streams.get('videos') or []):
format_url = video.get('url') format_url = video.get('url')
if not format_url: if not format_url:
@ -128,15 +133,15 @@ class YandexDiskIE(InfoExtractor):
}) })
self._sort_formats(formats) self._sort_formats(formats)
owner = resource.get('owner') or {} uid = resource.get('uid')
display_name = try_get(store, lambda x: x['users'][uid]['displayName'])
return { return {
'id': video_id, 'id': video_id,
'title': title, 'title': title,
'duration': float_or_none(video_streams.get('duration'), 1000), 'duration': float_or_none(video_streams.get('duration'), 1000),
'uploader': owner.get('display_name'), 'uploader': display_name,
'uploader_id': owner.get('uid'), 'uploader_id': uid,
'view_count': int_or_none(resource.get('views_count')), 'view_count': int_or_none(meta.get('views_counter')),
'timestamp': parse_iso8601(resource.get('created')),
'formats': formats, 'formats': formats,
} }