[crunchyroll] extract old rtmp formats

youtube_dl/extractor/crunchyroll.py

@@ -38,6 +38,16 @@ class CrunchyrollBaseIE(InfoExtractor):
     _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
 
+    def _call_rpc_api(self, method, video_id, note=None, data=None):
+        data = data or {}
+        data['req'] = 'RpcApi' + method
+        data = compat_urllib_parse_urlencode(data).encode('utf-8')
+        return self._download_xml(
+            'http://www.crunchyroll.com/xml/',
+            video_id, note, fatal=False, data=data, headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
@@ -377,15 +387,19 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
-            sub_page = self._download_webpage(
+            sub_doc = self._call_rpc_api(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
+                'Subtitle_GetXml', video_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
+                'Downloading subtitles for ' + sub_name, data={
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
+                    'subtitle_script_id': sub_id,
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
+                })
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
+            if not sub_doc:
-            if not id or not iv or not data:
                 continue
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            sid = sub_doc.get('id')
+            iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
+            data = xpath_text(sub_doc, 'data', 'subtitle data')
+            if not sid or not iv or not data:
+                continue
+            subtitle = self._decrypt_subtitles(data, iv, sid).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
@@ -456,15 +470,29 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in available_fmts:
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = sanitized_Request(
+            stream_infos = []
-                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
+            streamdata = self._call_rpc_api(
-                % (video_id, stream_format, stream_quality),
+                'VideoPlayer_GetStandardConfig', video_id,
-                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
+                'Downloading media info for %s' % video_format, data={
-            streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+                    'media_id': video_id,
-            streamdata = self._download_xml(
+                    'video_format': stream_format,
-                streamdata_req, video_id,
+                    'video_quality': stream_quality,
-                note='Downloading media info for %s' % video_format)
+                    'current_page': url,
+                })
+            if streamdata:
                 stream_info = streamdata.find('./{default}preload/stream_info')
+                if stream_info:
+                    stream_infos.append(stream_info)
+            stream_info = self._call_rpc_api(
+                'VideoEncode_GetStreamInfo', video_id,
+                'Downloading stream info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_encode_quality': stream_quality,
+                })
+            if stream_info:
+                stream_infos.append(stream_info)
+            for stream_info in stream_infos:
                 video_encode_id = xpath_text(stream_info, './video_encode_id')
                 if video_encode_id in video_encode_ids:
                     continue
@@ -485,7 +513,6 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 metadata = stream_info.find('./metadata')
                 format_info = {
                     'format': video_format,
-                'format_id': video_format,
                     'height': int_or_none(xpath_text(metadata, './height')),
                     'width': int_or_none(xpath_text(metadata, './width')),
                 }
@@ -498,23 +525,24 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                         path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
                     if self._is_valid_url(direct_video_url, video_id, video_format):
                         format_info.update({
+                            'format_id': 'http-' + video_format,
                             'url': direct_video_url,
                         })
                         formats.append(format_info)
                         continue
 
                 format_info.update({
+                    'format_id': 'rtmp-' + video_format,
                     'url': video_url,
                     'play_path': video_file,
                     'ext': 'flv',
                 })
                 formats.append(format_info)
-        self._sort_formats(formats)
+        self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
 
-        metadata = self._download_xml(
+        metadata = self._call_rpc_api(
-            'http://www.crunchyroll.com/xml', video_id,
+            'VideoPlayer_GetMediaMetadata', video_id,
-            note='Downloading media info', query={
+            note='Downloading media info', data={
-                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
                 'media_id': video_id,
             })