github-com-ytdl-org/youtube-dl
1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl.git synced 2024-01-07 17:16:08 +00:00
Code Issues Releases Wiki Activity

Move duplicate check to generic.py

Browse source
This commit is contained in:
anovicecodemonkey 2014-06-01 01:16:35 +09:30
parent 610134730a
commit 37e3cbe22e
2 changed files with 24 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
youtube_dl/extractor/common.py
View file

@ -343,16 +343,6 @@ class InfoExtractor(object):
@staticmethod @staticmethod
def playlist_result(entries, playlist_id=None, playlist_title=None): def playlist_result(entries, playlist_id=None, playlist_title=None):
"""Returns a playlist""" """Returns a playlist"""
# Ensure we don't have any duplicates in the playlist
seen = set()
new_list = []
for url in entries:
theurl = tuple(url.items())
if theurl not in seen:
seen.add(theurl)
new_list.append(url)
entries = new_list
video_info = {'_type': 'playlist', video_info = {'_type': 'playlist',
'entries': entries} 'entries': entries}
if playlist_id: if playlist_id:

24
youtube_dl/extractor/generic.py
View file

@ -494,6 +494,14 @@ class GenericIE(InfoExtractor):
if matches: if matches:
urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
for tuppl in matches] for tuppl in matches]
# First, ensure we have a duplicate free list of entries
seen = set()
new_list = []
theurl = tuple(url.items())
if theurl not in seen:
seen.add(theurl)
new_list.append(url)
urlrs = new_list
return self.playlist_result( return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title) urlrs, playlist_id=video_id, playlist_title=video_title)
@ -503,6 +511,14 @@ class GenericIE(InfoExtractor):
if matches: if matches:
urlrs = [self.url_result(unescapeHTML(tuppl[1])) urlrs = [self.url_result(unescapeHTML(tuppl[1]))
for tuppl in matches] for tuppl in matches]
# First, ensure we have a duplicate free list of entries
seen = set()
new_list = []
theurl = tuple(url.items())
if theurl not in seen:
seen.add(theurl)
new_list.append(url)
urlrs = new_list
return self.playlist_result( return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title) urlrs, playlist_id=video_id, playlist_title=video_title)
@ -615,6 +631,14 @@ class GenericIE(InfoExtractor):
if matches: if matches:
urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie') urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
for eurl in matches] for eurl in matches]
# First, ensure we have a duplicate free list of entries
seen = set()
new_list = []
theurl = tuple(url.items())
if theurl not in seen:
seen.add(theurl)
new_list.append(url)
urlrs = new_list
return self.playlist_result( return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title) urlrs, playlist_id=video_id, playlist_title=video_title)