From a7e999beeca17909bb0088d796c3181b4f35144e Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Tue, 7 Sep 2021 20:59:20 +0000 Subject: [PATCH] [pbs] Fix subtitle extraction (#813) Original PR: https://github.com/ytdl-org/youtube-dl/pull/24430, https://github.com/ytdl-org/youtube-dl/pull/17434 Closes: #836, https://github.com/ytdl-org/youtube-dl/issues/18796, https://github.com/ytdl-org/youtube-dl/issues/17273 Authored-by: coletdjnz, gesa, raphaeldore --- test/test_subtitles.py | 38 ++++++++++++++++++++++++++++++++++++++ yt_dlp/extractor/pbs.py | 31 ++++++++++--------------------- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 0c5b49ee8..9b39dbd39 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -19,6 +19,7 @@ from yt_dlp.extractor import ( CeskaTelevizeIE, LyndaIE, NPOIE, + PBSIE, ComedyCentralIE, NRKTVIE, RaiPlayIE, @@ -372,5 +373,42 @@ class TestDemocracynowSubtitles(BaseTestSubtitles): self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c') +@is_download_test +class TestPBSSubtitles(BaseTestSubtitles): + url = 'https://www.pbs.org/video/how-fantasy-reflects-our-world-picecq/' + IE = PBSIE + + def test_allsubtitles(self): + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(set(subtitles.keys()), set(['en'])) + + def test_subtitles_dfxp_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'dfxp' + subtitles = self.getSubtitles() + self.assertIn(md5(subtitles['en']), ['643b034254cdc3768ff1e750b6b5873b']) + + def test_subtitles_vtt_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'vtt' + subtitles = self.getSubtitles() + self.assertIn( + md5(subtitles['en']), ['937a05711555b165d4c55a9667017045', 'f49ea998d6824d94959c8152a368ff73']) + + def test_subtitles_srt_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'srt' + subtitles = self.getSubtitles() + self.assertIn(md5(subtitles['en']), ['2082c21b43759d9bf172931b2f2ca371']) + + def test_subtitles_sami_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'sami' + subtitles = self.getSubtitles() + self.assertIn(md5(subtitles['en']), ['4256b16ac7da6a6780fafd04294e85cd']) + + if __name__ == '__main__': unittest.main() diff --git a/yt_dlp/extractor/pbs.py b/yt_dlp/extractor/pbs.py index d68855d62..0eabf9bee 100644 --- a/yt_dlp/extractor/pbs.py +++ b/yt_dlp/extractor/pbs.py @@ -600,6 +600,7 @@ class PBSIE(InfoExtractor): formats = [] http_url = None + hls_subs = {} for num, redirect in enumerate(redirects): redirect_id = redirect.get('eeid') @@ -622,8 +623,9 @@ class PBSIE(InfoExtractor): continue if determine_ext(format_url) == 'm3u8': - formats.extend(self._extract_m3u8_formats( - format_url, display_id, 'mp4', m3u8_id='hls', fatal=False)) + hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles( + format_url, display_id, 'mp4', m3u8_id='hls', fatal=False) + formats.extend(hls_formats) else: formats.append({ 'url': format_url, @@ -666,25 +668,12 @@ class PBSIE(InfoExtractor): age_limit = US_RATINGS.get(rating_str) subtitles = {} - closed_captions_url = info.get('closed_captions_url') - if closed_captions_url: - subtitles['en'] = [{ - 'ext': 'ttml', - 'url': closed_captions_url, - }] - mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url) - if mobj: - ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1) - ttml_caption_id = int(ttml_caption_id) - subtitles['en'].extend([{ - 'url': closed_captions_url.replace( - ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)), - 'ext': 'srt', - }, { - 'url': closed_captions_url.replace( - ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)), - 'ext': 'vtt', - }]) + captions = info.get('cc') or {} + for caption_url in captions.values(): + subtitles.setdefault('en', []).append({ + 'url': caption_url + }) + subtitles = self._merge_subtitles(subtitles, hls_subs) # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc) # Try turning it to 'program - title' naming scheme if possible