diff --git a/settings.py b/settings.py index 4c08d8d13..2c9706983 100644 --- a/settings.py +++ b/settings.py @@ -224,6 +224,24 @@ 'category': 'interface', }), + ## mine + ('disable_dubbing', { + 'label': 'Disable dubbing', + 'type': bool, + 'default': True, + 'comment': '''Allows only original audio track''', + 'category': 'interface', + }), + ## mine + ('allowed_dubbing_languages', { + 'label': 'Allowed dubbing languages', + 'type': str, + 'default': 'English;Russian', + 'comment': '''Allows dubbing for listed languages''', + 'category': 'interface', + }), + + ('video_player', { 'type': int, 'default': 1, diff --git a/youtube/watch.py b/youtube/watch.py index 9e31202a0..9e8f8ad5e 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -104,65 +104,93 @@ def get_video_sources(info, target_resolution): audio_sources.sort(key=lambda source: source['audio_bitrate']) uni_sources.sort(key=lambda src: src['quality']) - webm_audios = [a for a in audio_sources if a['ext'] == 'webm'] - mp4_audios = [a for a in audio_sources if a['ext'] == 'mp4'] - - for quality_string, sources in video_only_sources.items(): - # choose an audio source to go with it - # 0.5 is semiarbitrary empirical constant to spread audio sources - # between 144p and 1080p. Use something better eventually. - quality, fps = map(int, quality_string.split('p')) - target_audio_bitrate = quality*fps/30*0.5 - pair_info = { - 'quality_string': quality_string, - 'quality': quality, - 'height': sources[0]['height'], - 'width': sources[0]['width'], - 'fps': fps, - 'videos': sources, - 'audios': [], - } - for audio_choices in (webm_audios, mp4_audios): - if not audio_choices: + ################################################################################## mine + # for multi tracks + res = {} + # group dicts by audio_track key + for item1 in audio_sources: + if item1['audio_track'] not in res: + res[item1['audio_track']] = [] + res[item1['audio_track']].append(item1) + + for k_lang, v_lang in res.items(): + k_lang_ = "" if k_lang == 'undefined' else " " + k_lang + + webm_audios = [a for a in v_lang if a['ext'] == 'webm'] + mp4_audios = [a for a in v_lang if a['ext'] == 'mp4'] + + for quality_string, sources in video_only_sources.items(): + # choose an audio source to go with it + # 0.5 is semiarbitrary empirical constant to spread audio sources + # between 144p and 1080p. Use something better eventually. + quality, fps = map(int, quality_string.split('p')) + target_audio_bitrate = quality*fps/30*0.5 + pair_info = { + 'quality_string': quality_string + k_lang_, # mine + 'quality': quality, + 'height': sources[0]['height'], + 'width': sources[0]['width'], + 'fps': fps, + 'videos': sources, + 'audios': [], + } + for audio_choices in (webm_audios, mp4_audios): + if not audio_choices: + continue + closest_audio_source = audio_choices[0] + best_err = target_audio_bitrate - audio_choices[0]['audio_bitrate'] + best_err = abs(best_err) + for audio_source in audio_choices[1:]: + err = abs(audio_source['audio_bitrate'] - target_audio_bitrate) + # once err gets worse we have passed the closest one + if err > best_err: + break + best_err = err + closest_audio_source = audio_source + pair_info['audios'].append(closest_audio_source) + + if not pair_info['audios']: continue - closest_audio_source = audio_choices[0] - best_err = target_audio_bitrate - audio_choices[0]['audio_bitrate'] - best_err = abs(best_err) - for audio_source in audio_choices[1:]: - err = abs(audio_source['audio_bitrate'] - target_audio_bitrate) - # once err gets worse we have passed the closest one - if err > best_err: - break - best_err = err - closest_audio_source = audio_source - pair_info['audios'].append(closest_audio_source) - - if not pair_info['audios']: - continue - def video_rank(src): - ''' Sort by settings preference. Use file size as tiebreaker ''' - setting_name = 'codec_rank_' + codec_name(src['vcodec']) - return (settings.current_settings_dict[setting_name], - src['file_size']) - pair_info['videos'].sort(key=video_rank) + def video_rank(src): + ''' Sort by settings preference. Use file size as tiebreaker ''' + setting_name = 'codec_rank_' + codec_name(src['vcodec']) + return (settings.current_settings_dict[setting_name], + src['file_size']) + pair_info['videos'].sort(key=video_rank) - pair_sources.append(pair_info) + pair_sources.append(pair_info) + pair_sources.sort(key=lambda src: src['quality_string']) pair_sources.sort(key=lambda src: src['quality']) + # if file_size is none for some videos url not work + # if clen is none for some videos url works + uni_sources1 = [] + for u in uni_sources: + if u['file_size'] != None: uni_sources1.append(u) + elif u['file_size'] == None and u['clen'] == None: uni_sources1.append(u) + uni_sources = uni_sources1[:] + uni_idx = 0 if uni_sources else None for i, source in enumerate(uni_sources): if source['quality'] > target_resolution: break uni_idx = i + # need original track as default pair_idx = 0 if pair_sources else None for i, pair_info in enumerate(pair_sources): - if pair_info['quality'] > target_resolution: - break + if 'undefined' not in pair_info['audios'][0]['audio_track']: continue + if pair_info['quality'] > target_resolution: break pair_idx = i + # if pair_idx is bigger than pair_sources list size + # try: _ = pair_sources[pair_idx] + # except IndexError: pair_idx = pair_idx - 1 + + ################################################################################## + return { 'uni_sources': uni_sources, 'uni_idx': uni_idx, @@ -366,7 +394,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index), - gevent.spawn(fetch_player_response, 'android_vr', video_id) + gevent.spawn(fetch_player_response, 'ios', video_id) ) gevent.joinall(tasks) util.check_gevent_exceptions(*tasks) diff --git a/youtube/yt_data_extract/watch_extraction.py b/youtube/yt_data_extract/watch_extraction.py index 6a13a2700..e9beac359 100644 --- a/youtube/yt_data_extract/watch_extraction.py +++ b/youtube/yt_data_extract/watch_extraction.py @@ -9,6 +9,7 @@ import urllib.parse import traceback import re +import settings # from https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/youtube.py _formats = { @@ -476,8 +477,29 @@ def _extract_formats(info, player_response): # Example: https://www.youtube.com/watch?v=gF9kkB0UWYQ # Only get the original language for now so a foreign # translation will not be picked just because it comes first - if deep_get(yt_fmt, 'audioTrack', 'audioIsDefault') is False: - continue + # if deep_get(yt_fmt, 'audioTrack', 'audioIsDefault') is False: + # continue + + # mine + # if deep_get(yt_fmt, 'audioTrack', 'audioIsDefault') is True and deep_get(yt_fmt, 'audioTrack', 'isAutoDubbed') is True: + # continue + + # mine + # in case no contentLength provided + # if itag in [18]: + # clen = int(str(deep_get(urllib.parse.parse_qs(urllib.parse.urlparse(yt_fmt.get('url')).query), 'clen', 0)) + '0') + # yt_fmt['contentLength'] = clen + # if itag not in [140, 134, 18]: + # continue + + # mine + # what if user upload audio track without using auto dubbing??? + if settings.disable_dubbing and deep_get(yt_fmt, 'audioTrack', 'isAutoDubbed') is True: continue + allowed = [a.strip().lower() for a in settings.allowed_dubbing_languages.split(';') if a not in ['', ' ', ',', ';', ':']] + ['undefined'] + lng = deep_get(yt_fmt, 'audioTrack', 'displayName', default='undefined') + if 'original' in lng.lower(): lng = 'undefined' + if len(allowed) == 1: pass # in case allowed list is empty + elif lng.lower().split(' ')[0] not in allowed: continue fmt = {} fmt['itag'] = itag @@ -494,6 +516,7 @@ def _extract_formats(info, player_response): fmt['fps'] = yt_fmt.get('fps') fmt['init_range'] = yt_fmt.get('initRange') fmt['index_range'] = yt_fmt.get('indexRange') + fmt['audio_track'] = lng # mine for key in ('init_range', 'index_range'): if fmt[key]: fmt[key]['start'] = int(fmt[key]['start']) @@ -522,6 +545,10 @@ def _extract_formats(info, player_response): extract_int(yt_fmt.get('qualityLabel'), whole_word=False) ) + # mine + # need clen for uni_sources + if itag in [18]: fmt['clen'] = urllib.parse.parse_qs(urllib.parse.urlparse(fmt['url']).query).get('clen') + info['formats'].append(fmt) # get ip address