From 8624358ddece714c9db9c3499a00821c3e4f2743 Mon Sep 17 00:00:00 2001 From: Gabe Kangas Date: Sun, 19 Jun 2022 21:20:24 -0700 Subject: [PATCH] Add latency compensator to player. Closes #1931 --- web/components/video/OwncastPlayer.tsx | 75 +++++++++++- .../components/video}/latencyCompensator.js | 109 +++++++----------- web/components/video/settings-menu.ts | 53 +++++---- 3 files changed, 145 insertions(+), 92 deletions(-) rename {webroot/js/components => web/components/video}/latencyCompensator.js (90%) diff --git a/web/components/video/OwncastPlayer.tsx b/web/components/video/OwncastPlayer.tsx index 357eb306e..83218ae14 100644 --- a/web/components/video/OwncastPlayer.tsx +++ b/web/components/video/OwncastPlayer.tsx @@ -8,12 +8,17 @@ import { getLocalStorage, setLocalStorage } from '../../utils/localStorage'; import { isVideoPlayingAtom, clockSkewAtom } from '../stores/ClientConfigStore'; import PlaybackMetrics from './metrics/playback'; import createVideoSettingsMenuButton from './settings-menu'; +import LatencyCompensator from './latencyCompensator'; const VIDEO_CONFIG_URL = '/api/video/variants'; const PLAYER_VOLUME = 'owncast_volume'; +const LATENCY_COMPENSATION_ENABLED = 'latencyCompensatorEnabled'; const ping = new ViewerPing(); let playbackMetrics = null; +let latencyCompensator = null; +let latencyCompensatorEnabled = false; + interface Props { source: string; online: boolean; @@ -73,6 +78,50 @@ export default function OwncastPlayer(props: Props) { } }; + const setLatencyCompensatorItemTitle = title => { + const item = document.querySelector('.latency-toggle-item > .vjs-menu-item-text'); + if (!item) { + return; + } + + item.innerHTML = title; + }; + + const startLatencyCompensator = () => { + if (latencyCompensator) { + latencyCompensator.stop(); + } + + latencyCompensatorEnabled = true; + + latencyCompensator = new LatencyCompensator(playerRef.current); + latencyCompensator.setClockSkew(clockSkew); + latencyCompensator.enable(); + setLocalStorage(LATENCY_COMPENSATION_ENABLED, true); + + setLatencyCompensatorItemTitle('disable minimized latency'); + }; + + const stopLatencyCompensator = () => { + if (latencyCompensator) { + latencyCompensator.disable(); + } + latencyCompensator = null; + latencyCompensatorEnabled = false; + setLocalStorage(LATENCY_COMPENSATION_ENABLED, false); + setLatencyCompensatorItemTitle( + 'enable minimized latency (experimental)', + ); + }; + + const toggleLatencyCompensator = () => { + if (latencyCompensatorEnabled) { + stopLatencyCompensator(); + } else { + startLatencyCompensator(); + } + }; + // Register keyboard shortcut for the space bar to toggle playback useHotkeys('space', togglePlayback, { enableOnContentEditable: false, @@ -133,6 +182,23 @@ export default function OwncastPlayer(props: Props) { playerRef.current = player; setSavedVolume(); + const setupLatencyCompensator = () => { + const tech = player.tech({ IWillNotUseThisInPlugins: true }); + + // VHS is required. + if (!tech || !tech.vhs) { + return; + } + + const latencyCompensatorEnabledSaved = getLocalStorage(LATENCY_COMPENSATION_ENABLED); + + if (latencyCompensatorEnabledSaved === 'true' && tech && tech.vhs) { + startLatencyCompensator(); + } else { + stopLatencyCompensator(); + } + }; + // You can handle player events here, for example: player.on('waiting', () => { player.log('player is waiting'); @@ -170,14 +236,19 @@ export default function OwncastPlayer(props: Props) { const createSettings = async () => { const videoQualities = await getVideoSettings(); - const menuButton = createVideoSettingsMenuButton(player, videojs, videoQualities); + const menuButton = createVideoSettingsMenuButton( + player, + videojs, + videoQualities, + toggleLatencyCompensator, + ); player.controlBar.addChild( menuButton, {}, // eslint-disable-next-line no-underscore-dangle player.controlBar.children_.length - 2, ); - // this.latencyCompensatorToggleButton = lowLatencyItem; + setupLatencyCompensator(); }; createSettings(); diff --git a/webroot/js/components/latencyCompensator.js b/web/components/video/latencyCompensator.js similarity index 90% rename from webroot/js/components/latencyCompensator.js rename to web/components/video/latencyCompensator.js index bd7cdcf47..9f446e2f5 100644 --- a/webroot/js/components/latencyCompensator.js +++ b/web/components/video/latencyCompensator.js @@ -44,6 +44,28 @@ const MAX_JUMP_FREQUENCY = 20 * 1000; // How often we'll allow a time jump. const MAX_ACTIONABLE_LATENCY = 80 * 1000; // If latency is seen to be greater than this then something is wrong. const STARTUP_WAIT_TIME = 10 * 1000; // The amount of time after we start up that we'll allow monitoring to occur. +function getCurrentlyPlayingSegment(tech) { + const targetMedia = tech.vhs.playlists.media(); + const snapshotTime = tech.currentTime(); + let segment; + + // Iterate trough available segments and get first within which snapshot_time is + // eslint-disable-next-line no-plusplus + for (let i = 0, l = targetMedia.segments.length; i < l; i++) { + // Note: segment.end may be undefined or is not properly set + if (snapshotTime < targetMedia.segments[i].end) { + segment = targetMedia.segments[i]; + break; + } + } + + if (!segment) { + [segment] = targetMedia.segments; + } + + return segment; +} + class LatencyCompensator { constructor(player) { this.player = player; @@ -143,10 +165,12 @@ class LatencyCompensator { return; } - tech.vhs.stats.buffered.forEach((buffer) => { + tech.vhs.stats.buffered.forEach(buffer => { totalBuffered += buffer.end - buffer.start; }); - } catch (e) {} + } catch (e) { + console.error(e); + } // Determine how much of the current playlist's bandwidth requirements // we're utilizing. If it's too high then we can't afford to push @@ -164,10 +188,7 @@ class LatencyCompensator { // If we're downloading media fast enough or we feel like we have a large // enough buffer then continue. Otherwise timeout for a bit. - if ( - bandwidthRatio < REQUIRED_BANDWIDTH_RATIO && - totalBuffered < segment.duration * 6 - ) { + if (bandwidthRatio < REQUIRED_BANDWIDTH_RATIO && totalBuffered < segment.duration * 6) { this.timeout(); return; } @@ -175,30 +196,24 @@ class LatencyCompensator { // How far away from live edge do we stop the compensator. const computedMinLatencyThreshold = Math.max( MIN_LATENCY, - segment.duration * 1000 * LOWEST_LATENCY_SEGMENT_LENGTH_MULTIPLIER + segment.duration * 1000 * LOWEST_LATENCY_SEGMENT_LENGTH_MULTIPLIER, ); // Create an array of all the buffering events in the past along with // the computed min latency above. - const targetLatencies = this.bufferedAtLatency.concat([ - computedMinLatencyThreshold, - ]); + const targetLatencies = this.bufferedAtLatency.concat([computedMinLatencyThreshold]); // Determine if we need to reduce the minimum latency we computed // above based on buffering events that have taken place in the past by // creating an array of all the buffering events and the above computed // minimum latency target and averaging all those values. const minLatencyThreshold = - targetLatencies.reduce((sum, current) => sum + current, 0) / - targetLatencies.length; + targetLatencies.reduce((sum, current) => sum + current, 0) / targetLatencies.length; // How far away from live edge do we start the compensator. let maxLatencyThreshold = Math.max( minLatencyThreshold * 1.4, - Math.min( - segment.duration * 1000 * HIGHEST_LATENCY_SEGMENT_LENGTH_MULTIPLIER, - MAX_LATENCY - ) + Math.min(segment.duration * 1000 * HIGHEST_LATENCY_SEGMENT_LENGTH_MULTIPLIER, MAX_LATENCY), ); // If this newly adjusted minimum latency ends up being greater than @@ -228,10 +243,7 @@ class LatencyCompensator { if (latency > maxLatencyThreshold) { // If the current latency exceeds the max jump amount then // force jump into the future, skipping all the video in between. - if ( - this.shouldJumpToLive() && - latency > maxLatencyThreshold + MAX_JUMP_LATENCY - ) { + if (this.shouldJumpToLive() && latency > maxLatencyThreshold + MAX_JUMP_LATENCY) { const jumpAmount = latency / 1000 - segment.duration * 3; const seekPosition = this.player.currentTime() + jumpAmount; console.info( @@ -242,17 +254,13 @@ class LatencyCompensator { 'to live from ', this.player.currentTime(), ' to ', - seekPosition + seekPosition, ); // Verify we have the seek position buffered before jumping. const availableBufferedTimeEnd = tech.vhs.stats.buffered[0].end; const availableBufferedTimeStart = tech.vhs.stats.buffered[0].start; - if ( - seekPosition > - availableBufferedTimeStart < - availableBufferedTimeEnd - ) { + if (seekPosition > availableBufferedTimeStart < availableBufferedTimeEnd) { this.jump(seekPosition); return; @@ -260,13 +268,10 @@ class LatencyCompensator { } // Using our bandwidth ratio determine a wide guess at how fast we can play. - var proposedPlaybackRate = bandwidthRatio * 0.33; + let proposedPlaybackRate = bandwidthRatio * 0.33; // But limit the playback rate to a max value. - proposedPlaybackRate = Math.max( - Math.min(proposedPlaybackRate, MAX_SPEEDUP_RATE), - 1.0 - ); + proposedPlaybackRate = Math.max(Math.min(proposedPlaybackRate, MAX_SPEEDUP_RATE), 1.0); if (proposedPlaybackRate > this.playbackRate + MAX_SPEEDUP_RAMP) { // If this proposed speed is substantially faster than the current rate, @@ -275,8 +280,7 @@ class LatencyCompensator { } // Limit to 3 decimal places of precision. - proposedPlaybackRate = - Math.round(proposedPlaybackRate * Math.pow(10, 3)) / Math.pow(10, 3); + proposedPlaybackRate = Math.round(proposedPlaybackRate * 10 ** 3) / 10 ** 3; // Otherwise start the playback rate adjustment. this.start(proposedPlaybackRate); @@ -300,7 +304,7 @@ class LatencyCompensator { 'skew: ', this.clockSkewMs, 'rebuffer events: ', - this.bufferingCounter + this.bufferingCounter, ); } catch (err) { // console.error(err); @@ -325,12 +329,7 @@ class LatencyCompensator { this.lastJumpOccurred = new Date(); - console.info( - 'current time', - this.player.currentTime(), - 'seeking to', - seekPosition - ); + console.info('current time', this.player.currentTime(), 'seeking to', seekPosition); this.player.currentTime(seekPosition); setTimeout(() => { @@ -435,7 +434,7 @@ class LatencyCompensator { this.disable(); } - handleError(e) { + handleError() { if (!this.enabled) { return; } @@ -444,7 +443,7 @@ class LatencyCompensator { } countBufferingEvent() { - this.bufferingCounter++; + this.bufferingCounter += 1; if (this.bufferingCounter > REBUFFER_EVENT_LIMIT) { this.disable(); @@ -457,13 +456,13 @@ class LatencyCompensator { 'latency compensation timeout due to buffering:', this.bufferingCounter, 'buffering events of', - REBUFFER_EVENT_LIMIT + REBUFFER_EVENT_LIMIT, ); // Allow us to forget about old buffering events if enough time goes by. setTimeout(() => { if (this.bufferingCounter > 0) { - this.bufferingCounter--; + this.bufferingCounter -= 1; } }, BUFFERING_AMNESTY_DURATION); } @@ -487,26 +486,4 @@ class LatencyCompensator { } } -function getCurrentlyPlayingSegment(tech) { - var target_media = tech.vhs.playlists.media(); - var snapshot_time = tech.currentTime(); - - var segment; - - // Iterate trough available segments and get first within which snapshot_time is - for (var i = 0, l = target_media.segments.length; i < l; i++) { - // Note: segment.end may be undefined or is not properly set - if (snapshot_time < target_media.segments[i].end) { - segment = target_media.segments[i]; - break; - } - } - - if (!segment) { - segment = target_media.segments[0]; - } - - return segment; -} - export default LatencyCompensator; diff --git a/web/components/video/settings-menu.ts b/web/components/video/settings-menu.ts index 90ec80b9f..43d6b3c58 100644 --- a/web/components/video/settings-menu.ts +++ b/web/components/video/settings-menu.ts @@ -1,32 +1,37 @@ -export default function createVideoSettingsMenuButton(player, videojs, qualities): any { - // const VjsMenuItem = videojs.getComponent('MenuItem'); +export default function createVideoSettingsMenuButton( + player, + videojs, + qualities, + latencyItemPressed, +): any { + const VjsMenuItem = videojs.getComponent('MenuItem'); const MenuItem = videojs.getComponent('MenuItem'); const MenuButtonClass = videojs.getComponent('MenuButton'); - // class MenuSeparator extends VjsMenuItem { - // // eslint-disable-next-line no-useless-constructor - // constructor(p: any, options: { selectable: boolean }) { - // super(p, options); - // } + class MenuSeparator extends VjsMenuItem { + // eslint-disable-next-line no-useless-constructor + constructor(p: any, options: { selectable: boolean }) { + super(p, options); + } - // createEl(tag = 'button', props = {}, attributes = {}) { - // const el = super.createEl(tag, props, attributes); - // el.innerHTML = '
'; - // return el; - // } - // } + createEl(tag = 'button', props = {}, attributes = {}) { + const el = super.createEl(tag, props, attributes); + el.innerHTML = '
'; + return el; + } + } const lowLatencyItem = new MenuItem(player, { selectable: true, }); lowLatencyItem.setAttribute('class', 'latency-toggle-item'); lowLatencyItem.on('click', () => { - this.toggleLatencyCompensator(); + latencyItemPressed(); }); - // const separator = new MenuSeparator(player, { - // selectable: false, - // }); + const separator = new MenuSeparator(player, { + selectable: false, + }); const MenuButton = videojs.extend(MenuButtonClass, { // The `init()` method will also work for constructor logic here, but it is @@ -75,16 +80,16 @@ export default function createVideoSettingsMenuButton(player, videojs, qualities defaultAutoItem.selected(false); }); - const supportsLatencyCompensator = false; // !!tech && !!tech.vhs; + const supportsLatencyCompensator = !!tech && !!tech.vhs; // Only show the quality selector if there is more than one option. - // if (qualities.length < 2 && supportsLatencyCompensator) { - // return [lowLatencyItem]; - // } + if (qualities.length < 2 && supportsLatencyCompensator) { + return [lowLatencyItem]; + } - // if (qualities.length > 1 && supportsLatencyCompensator) { - // return [defaultAutoItem, ...items, separator, lowLatencyItem]; - // } + if (qualities.length > 1 && supportsLatencyCompensator) { + return [defaultAutoItem, ...items, separator, lowLatencyItem]; + } if (!supportsLatencyCompensator && qualities.length === 1) { return []; }