Feature/forced subtitles (#4545)

* Filter forced subtitles from controlbar menu

* Select forced subtitle if available and no other subtitles are selected

* Fix index selection in controlbar

* Fix a bug in the initial track selection

* React to audio track changes and enable corresponding forced subtitles. If different forced subtitles are active at this point and there are no matching forced subtitles for the new track we do not show any subtitles.

* Fix unit tests

* Add functional test to check automatic forced-subtitle selection

* Do not select wrong forced subtitles at startup

* Fix accessing non defined property

src/core/events/CoreEvents.js

@@ -62,6 +62,7 @@ class CoreEvents extends EventsBase {
         this.MANIFEST_UPDATED = 'manifestUpdated';
         this.MEDIA_FRAGMENT_LOADED = 'mediaFragmentLoaded';
         this.MEDIA_FRAGMENT_NEEDED = 'mediaFragmentNeeded';
+        this.MEDIAINFO_UPDATED = 'mediaInfoUpdated';
         this.QUOTA_EXCEEDED = 'quotaExceeded';
         this.SEGMENT_LOCATION_BLACKLIST_ADD = 'segmentLocationBlacklistAdd';
         this.SEGMENT_LOCATION_BLACKLIST_CHANGED = 'segmentLocationBlacklistChanged';

src/dash/constants/DashConstants.js

@@ -51,6 +51,7 @@ export default {
     BITSTREAM_SWITCHING: 'BitstreamSwitching',
     BITSTREAM_SWITCHING_MINUS: 'bitstreamSwitching',
     BYTE_RANGE: 'byteRange',
+    CAPTION: 'caption',
     CENC_DEFAULT_KID: 'cenc:default_KID',
     CLIENT_DATA_REPORTING: 'ClientDataReporting',
     CLIENT_REQUIREMENT: 'clientRequirement',
@@ -86,6 +87,7 @@ export default {
     ESSENTIAL_PROPERTY: 'EssentialProperty',
     EVENT: 'Event',
     EVENT_STREAM: 'EventStream',
+    FORCED_SUBTITLE: 'forced-subtitle',
     FRAMERATE: 'frameRate',
     FRAME_PACKING: 'FramePacking',
     GROUP_LABEL: 'GroupLabel',
@@ -171,6 +173,7 @@ export default {
     START_WITH_SAP: 'startWithSAP',
     STATIC: 'static',
     SUBSET: 'Subset',
+    SUBTITLE: 'subtitle',
     SUB_REPRESENTATION: 'SubRepresentation',
     SUB_SEGMENT_ALIGNMENT: 'subsegmentAlignment',
     SUGGESTED_PRESENTATION_DELAY: 'suggestedPresentationDelay',

src/streaming/StreamProcessor.js

@@ -659,7 +659,9 @@ function StreamProcessor(config) {
                 selectedValues = _handleMpdUpdate(selectionInput);
             }
 
-            currentMediaInfo = selectedValues.currentMediaInfo;
+            _setCurrentMediaInfo(selectedValues.currentMediaInfo);
+
+            eventBus.trigger()
 
             // Update Representation Controller with the new data. Note we do not filter any Representations here as the filter values might change over time.
             const voRepresentations = abrController.getPossibleVoRepresentations(currentMediaInfo, false);
@@ -676,6 +678,15 @@ function StreamProcessor(config) {
         })
     }
 
+    function _setCurrentMediaInfo(value) {
+        currentMediaInfo = value;
+        eventBus.trigger(Events.MEDIAINFO_UPDATED, {
+            mediaType: type,
+            streamId: streamInfo.id,
+            currentMediaInfo
+        });
+    }
+
     function _handleAdaptationSetQualitySwitch(selectionInput) {
         return {
             selectedRepresentation: selectionInput.newRepresentation,
@@ -756,7 +767,7 @@ function StreamProcessor(config) {
         scheduleController.setSwitchTrack(true);
 
         const newMediaInfo = newRepresentation.mediaInfo;
-        currentMediaInfo = newMediaInfo;
+        _setCurrentMediaInfo(newMediaInfo);
 
         selectMediaInfo({ newMediaInfo, newRepresentation })
             .then(() => {

src/streaming/constants/Constants.js

@@ -330,5 +330,6 @@ export default {
      *  @static
      */
     ID3_SCHEME_ID_URI: 'https://aomedia.org/emsg/ID3',
-    COMMON_ACCESS_TOKEN_HEADER: 'common-access-token'
+    COMMON_ACCESS_TOKEN_HEADER: 'common-access-token',
+    DASH_ROLE_SCHEME_ID : 'urn:mpeg:dash:role:2011',
 }

src/streaming/controllers/MediaController.js

@@ -36,6 +36,7 @@ import Debug from '../../core/Debug.js';
 import {bcp47Normalize} from 'bcp-47-normalize';
 import {extendedFilter} from 'bcp-47-match';
 import MediaPlayerEvents from '../MediaPlayerEvents.js';
+import DashConstants from '../../dash/constants/DashConstants.js';
 
 function MediaController() {
 
@@ -121,7 +122,7 @@ function MediaController() {
         const possibleTracks = getTracksFor(type, streamInfo.id);
         let filteredTracks = [];
 
-        if (!settings) {
+        if (!settings || Object.keys(settings).length === 0) {
             settings = domStorage.getSavedMediaSettings(type);
             if (settings) {
                 // If the settings are defined locally, do not take codec into account or it'll be too strict.
@@ -603,7 +604,7 @@ function MediaController() {
 
     function selectInitialTrack(type, mediaInfos) {
         if (type === Constants.TEXT) {
-            return mediaInfos[0];
+            return _handleInitialTextTrackSelection(mediaInfos);
         }
 
         let tmpArr;
@@ -650,6 +651,23 @@ function MediaController() {
         return tmpArr.length > 0 ? tmpArr[0] : mediaInfos[0];
     }
 
+    function _handleInitialTextTrackSelection(mediaInfos) {
+        const filteredMediaInfos = mediaInfos.filter((mediaInfo) => {
+            if (mediaInfo && mediaInfo.roles && mediaInfo.roles.length > 0) {
+                return mediaInfo.roles.every((role) => {
+                    return role.schemeIdUri !== Constants.DASH_ROLE_SCHEME_ID || role.value !== DashConstants.FORCED_SUBTITLE
+                })
+            }
+            return true
+        })
+
+        if (filteredMediaInfos.length > 0) {
+            return filteredMediaInfos[0];
+        }
+
+        return null
+    }
+
     /**
      * @param {MediaInfo[]} mediaInfos
      * @return {MediaInfo[]}

src/streaming/text/TextController.js

@@ -41,6 +41,7 @@ import Events from '../../core/events/Events.js';
 import MediaPlayerEvents from '../../streaming/MediaPlayerEvents.js';
 import {checkParameterType} from '../utils/SupervisorTools.js';
 import DVBFonts from './DVBFonts.js';
+import DashConstants from '../../dash/constants/DashConstants.js';
 
 function TextController(config) {
 
@@ -91,6 +92,7 @@ function TextController(config) {
         eventBus.on(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance);
         eventBus.on(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance);
         eventBus.on(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance);
+        eventBus.on(Events.MEDIAINFO_UPDATED, _onMediaInfoUpdated, instance);
         if (settings.get().streaming.text.webvtt.customRenderingEnabled) {
             eventBus.on(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance);
             eventBus.on(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance);
@@ -290,6 +292,32 @@ function TextController(config) {
         }
     }
 
+    function _onMediaInfoUpdated(e) {
+        try {
+            if (!e || !e.mediaType || e.mediaType !== Constants.AUDIO || !e.currentMediaInfo) {
+                return
+            }
+
+            const currentTextTrackInfo = textTracks[e.streamId].getCurrentTextTrackInfo();
+            let suitableForcedSubtitleIndex = NaN;
+            if (allTracksAreDisabled) {
+                suitableForcedSubtitleIndex = _getSuitableForceSubtitleTrackIndex(e.streamId);
+            } else if (_isForcedSubtitleTrack(currentTextTrackInfo) && e.currentMediaInfo.lang && e.currentMediaInfo.lang !== currentTextTrackInfo.lang) {
+                suitableForcedSubtitleIndex = _getSuitableForceSubtitleTrackIndex(e.streamId);
+                if (isNaN(suitableForcedSubtitleIndex)) {
+                    suitableForcedSubtitleIndex = -1;
+                }
+            }
+
+            if (!isNaN(suitableForcedSubtitleIndex)) {
+                setTextTrack(e.streamId, suitableForcedSubtitleIndex);
+            }
+
+        } catch (e) {
+            logger.error(e);
+        }
+    }
+
     function enableText(streamId, enable) {
         checkParameterType(enable, 'boolean');
         if (isTextEnabled() !== enable) {
@@ -346,7 +374,7 @@ function TextController(config) {
 
         textTracks[streamId].disableManualTracks();
 
-        let currentTrackInfo = textTracks[streamId].getCurrentTrackInfo();
+        let currentTrackInfo = textTracks[streamId].getCurrentTextTrackInfo();
         let currentNativeTrackInfo = (currentTrackInfo) ? videoModel.getTextTrack(currentTrackInfo.kind, currentTrackInfo.id, currentTrackInfo.lang, currentTrackInfo.isTTML, currentTrackInfo.isEmbedded) : null;
 
         // Don't change disabled tracks - dvb font download for essential property failed or not complete
@@ -356,7 +384,7 @@ function TextController(config) {
 
         textTracks[streamId].setCurrentTrackIdx(idx);
 
-        currentTrackInfo = textTracks[streamId].getCurrentTrackInfo();
+        currentTrackInfo = textTracks[streamId].getCurrentTextTrackInfo();
 
         const dispatchForManualRendering = settings.get().streaming.text.dispatchForManualRendering;
 
@@ -368,6 +396,12 @@ function TextController(config) {
             _setFragmentedTextTrack(streamId, currentTrackInfo, oldTrackIdx);
         } else if (currentTrackInfo && !currentTrackInfo.isFragmented) {
             _setNonFragmentedTextTrack(streamId, currentTrackInfo);
+        } else if (!currentTrackInfo && allTracksAreDisabled) {
+            const forcedSubtitleTrackIndex = _getSuitableForceSubtitleTrackIndex(streamId)
+            if (!isNaN(forcedSubtitleTrackIndex)) {
+                setTextTrack(streamId, forcedSubtitleTrackIndex);
+            }
+            return
         }
 
         mediaController.setTrack(currentTrackInfo);
@@ -412,6 +446,49 @@ function TextController(config) {
         });
     }
 
+    function _getSuitableForceSubtitleTrackIndex(streamId) {
+        const forcedSubtitleTracks = _getForcedSubtitleTracks(streamId);
+
+        if (!forcedSubtitleTracks || forcedSubtitleTracks.length <= 0) {
+            return NaN
+        }
+
+        const currentAudioTrack = mediaController.getCurrentTrackFor(Constants.AUDIO, streamId);
+        if (!currentAudioTrack) {
+            return NaN
+        }
+
+        const suitableTrack = forcedSubtitleTracks.find((track) => {
+            return currentAudioTrack.lang === track.lang
+        })
+
+        if (suitableTrack) {
+            return suitableTrack._indexToSelect
+        }
+
+        return NaN
+    }
+
+    function _getForcedSubtitleTracks(streamId) {
+        const textTrackInfos = textTracks[streamId].getTextTrackInfos();
+        return textTrackInfos.filter((textTrackInfo, index) => {
+            textTrackInfo._indexToSelect = index;
+            if (textTrackInfo && textTrackInfo.roles && textTrackInfo.roles.length > 0) {
+                return _isForcedSubtitleTrack(textTrackInfo);
+            }
+            return false
+        });
+    }
+
+    function _isForcedSubtitleTrack(textTrackInfo) {
+        if (!textTrackInfo || !textTrackInfo.roles || textTrackInfo.roles.length === 0) {
+            return false
+        }
+        return textTrackInfo.roles.some((role) => {
+            return role.schemeIdUri === Constants.DASH_ROLE_SCHEME_ID && role.value === DashConstants.FORCED_SUBTITLE
+        })
+    }
+
     function getCurrentTrackIdx(streamId) {
         return textTracks[streamId].getCurrentTrackIdx();
     }
@@ -446,6 +523,7 @@ function TextController(config) {
         eventBus.off(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance);
         eventBus.off(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance);
         eventBus.off(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance);
+        eventBus.off(Events.MEDIAINFO_UPDATED, _onMediaInfoUpdated, instance);
         if (settings.get().streaming.text.webvtt.customRenderingEnabled) {
             eventBus.off(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance);
             eventBus.off(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance)
@@ -458,20 +536,20 @@ function TextController(config) {
     }
 
     instance = {
+        addEmbeddedTrack,
+        addMediaInfosToBuffer,
+        createTracks,
         deactivateStream,
+        enableForcedTextStreaming,
+        enableText,
+        getAllTracksAreDisabled,
+        getCurrentTrackIdx,
+        getTextSourceBuffer,
         initialize,
         initializeForStream,
-        createTracks,
-        getTextSourceBuffer,
-        getAllTracksAreDisabled,
-        addEmbeddedTrack,
-        enableText,
         isTextEnabled,
+        reset,
         setTextTrack,
-        getCurrentTrackIdx,
-        enableForcedTextStreaming,
-        addMediaInfosToBuffer,
-        reset
     };
     setup();
     return instance;

src/streaming/text/TextSourceBuffer.js

@@ -43,6 +43,7 @@ import DashJSError from '../vo/DashJSError.js';
 import Errors from '../../core/errors/Errors.js';
 import {Cta608Parser} from '@svta/common-media-library/cta/608/Cta608Parser';
 import {extractCta608DataFromSample} from '@svta/common-media-library/cta/608/extractCta608DataFromSample';
+import DashConstants from '../../dash/constants/DashConstants.js';
 
 function TextSourceBuffer(config) {
     const errHandler = config.errHandler;
@@ -136,7 +137,7 @@ function TextSourceBuffer(config) {
         }
 
         for (let i = 0; i < mInfos.length; i++) {
-            _createTextTrackFromMediaInfo(mInfos[i]);
+            _createTextTrackInfoFromMediaInfo(mInfos[i]);
         }
 
     }
@@ -146,22 +147,26 @@ function TextSourceBuffer(config) {
      * @param {object} mediaInfo
      * @private
      */
-    function _createTextTrackFromMediaInfo(mediaInfo) {
+    function _createTextTrackInfoFromMediaInfo(mediaInfo) {
+
+        // We are mapping DASH specification strings to the ones of the HTML specification.
+        // See also https://html.spec.whatwg.org/multipage/media.html#text-track-kind
+        const trackKindMap = {};
+        trackKindMap[DashConstants.SUBTITLE] = 'subtitles'
+        trackKindMap[DashConstants.CAPTION] = 'captions'
+        trackKindMap[DashConstants.FORCED_SUBTITLE] = 'subtitles'
+
         const textTrackInfo = new TextTrackInfo();
-        const trackKindMap = { subtitle: 'subtitles', caption: 'captions' }; //Dash Spec has no "s" on end of KIND but HTML needs plural.
 
         for (let key in mediaInfo) {
             textTrackInfo[key] = mediaInfo[key];
         }
 
-        textTrackInfo.labels = mediaInfo.labels;
         textTrackInfo.defaultTrack = getIsDefault(mediaInfo);
-        textTrackInfo.isFragmented = mediaInfo.isFragmented;
-        textTrackInfo.isEmbedded = !!mediaInfo.isEmbedded;
         textTrackInfo.isTTML = _checkTtml(mediaInfo);
         textTrackInfo.kind = _getKind(mediaInfo, trackKindMap);
 
-        textTracks.addTextTrack(textTrackInfo);
+        textTracks.addTextTrackInfo(textTrackInfo);
     }
 
     function abort() {
@@ -250,7 +255,7 @@ function TextSourceBuffer(config) {
     function _getKind(mediaInfo, trackKindMap) {
         let kind = (mediaInfo.roles && mediaInfo.roles.length > 0) ? trackKindMap[mediaInfo.roles[0].value] : trackKindMap.caption;
 
-        kind = (kind === trackKindMap.caption || kind === trackKindMap.subtitle) ? kind : trackKindMap.caption;
+        kind = Object.values(trackKindMap).includes(kind) ? kind : trackKindMap.caption;
 
         return kind;
     }
@@ -331,7 +336,7 @@ function TextSourceBuffer(config) {
                 const offsetTime = manifest.ttmlTimeIsRelative ? sampleStart / timescale : 0;
                 const result = parser.parse(ccContent, offsetTime, (sampleStart / timescale), ((sampleStart + sample.duration) / timescale), images);
                 textTracks.addCaptions(currFragmentedTrackIdx, timestampOffset, result);
-            
+
             } catch (e) {
                 fragmentModel.removeExecutedRequestsBeforeTime();
                 this.remove();
@@ -542,7 +547,7 @@ function TextSourceBuffer(config) {
         for (let i = 0; i < samples.length; i++) {
             const sample = samples[i];
             const ccData = extractCta608DataFromSample(raw, sample.offset, sample.size);
-            
+
             let lastSampleTime = null;
             let idx = 0;
             for (let k = 0; k < 2; k++) {