From 36a4048c4f5081133fec8669a8e1bce544a0a344 Mon Sep 17 00:00:00 2001 From: Daniel Silhavy Date: Wed, 14 Aug 2024 08:56:38 +0200 Subject: [PATCH] Feature/forced subtitles (#4545) * Filter forced subtitles from controlbar menu * Select forced subtitle if available and no other subtitles are selected * Fix index selection in controlbar * Fix a bug in the initial track selection * React to audio track changes and enable corresponding forced subtitles. If different forced subtitles are active at this point and there are no matching forced subtitles for the new track we do not show any subtitles. * Fix unit tests * Add functional test to check automatic forced-subtitle selection * Do not select wrong forced subtitles at startup * Fix accessing non defined property --- src/core/events/CoreEvents.js | 1 + src/dash/constants/DashConstants.js | 3 + src/streaming/StreamProcessor.js | 15 ++- src/streaming/constants/Constants.js | 3 +- src/streaming/controllers/MediaController.js | 22 +++- src/streaming/text/TextController.js | 100 +++++++++++++-- src/streaming/text/TextSourceBuffer.js | 25 ++-- src/streaming/text/TextTracks.js | 104 ++++++++-------- test/functional/adapter/DashJsAdapter.js | 6 + .../test-configurations/streams/single.json | 14 ++- .../test-configurations/streams/smoke.json | 11 ++ .../test-configurations/streams/text.json | 11 ++ test/functional/src/Constants.js | 12 +- test/functional/test/text/forced-subtitles.js | 117 ++++++++++++++++++ .../streaming/streaming.text.TextTracks.js | 8 +- 15 files changed, 365 insertions(+), 87 deletions(-) create mode 100644 test/functional/test/text/forced-subtitles.js diff --git a/src/core/events/CoreEvents.js b/src/core/events/CoreEvents.js index 9948be15aa..8bffb0aa56 100644 --- a/src/core/events/CoreEvents.js +++ b/src/core/events/CoreEvents.js @@ -62,6 +62,7 @@ class CoreEvents extends EventsBase { this.MANIFEST_UPDATED = 'manifestUpdated'; this.MEDIA_FRAGMENT_LOADED = 'mediaFragmentLoaded'; this.MEDIA_FRAGMENT_NEEDED = 'mediaFragmentNeeded'; + this.MEDIAINFO_UPDATED = 'mediaInfoUpdated'; this.QUOTA_EXCEEDED = 'quotaExceeded'; this.SEGMENT_LOCATION_BLACKLIST_ADD = 'segmentLocationBlacklistAdd'; this.SEGMENT_LOCATION_BLACKLIST_CHANGED = 'segmentLocationBlacklistChanged'; diff --git a/src/dash/constants/DashConstants.js b/src/dash/constants/DashConstants.js index 8495a7954f..347e33f2c3 100644 --- a/src/dash/constants/DashConstants.js +++ b/src/dash/constants/DashConstants.js @@ -51,6 +51,7 @@ export default { BITSTREAM_SWITCHING: 'BitstreamSwitching', BITSTREAM_SWITCHING_MINUS: 'bitstreamSwitching', BYTE_RANGE: 'byteRange', + CAPTION: 'caption', CENC_DEFAULT_KID: 'cenc:default_KID', CLIENT_DATA_REPORTING: 'ClientDataReporting', CLIENT_REQUIREMENT: 'clientRequirement', @@ -86,6 +87,7 @@ export default { ESSENTIAL_PROPERTY: 'EssentialProperty', EVENT: 'Event', EVENT_STREAM: 'EventStream', + FORCED_SUBTITLE: 'forced-subtitle', FRAMERATE: 'frameRate', FRAME_PACKING: 'FramePacking', GROUP_LABEL: 'GroupLabel', @@ -171,6 +173,7 @@ export default { START_WITH_SAP: 'startWithSAP', STATIC: 'static', SUBSET: 'Subset', + SUBTITLE: 'subtitle', SUB_REPRESENTATION: 'SubRepresentation', SUB_SEGMENT_ALIGNMENT: 'subsegmentAlignment', SUGGESTED_PRESENTATION_DELAY: 'suggestedPresentationDelay', diff --git a/src/streaming/StreamProcessor.js b/src/streaming/StreamProcessor.js index b6c86c98fb..2a8436b36c 100644 --- a/src/streaming/StreamProcessor.js +++ b/src/streaming/StreamProcessor.js @@ -659,7 +659,9 @@ function StreamProcessor(config) { selectedValues = _handleMpdUpdate(selectionInput); } - currentMediaInfo = selectedValues.currentMediaInfo; + _setCurrentMediaInfo(selectedValues.currentMediaInfo); + + eventBus.trigger() // Update Representation Controller with the new data. Note we do not filter any Representations here as the filter values might change over time. const voRepresentations = abrController.getPossibleVoRepresentations(currentMediaInfo, false); @@ -676,6 +678,15 @@ function StreamProcessor(config) { }) } + function _setCurrentMediaInfo(value) { + currentMediaInfo = value; + eventBus.trigger(Events.MEDIAINFO_UPDATED, { + mediaType: type, + streamId: streamInfo.id, + currentMediaInfo + }); + } + function _handleAdaptationSetQualitySwitch(selectionInput) { return { selectedRepresentation: selectionInput.newRepresentation, @@ -756,7 +767,7 @@ function StreamProcessor(config) { scheduleController.setSwitchTrack(true); const newMediaInfo = newRepresentation.mediaInfo; - currentMediaInfo = newMediaInfo; + _setCurrentMediaInfo(newMediaInfo); selectMediaInfo({ newMediaInfo, newRepresentation }) .then(() => { diff --git a/src/streaming/constants/Constants.js b/src/streaming/constants/Constants.js index d63908f3e8..390619222c 100644 --- a/src/streaming/constants/Constants.js +++ b/src/streaming/constants/Constants.js @@ -330,5 +330,6 @@ export default { * @static */ ID3_SCHEME_ID_URI: 'https://aomedia.org/emsg/ID3', - COMMON_ACCESS_TOKEN_HEADER: 'common-access-token' + COMMON_ACCESS_TOKEN_HEADER: 'common-access-token', + DASH_ROLE_SCHEME_ID : 'urn:mpeg:dash:role:2011', } diff --git a/src/streaming/controllers/MediaController.js b/src/streaming/controllers/MediaController.js index d8715aadda..2bb032f137 100644 --- a/src/streaming/controllers/MediaController.js +++ b/src/streaming/controllers/MediaController.js @@ -36,6 +36,7 @@ import Debug from '../../core/Debug.js'; import {bcp47Normalize} from 'bcp-47-normalize'; import {extendedFilter} from 'bcp-47-match'; import MediaPlayerEvents from '../MediaPlayerEvents.js'; +import DashConstants from '../../dash/constants/DashConstants.js'; function MediaController() { @@ -121,7 +122,7 @@ function MediaController() { const possibleTracks = getTracksFor(type, streamInfo.id); let filteredTracks = []; - if (!settings) { + if (!settings || Object.keys(settings).length === 0) { settings = domStorage.getSavedMediaSettings(type); if (settings) { // If the settings are defined locally, do not take codec into account or it'll be too strict. @@ -603,7 +604,7 @@ function MediaController() { function selectInitialTrack(type, mediaInfos) { if (type === Constants.TEXT) { - return mediaInfos[0]; + return _handleInitialTextTrackSelection(mediaInfos); } let tmpArr; @@ -650,6 +651,23 @@ function MediaController() { return tmpArr.length > 0 ? tmpArr[0] : mediaInfos[0]; } + function _handleInitialTextTrackSelection(mediaInfos) { + const filteredMediaInfos = mediaInfos.filter((mediaInfo) => { + if (mediaInfo && mediaInfo.roles && mediaInfo.roles.length > 0) { + return mediaInfo.roles.every((role) => { + return role.schemeIdUri !== Constants.DASH_ROLE_SCHEME_ID || role.value !== DashConstants.FORCED_SUBTITLE + }) + } + return true + }) + + if (filteredMediaInfos.length > 0) { + return filteredMediaInfos[0]; + } + + return null + } + /** * @param {MediaInfo[]} mediaInfos * @return {MediaInfo[]} diff --git a/src/streaming/text/TextController.js b/src/streaming/text/TextController.js index 6cc37bb9bf..ab479f3d38 100644 --- a/src/streaming/text/TextController.js +++ b/src/streaming/text/TextController.js @@ -41,6 +41,7 @@ import Events from '../../core/events/Events.js'; import MediaPlayerEvents from '../../streaming/MediaPlayerEvents.js'; import {checkParameterType} from '../utils/SupervisorTools.js'; import DVBFonts from './DVBFonts.js'; +import DashConstants from '../../dash/constants/DashConstants.js'; function TextController(config) { @@ -91,6 +92,7 @@ function TextController(config) { eventBus.on(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance); eventBus.on(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance); eventBus.on(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance); + eventBus.on(Events.MEDIAINFO_UPDATED, _onMediaInfoUpdated, instance); if (settings.get().streaming.text.webvtt.customRenderingEnabled) { eventBus.on(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance); eventBus.on(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance); @@ -290,6 +292,32 @@ function TextController(config) { } } + function _onMediaInfoUpdated(e) { + try { + if (!e || !e.mediaType || e.mediaType !== Constants.AUDIO || !e.currentMediaInfo) { + return + } + + const currentTextTrackInfo = textTracks[e.streamId].getCurrentTextTrackInfo(); + let suitableForcedSubtitleIndex = NaN; + if (allTracksAreDisabled) { + suitableForcedSubtitleIndex = _getSuitableForceSubtitleTrackIndex(e.streamId); + } else if (_isForcedSubtitleTrack(currentTextTrackInfo) && e.currentMediaInfo.lang && e.currentMediaInfo.lang !== currentTextTrackInfo.lang) { + suitableForcedSubtitleIndex = _getSuitableForceSubtitleTrackIndex(e.streamId); + if (isNaN(suitableForcedSubtitleIndex)) { + suitableForcedSubtitleIndex = -1; + } + } + + if (!isNaN(suitableForcedSubtitleIndex)) { + setTextTrack(e.streamId, suitableForcedSubtitleIndex); + } + + } catch (e) { + logger.error(e); + } + } + function enableText(streamId, enable) { checkParameterType(enable, 'boolean'); if (isTextEnabled() !== enable) { @@ -346,7 +374,7 @@ function TextController(config) { textTracks[streamId].disableManualTracks(); - let currentTrackInfo = textTracks[streamId].getCurrentTrackInfo(); + let currentTrackInfo = textTracks[streamId].getCurrentTextTrackInfo(); let currentNativeTrackInfo = (currentTrackInfo) ? videoModel.getTextTrack(currentTrackInfo.kind, currentTrackInfo.id, currentTrackInfo.lang, currentTrackInfo.isTTML, currentTrackInfo.isEmbedded) : null; // Don't change disabled tracks - dvb font download for essential property failed or not complete @@ -356,7 +384,7 @@ function TextController(config) { textTracks[streamId].setCurrentTrackIdx(idx); - currentTrackInfo = textTracks[streamId].getCurrentTrackInfo(); + currentTrackInfo = textTracks[streamId].getCurrentTextTrackInfo(); const dispatchForManualRendering = settings.get().streaming.text.dispatchForManualRendering; @@ -368,6 +396,12 @@ function TextController(config) { _setFragmentedTextTrack(streamId, currentTrackInfo, oldTrackIdx); } else if (currentTrackInfo && !currentTrackInfo.isFragmented) { _setNonFragmentedTextTrack(streamId, currentTrackInfo); + } else if (!currentTrackInfo && allTracksAreDisabled) { + const forcedSubtitleTrackIndex = _getSuitableForceSubtitleTrackIndex(streamId) + if (!isNaN(forcedSubtitleTrackIndex)) { + setTextTrack(streamId, forcedSubtitleTrackIndex); + } + return } mediaController.setTrack(currentTrackInfo); @@ -412,6 +446,49 @@ function TextController(config) { }); } + function _getSuitableForceSubtitleTrackIndex(streamId) { + const forcedSubtitleTracks = _getForcedSubtitleTracks(streamId); + + if (!forcedSubtitleTracks || forcedSubtitleTracks.length <= 0) { + return NaN + } + + const currentAudioTrack = mediaController.getCurrentTrackFor(Constants.AUDIO, streamId); + if (!currentAudioTrack) { + return NaN + } + + const suitableTrack = forcedSubtitleTracks.find((track) => { + return currentAudioTrack.lang === track.lang + }) + + if (suitableTrack) { + return suitableTrack._indexToSelect + } + + return NaN + } + + function _getForcedSubtitleTracks(streamId) { + const textTrackInfos = textTracks[streamId].getTextTrackInfos(); + return textTrackInfos.filter((textTrackInfo, index) => { + textTrackInfo._indexToSelect = index; + if (textTrackInfo && textTrackInfo.roles && textTrackInfo.roles.length > 0) { + return _isForcedSubtitleTrack(textTrackInfo); + } + return false + }); + } + + function _isForcedSubtitleTrack(textTrackInfo) { + if (!textTrackInfo || !textTrackInfo.roles || textTrackInfo.roles.length === 0) { + return false + } + return textTrackInfo.roles.some((role) => { + return role.schemeIdUri === Constants.DASH_ROLE_SCHEME_ID && role.value === DashConstants.FORCED_SUBTITLE + }) + } + function getCurrentTrackIdx(streamId) { return textTracks[streamId].getCurrentTrackIdx(); } @@ -446,6 +523,7 @@ function TextController(config) { eventBus.off(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance); eventBus.off(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance); eventBus.off(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance); + eventBus.off(Events.MEDIAINFO_UPDATED, _onMediaInfoUpdated, instance); if (settings.get().streaming.text.webvtt.customRenderingEnabled) { eventBus.off(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance); eventBus.off(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance) @@ -458,20 +536,20 @@ function TextController(config) { } instance = { + addEmbeddedTrack, + addMediaInfosToBuffer, + createTracks, deactivateStream, + enableForcedTextStreaming, + enableText, + getAllTracksAreDisabled, + getCurrentTrackIdx, + getTextSourceBuffer, initialize, initializeForStream, - createTracks, - getTextSourceBuffer, - getAllTracksAreDisabled, - addEmbeddedTrack, - enableText, isTextEnabled, + reset, setTextTrack, - getCurrentTrackIdx, - enableForcedTextStreaming, - addMediaInfosToBuffer, - reset }; setup(); return instance; diff --git a/src/streaming/text/TextSourceBuffer.js b/src/streaming/text/TextSourceBuffer.js index 5a4155eca7..d8e44b57f8 100644 --- a/src/streaming/text/TextSourceBuffer.js +++ b/src/streaming/text/TextSourceBuffer.js @@ -43,6 +43,7 @@ import DashJSError from '../vo/DashJSError.js'; import Errors from '../../core/errors/Errors.js'; import {Cta608Parser} from '@svta/common-media-library/cta/608/Cta608Parser'; import {extractCta608DataFromSample} from '@svta/common-media-library/cta/608/extractCta608DataFromSample'; +import DashConstants from '../../dash/constants/DashConstants.js'; function TextSourceBuffer(config) { const errHandler = config.errHandler; @@ -136,7 +137,7 @@ function TextSourceBuffer(config) { } for (let i = 0; i < mInfos.length; i++) { - _createTextTrackFromMediaInfo(mInfos[i]); + _createTextTrackInfoFromMediaInfo(mInfos[i]); } } @@ -146,22 +147,26 @@ function TextSourceBuffer(config) { * @param {object} mediaInfo * @private */ - function _createTextTrackFromMediaInfo(mediaInfo) { + function _createTextTrackInfoFromMediaInfo(mediaInfo) { + + // We are mapping DASH specification strings to the ones of the HTML specification. + // See also https://html.spec.whatwg.org/multipage/media.html#text-track-kind + const trackKindMap = {}; + trackKindMap[DashConstants.SUBTITLE] = 'subtitles' + trackKindMap[DashConstants.CAPTION] = 'captions' + trackKindMap[DashConstants.FORCED_SUBTITLE] = 'subtitles' + const textTrackInfo = new TextTrackInfo(); - const trackKindMap = { subtitle: 'subtitles', caption: 'captions' }; //Dash Spec has no "s" on end of KIND but HTML needs plural. for (let key in mediaInfo) { textTrackInfo[key] = mediaInfo[key]; } - textTrackInfo.labels = mediaInfo.labels; textTrackInfo.defaultTrack = getIsDefault(mediaInfo); - textTrackInfo.isFragmented = mediaInfo.isFragmented; - textTrackInfo.isEmbedded = !!mediaInfo.isEmbedded; textTrackInfo.isTTML = _checkTtml(mediaInfo); textTrackInfo.kind = _getKind(mediaInfo, trackKindMap); - textTracks.addTextTrack(textTrackInfo); + textTracks.addTextTrackInfo(textTrackInfo); } function abort() { @@ -250,7 +255,7 @@ function TextSourceBuffer(config) { function _getKind(mediaInfo, trackKindMap) { let kind = (mediaInfo.roles && mediaInfo.roles.length > 0) ? trackKindMap[mediaInfo.roles[0].value] : trackKindMap.caption; - kind = (kind === trackKindMap.caption || kind === trackKindMap.subtitle) ? kind : trackKindMap.caption; + kind = Object.values(trackKindMap).includes(kind) ? kind : trackKindMap.caption; return kind; } @@ -331,7 +336,7 @@ function TextSourceBuffer(config) { const offsetTime = manifest.ttmlTimeIsRelative ? sampleStart / timescale : 0; const result = parser.parse(ccContent, offsetTime, (sampleStart / timescale), ((sampleStart + sample.duration) / timescale), images); textTracks.addCaptions(currFragmentedTrackIdx, timestampOffset, result); - + } catch (e) { fragmentModel.removeExecutedRequestsBeforeTime(); this.remove(); @@ -542,7 +547,7 @@ function TextSourceBuffer(config) { for (let i = 0; i < samples.length; i++) { const sample = samples[i]; const ccData = extractCta608DataFromSample(raw, sample.offset, sample.size); - + let lastSampleTime = null; let idx = 0; for (let k = 0; k < 2; k++) { diff --git a/src/streaming/text/TextTracks.js b/src/streaming/text/TextTracks.js index 7befd10ce9..6269a02178 100644 --- a/src/streaming/text/TextTracks.js +++ b/src/streaming/text/TextTracks.js @@ -65,8 +65,8 @@ function TextTracks(config) { let instance, logger, Cue, - textTrackQueue, - nativeTrackElementArr, + textTrackInfos, + nativeTexttracks, currentTrackIdx, actualVideoLeft, actualVideoTop, @@ -93,8 +93,8 @@ function TextTracks(config) { } Cue = window.VTTCue || window.TextTrackCue; - textTrackQueue = []; - nativeTrackElementArr = []; + textTrackInfos = []; + nativeTexttracks = []; currentTrackIdx = -1; actualVideoLeft = 0; actualVideoTop = 0; @@ -123,43 +123,24 @@ function TextTracks(config) { return streamInfo.id; } - function _createTrackForUserAgent(element) { - const kind = element.kind; - const label = element.id !== undefined ? element.id : element.lang; - const lang = element.lang; - const isTTML = element.isTTML; - const isEmbedded = element.isEmbedded; - const track = videoModel.addTextTrack(kind, label, lang, isTTML, isEmbedded); - - return track; - } - - function addTextTrack(textTrackInfoVO) { - textTrackQueue.push(textTrackInfoVO); - } - function createTracks() { - const dispatchForManualRendering = settings.get().streaming.text.dispatchForManualRendering; - //Sort in same order as in manifest - textTrackQueue.sort(function (a, b) { + textTrackInfos.sort(function (a, b) { return a.index - b.index; }); captionContainer = videoModel.getTTMLRenderingDiv(); vttCaptionContainer = videoModel.getVttRenderingDiv(); let defaultIndex = -1; - for (let i = 0; i < textTrackQueue.length; i++) { - const track = _createTrackForUserAgent(textTrackQueue[i]); + for (let i = 0; i < textTrackInfos.length; i++) { + const nativeTexttrack = _createNativeTextrackElement(textTrackInfos[i]); //used to remove tracks from video element when added manually - nativeTrackElementArr.push(track); + nativeTexttracks.push(nativeTexttrack); - if (textTrackQueue[i].defaultTrack) { + if (textTrackInfos[i].defaultTrack) { // track.default is an object property identifier that is a reserved word - // The following jshint directive is used to suppressed the warning "Expected an identifier and instead saw 'default' (a reserved word)" - /*jshint -W024 */ - track.default = true; + nativeTexttrack.default = true; defaultIndex = i; } @@ -168,14 +149,14 @@ function TextTracks(config) { //each time a track is created, its mode should be showing by default //sometime, it's not on Chrome textTrack.mode = Constants.TEXT_SHOWING; - if (captionContainer && (textTrackQueue[i].isTTML || textTrackQueue[i].isEmbedded)) { + if (captionContainer && (textTrackInfos[i].isTTML || textTrackInfos[i].isEmbedded)) { textTrack.renderingType = 'html'; } else { textTrack.renderingType = 'default'; } } - addCaptions(i, 0, textTrackQueue[i].captionData); + addCaptions(i, 0, textTrackInfos[i].captionData); eventBus.trigger(MediaPlayerEvents.TEXT_TRACK_ADDED); } @@ -194,9 +175,10 @@ function TextTracks(config) { eventBus.on(MediaPlayerEvents.PLAYBACK_METADATA_LOADED, onMetadataLoaded, this); - for (let idx = 0; idx < textTrackQueue.length; idx++) { + for (let idx = 0; idx < textTrackInfos.length; idx++) { const videoTextTrack = getTrackByIdx(idx); if (videoTextTrack) { + const dispatchForManualRendering = settings.get().streaming.text.dispatchForManualRendering; videoTextTrack.mode = (idx === defaultIndex && !dispatchForManualRendering) ? Constants.TEXT_SHOWING : Constants.TEXT_HIDDEN; videoTextTrack.manualMode = (idx === defaultIndex) ? Constants.TEXT_SHOWING : Constants.TEXT_HIDDEN; } @@ -205,11 +187,26 @@ function TextTracks(config) { eventBus.trigger(Events.TEXT_TRACKS_QUEUE_INITIALIZED, { index: currentTrackIdx, - tracks: textTrackQueue, + tracks: textTrackInfos, streamId: streamInfo.id }); } + function _createNativeTextrackElement(element) { + const kind = element.kind; + const label = element.id !== undefined ? element.id : element.lang; + const lang = element.lang; + const isTTML = element.isTTML; + const isEmbedded = element.isEmbedded; + const track = videoModel.addTextTrack(kind, label, lang, isTTML, isEmbedded); + + return track; + } + + function addTextTrackInfo(textTrackInfoVO) { + textTrackInfos.push(textTrackInfoVO); + } + function getVideoVisibleVideoSize(viewWidth, viewHeight, videoWidth, videoHeight, aspectRatio, use80Percent) { const viewAspectRatio = viewWidth / viewHeight; const videoAspectRatio = videoWidth / videoHeight; @@ -812,8 +809,8 @@ function TextTracks(config) { } function getTrackByIdx(idx) { - return idx >= 0 && textTrackQueue[idx] ? - videoModel.getTextTrack(textTrackQueue[idx].kind, textTrackQueue[idx].id, textTrackQueue[idx].lang, textTrackQueue[idx].isTTML, textTrackQueue[idx].isEmbedded) : null; + return idx >= 0 && textTrackInfos[idx] ? + videoModel.getTextTrack(textTrackInfos[idx].kind, textTrackInfos[idx].id, textTrackInfos[idx].lang, textTrackInfos[idx].isTTML, textTrackInfos[idx].isEmbedded) : null; } function getCurrentTrackIdx() { @@ -822,8 +819,8 @@ function TextTracks(config) { function getTrackIdxForId(trackId) { let idx = -1; - for (let i = 0; i < textTrackQueue.length; i++) { - if (textTrackQueue[i].id === trackId) { + for (let i = 0; i < textTrackInfos.length; i++) { + if (textTrackInfos[i].id === trackId) { idx = i; break; } @@ -959,15 +956,15 @@ function TextTracks(config) { } function deleteAllTextTracks() { - const ln = nativeTrackElementArr ? nativeTrackElementArr.length : 0; + const ln = nativeTexttracks ? nativeTexttracks.length : 0; for (let i = 0; i < ln; i++) { const track = getTrackByIdx(i); if (track) { _deleteTrackCues.call(this, track, streamInfo.start, streamInfo.start + streamInfo.duration, false); } } - nativeTrackElementArr = []; - textTrackQueue = []; + nativeTexttracks = []; + textTrackInfos = []; if (videoSizeCheckInterval) { clearInterval(videoSizeCheckInterval); videoSizeCheckInterval = null; @@ -1033,25 +1030,30 @@ function TextTracks(config) { } } - function getCurrentTrackInfo() { - return textTrackQueue[currentTrackIdx]; + function getCurrentTextTrackInfo() { + return textTrackInfos[currentTrackIdx]; + } + + function getTextTrackInfos() { + return textTrackInfos } instance = { - initialize, - getStreamId, - addTextTrack, addCaptions, + addTextTrackInfo, createTracks, + deleteAllTextTracks, + deleteCuesFromTrackIdx, + disableManualTracks, getCurrentTrackIdx, - setCurrentTrackIdx, + getCurrentTextTrackInfo, + getStreamId, + getTextTrackInfos, getTrackIdxForId, - getCurrentTrackInfo, - setModeForTrackIdx, - deleteCuesFromTrackIdx, - deleteAllTextTracks, + initialize, manualCueProcessing, - disableManualTracks + setCurrentTrackIdx, + setModeForTrackIdx, }; setup(); diff --git a/test/functional/adapter/DashJsAdapter.js b/test/functional/adapter/DashJsAdapter.js index 400b1dfd21..c9bdf9c7dc 100644 --- a/test/functional/adapter/DashJsAdapter.js +++ b/test/functional/adapter/DashJsAdapter.js @@ -622,6 +622,12 @@ class DashJsAdapter { delayPollInterval = setInterval(_checkBuffer, 100); }) } + + async sleep(timeoutValue) { + return new Promise((resolve) => { + setTimeout(resolve, timeoutValue) + }); + } } export default DashJsAdapter; diff --git a/test/functional/config/test-configurations/streams/single.json b/test/functional/config/test-configurations/streams/single.json index 2c26ef27b1..c6ed0335f0 100644 --- a/test/functional/config/test-configurations/streams/single.json +++ b/test/functional/config/test-configurations/streams/single.json @@ -4,17 +4,21 @@ "all" ], "excluded": [ - "vendor/google-ad-manager-emsg" + "vendor/google-ad-manager-emsg", + "buffer/buffer-to-keep-seek" ] }, "testvectors": [ { - "name": "Segment Base", + "name": "Arte forced-subtitles", "type": "vod", - "url": "https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd", + "url": "https://arteamd1.akamaized.net/GPU/034000/034700/034755-230-A/221125154117/034755-230-A_8_DA_v20221125.mpd", "includedTestfiles": [ - "playback/*" - ] + "text/forced-subtitles" + ], + "testdata": { + "forcedSubtitles": {} + } } ] } diff --git a/test/functional/config/test-configurations/streams/smoke.json b/test/functional/config/test-configurations/streams/smoke.json index 8b7fd4e531..b3edf7165f 100644 --- a/test/functional/config/test-configurations/streams/smoke.json +++ b/test/functional/config/test-configurations/streams/smoke.json @@ -169,6 +169,17 @@ "includedTestfiles": [ "feature-support/mpd-patching" ] + }, + { + "name": "Arte forced-subtitles", + "type": "vod", + "url": "https://arteamd1.akamaized.net/GPU/034000/034700/034755-230-A/221125154117/034755-230-A_8_DA_v20221125.mpd", + "includedTestfiles": [ + "text/forced-subtitles" + ], + "testdata": { + "forcedSubtitles": {} + } } ] } diff --git a/test/functional/config/test-configurations/streams/text.json b/test/functional/config/test-configurations/streams/text.json index bfbce0bc5c..367493741a 100644 --- a/test/functional/config/test-configurations/streams/text.json +++ b/test/functional/config/test-configurations/streams/text.json @@ -176,6 +176,17 @@ "includedTestfiles": [ "text/*" ] + }, + { + "name": "Arte forced-subtitles", + "type": "vod", + "url": "https://arteamd1.akamaized.net/GPU/034000/034700/034755-230-A/221125154117/034755-230-A_8_DA_v20221125.mpd", + "includedTestfiles": [ + "text/forced-subtitles" + ], + "testdata": { + "forcedSubtitles": {} + } } ] } diff --git a/test/functional/src/Constants.js b/test/functional/src/Constants.js index 806686d065..0088f54675 100644 --- a/test/functional/src/Constants.js +++ b/test/functional/src/Constants.js @@ -114,6 +114,13 @@ const DRM_SYSTEMS = { PLAYREADY: 'com.microsoft.playready' }; +const ROLES = { + FORCED_SUBTITLES: { + SCHEME_ID_URI: 'urn:mpeg:dash:role:2011', + VALUE: 'forced-subtitle' + } +}; + const exportedObject = { DASH_JS: { MEDIA_TYPES: { @@ -127,7 +134,8 @@ const exportedObject = { TEST_INPUTS: TEST_INPUTS, CONTENT_TYPES: CONTENT_TYPES, SEGMENT_TYPES: SEGMENT_TYPES, - DRM_SYSTEMS: DRM_SYSTEMS + DRM_SYSTEMS: DRM_SYSTEMS, + ROLES: ROLES }; TESTCASES.ADVANCED.NO_RELOAD_AFTER_SEEK = TESTCASES.CATEGORIES.ADVANCED + 'no-reload-after-seek'; @@ -163,9 +171,11 @@ TESTCASES.PLAYBACK_ADVANCED.PRELOAD = TESTCASES.CATEGORIES.PLAYBACK_ADVANCED + ' TESTCASES.TEXT.INITIAL = TESTCASES.CATEGORIES.TEXT + 'initial-text'; TESTCASES.TEXT.SWITCH = TESTCASES.CATEGORIES.TEXT + 'switch-text'; +TESTCASES.TEXT.FORCED_SUBTITLES = TESTCASES.CATEGORIES.TEXT + 'forced-subtitles'; TESTCASES.VENDOR.GOOGLE_AD_MANAGER_EMSG = TESTCASES.CATEGORIES.VENDOR + 'google-ad-manager-emsg'; TESTCASES.VIDEO.SWITCH = TESTCASES.CATEGORIES.VIDEO + 'switch-video'; + export default exportedObject; diff --git a/test/functional/test/text/forced-subtitles.js b/test/functional/test/text/forced-subtitles.js new file mode 100644 index 0000000000..1dd48af084 --- /dev/null +++ b/test/functional/test/text/forced-subtitles.js @@ -0,0 +1,117 @@ +import Constants from '../../src/Constants.js'; +import Utils from '../../src/Utils.js'; +import {expect} from 'chai' +import { + checkIsPlaying, + checkIsProgressing, + checkNoCriticalErrors, + initializeDashJsAdapter +} from '../common/common.js'; + +const TESTCASE = Constants.TESTCASES.TEXT.FORCED_SUBTITLES; + +Utils.getTestvectorsForTestcase(TESTCASE).forEach((item) => { + const mpd = item.url; + + describe(`${TESTCASE} - ${item.name} - ${mpd}`, () => { + + let playerAdapter + + before(function () { + if (!item.testdata || !item.testdata.forcedSubtitles) { + this.skip(); + } + playerAdapter = initializeDashJsAdapter(item, mpd); + }) + + after(() => { + if (playerAdapter) { + playerAdapter.destroy(); + } + }) + + it(`Checking playing state`, async () => { + await checkIsPlaying(playerAdapter, true); + }) + + it(`Checking progressing state`, async () => { + await checkIsProgressing(playerAdapter); + }); + + it(`Turn off subtitles`, async () => { + playerAdapter.setTextTrack(-1); + }); + + it(`Switch to audio tracks for which we got a forced subtitle. Expect the forced subtitle to be active`, async () => { + const availableTextTracks = playerAdapter.getTracksFor(Constants.DASH_JS.MEDIA_TYPES.TEXT); + const availableAudioTracks = playerAdapter.getTracksFor(Constants.DASH_JS.MEDIA_TYPES.AUDIO); + const forcedSubtitleTracks = _getForcedSubtitleTracks(availableTextTracks); + + for (let i = 0; i < forcedSubtitleTracks.length; i++) { + const forcedSubtitleTrack = forcedSubtitleTracks[i]; + const suitableAudioTrack = _getAudioTrackForForcedSubtitleTrack(forcedSubtitleTrack, availableAudioTracks); + if (suitableAudioTrack) { + playerAdapter.setCurrentTrack(suitableAudioTrack); + await playerAdapter.sleep(1000); + const currentTextTrack = playerAdapter.getCurrentTrackFor(Constants.DASH_JS.MEDIA_TYPES.TEXT); + const currentAudioTrack = playerAdapter.getCurrentTrackFor(Constants.DASH_JS.MEDIA_TYPES.AUDIO); + expect(currentAudioTrack.lang).to.be.equal(suitableAudioTrack.lang); + expect(currentAudioTrack.index).to.be.equal(suitableAudioTrack.index); + expect(currentTextTrack.lang).to.be.equal(forcedSubtitleTrack.lang); + expect(currentTextTrack.index).to.be.equal(forcedSubtitleTrack.index); + } + } + }); + + it(`Switch to audio tracks for which we got no forced subtitle. Expect no subtitle to be active`, async () => { + playerAdapter.setTextTrack(-1); + const availableTextTracks = playerAdapter.getTracksFor(Constants.DASH_JS.MEDIA_TYPES.TEXT); + const availableAudioTracks = playerAdapter.getTracksFor(Constants.DASH_JS.MEDIA_TYPES.AUDIO); + const forcedSubtitleTracks = _getForcedSubtitleTracks(availableTextTracks); + + for (let i = 0; i < availableAudioTracks.length; i++) { + const currentAudioTrack = availableAudioTracks[i]; + const suitableForcedSubtitleTrack = _getForcedSubtitleTrackForAudioTrack(currentAudioTrack, forcedSubtitleTracks); + if (!suitableForcedSubtitleTrack) { + playerAdapter.setCurrentTrack(currentAudioTrack); + await playerAdapter.sleep(1000); + const currentIndex = playerAdapter.getCurrentTextTrackIndex(); + expect(currentIndex).to.be.equal(-1); + } + } + }); + + it(`Expect no critical errors to be thrown`, () => { + checkNoCriticalErrors(playerAdapter); + }) + }) +}) + +function _getForcedSubtitleTracks(textTrackInfos) { + return textTrackInfos.filter((textTrackInfo, index) => { + textTrackInfo._indexToSelect = index; + if (textTrackInfo && textTrackInfo.roles && textTrackInfo.roles.length > 0) { + return _isForcedSubtitleTrack(textTrackInfo); + } + return false + }); +} + +function _isForcedSubtitleTrack(textTrackInfo) { + return textTrackInfo.roles.some((role) => { + return role.schemeIdUri === Constants.ROLES.FORCED_SUBTITLES.SCHEME_ID_URI && role.value === Constants.ROLES.FORCED_SUBTITLES.VALUE + }) +} + +function _getAudioTrackForForcedSubtitleTrack(textTrackInfo, availableAudioTracks) { + return availableAudioTracks.find((audioTrack) => { + return audioTrack.lang === textTrackInfo.lang + }) +} + +function _getForcedSubtitleTrackForAudioTrack(audioTrack, forcedSubtitleTracks) { + return forcedSubtitleTracks.find((forcedSubtitleTrack) => { + return audioTrack.lang === forcedSubtitleTrack.lang + }) +} + diff --git a/test/unit/test/streaming/streaming.text.TextTracks.js b/test/unit/test/streaming/streaming.text.TextTracks.js index 4c8a784071..8f70fa7ed4 100644 --- a/test/unit/test/streaming/streaming.text.TextTracks.js +++ b/test/unit/test/streaming/streaming.text.TextTracks.js @@ -48,15 +48,15 @@ describe('TextTracks', function () { }); }); - describe('Method addTextTrack', function () { - it('should trigger TEXT_TRACK_ADDED and TEXT_TRACKS_QUEUE_INITIALIZED events when a call to addTextTrackfunction is made', function () { + describe('Method addTextTrackInfo', function () { + it('should trigger TEXT_TRACK_ADDED and TEXT_TRACKS_QUEUE_INITIALIZED events when a call to addTextTrackInfo function is made', function () { const spyTrackAdded = chai.spy(); const spyTracksQueueInit = chai.spy(); eventBus.on(Events.TEXT_TRACK_ADDED, spyTrackAdded); eventBus.on(Events.TEXT_TRACKS_QUEUE_INITIALIZED, spyTracksQueueInit); - textTracks.addTextTrack({ + textTracks.addTextTrackInfo({ index: 0, kind: 'subtitles', label: 'eng', @@ -76,7 +76,7 @@ describe('TextTracks', function () { describe('Method addCaptions', function () { it('should call addCue function when a call to addCaptions is made', function () { - textTracks.addTextTrack({ + textTracks.addTextTrackInfo({ index: 0, kind: 'subtitles', id: 'eng',