Source: lib/media/streaming_engine.js

  1. /*! @license
  2. * Shaka Player
  3. * Copyright 2016 Google LLC
  4. * SPDX-License-Identifier: Apache-2.0
  5. */
  6. /**
  7. * @fileoverview
  8. * @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
  9. */
  10. goog.provide('shaka.media.StreamingEngine');
  11. goog.require('goog.asserts');
  12. goog.require('shaka.log');
  13. goog.require('shaka.media.InitSegmentReference');
  14. goog.require('shaka.media.MediaSourceEngine');
  15. goog.require('shaka.media.SegmentIterator');
  16. goog.require('shaka.media.SegmentReference');
  17. goog.require('shaka.net.Backoff');
  18. goog.require('shaka.net.NetworkingEngine');
  19. goog.require('shaka.util.DelayedTick');
  20. goog.require('shaka.util.Destroyer');
  21. goog.require('shaka.util.Error');
  22. goog.require('shaka.util.FakeEvent');
  23. goog.require('shaka.util.IDestroyable');
  24. goog.require('shaka.util.ManifestParserUtils');
  25. goog.require('shaka.util.MimeUtils');
  26. goog.require('shaka.util.Mp4Parser');
  27. goog.require('shaka.util.Networking');
  28. /**
  29. * @summary Creates a Streaming Engine.
  30. * The StreamingEngine is responsible for setting up the Manifest's Streams
  31. * (i.e., for calling each Stream's createSegmentIndex() function), for
  32. * downloading segments, for co-ordinating audio, video, and text buffering.
  33. * The StreamingEngine provides an interface to switch between Streams, but it
  34. * does not choose which Streams to switch to.
  35. *
  36. * The StreamingEngine does not need to be notified about changes to the
  37. * Manifest's SegmentIndexes; however, it does need to be notified when new
  38. * Variants are added to the Manifest.
  39. *
  40. * To start the StreamingEngine the owner must first call configure(), followed
  41. * by one call to switchVariant(), one optional call to switchTextStream(), and
  42. * finally a call to start(). After start() resolves, switch*() can be used
  43. * freely.
  44. *
  45. * The owner must call seeked() each time the playhead moves to a new location
  46. * within the presentation timeline; however, the owner may forego calling
  47. * seeked() when the playhead moves outside the presentation timeline.
  48. *
  49. * @implements {shaka.util.IDestroyable}
  50. */
  51. shaka.media.StreamingEngine = class {
  52. /**
  53. * @param {shaka.extern.Manifest} manifest
  54. * @param {shaka.media.StreamingEngine.PlayerInterface} playerInterface
  55. */
  56. constructor(manifest, playerInterface) {
  57. /** @private {?shaka.media.StreamingEngine.PlayerInterface} */
  58. this.playerInterface_ = playerInterface;
  59. /** @private {?shaka.extern.Manifest} */
  60. this.manifest_ = manifest;
  61. /** @private {?shaka.extern.StreamingConfiguration} */
  62. this.config_ = null;
  63. /** @private {number} */
  64. this.bufferingGoalScale_ = 1;
  65. /** @private {?shaka.extern.Variant} */
  66. this.currentVariant_ = null;
  67. /** @private {?shaka.extern.Stream} */
  68. this.currentTextStream_ = null;
  69. /**
  70. * Maps a content type, e.g., 'audio', 'video', or 'text', to a MediaState.
  71. *
  72. * @private {!Map.<shaka.util.ManifestParserUtils.ContentType,
  73. * !shaka.media.StreamingEngine.MediaState_>}
  74. */
  75. this.mediaStates_ = new Map();
  76. /**
  77. * Set to true once the initial media states have been created.
  78. *
  79. * @private {boolean}
  80. */
  81. this.startupComplete_ = false;
  82. /**
  83. * Used for delay and backoff of failure callbacks, so that apps do not
  84. * retry instantly.
  85. *
  86. * @private {shaka.net.Backoff}
  87. */
  88. this.failureCallbackBackoff_ = null;
  89. /**
  90. * Set to true on fatal error. Interrupts fetchAndAppend_().
  91. *
  92. * @private {boolean}
  93. */
  94. this.fatalError_ = false;
  95. /** @private {!shaka.util.Destroyer} */
  96. this.destroyer_ = new shaka.util.Destroyer(() => this.doDestroy_());
  97. }
  98. /** @override */
  99. destroy() {
  100. return this.destroyer_.destroy();
  101. }
  102. /**
  103. * @return {!Promise}
  104. * @private
  105. */
  106. async doDestroy_() {
  107. const aborts = [];
  108. for (const state of this.mediaStates_.values()) {
  109. this.cancelUpdate_(state);
  110. aborts.push(this.abortOperations_(state));
  111. }
  112. await Promise.all(aborts);
  113. this.mediaStates_.clear();
  114. this.playerInterface_ = null;
  115. this.manifest_ = null;
  116. this.config_ = null;
  117. }
  118. /**
  119. * Called by the Player to provide an updated configuration any time it
  120. * changes. Must be called at least once before start().
  121. *
  122. * @param {shaka.extern.StreamingConfiguration} config
  123. */
  124. configure(config) {
  125. this.config_ = config;
  126. // Create separate parameters for backoff during streaming failure.
  127. /** @type {shaka.extern.RetryParameters} */
  128. const failureRetryParams = {
  129. // The term "attempts" includes the initial attempt, plus all retries.
  130. // In order to see a delay, there would have to be at least 2 attempts.
  131. maxAttempts: Math.max(config.retryParameters.maxAttempts, 2),
  132. baseDelay: config.retryParameters.baseDelay,
  133. backoffFactor: config.retryParameters.backoffFactor,
  134. fuzzFactor: config.retryParameters.fuzzFactor,
  135. timeout: 0, // irrelevant
  136. stallTimeout: 0, // irrelevant
  137. connectionTimeout: 0, // irrelevant
  138. };
  139. // We don't want to ever run out of attempts. The application should be
  140. // allowed to retry streaming infinitely if it wishes.
  141. const autoReset = true;
  142. this.failureCallbackBackoff_ =
  143. new shaka.net.Backoff(failureRetryParams, autoReset);
  144. }
  145. /**
  146. * Initialize and start streaming.
  147. *
  148. * By calling this method, StreamingEngine will start streaming the variant
  149. * chosen by a prior call to switchVariant(), and optionally, the text stream
  150. * chosen by a prior call to switchTextStream(). Once the Promise resolves,
  151. * switch*() may be called freely.
  152. *
  153. * @return {!Promise}
  154. */
  155. async start() {
  156. goog.asserts.assert(this.config_,
  157. 'StreamingEngine configure() must be called before init()!');
  158. // Setup the initial set of Streams and then begin each update cycle.
  159. await this.initStreams_();
  160. this.destroyer_.ensureNotDestroyed();
  161. shaka.log.debug('init: completed initial Stream setup');
  162. this.startupComplete_ = true;
  163. }
  164. /**
  165. * Get the current variant we are streaming. Returns null if nothing is
  166. * streaming.
  167. * @return {?shaka.extern.Variant}
  168. */
  169. getCurrentVariant() {
  170. return this.currentVariant_;
  171. }
  172. /**
  173. * Get the text stream we are streaming. Returns null if there is no text
  174. * streaming.
  175. * @return {?shaka.extern.Stream}
  176. */
  177. getCurrentTextStream() {
  178. return this.currentTextStream_;
  179. }
  180. /**
  181. * Start streaming text, creating a new media state.
  182. *
  183. * @param {shaka.extern.Stream} stream
  184. * @return {!Promise}
  185. * @private
  186. */
  187. async loadNewTextStream_(stream) {
  188. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  189. goog.asserts.assert(!this.mediaStates_.has(ContentType.TEXT),
  190. 'Should not call loadNewTextStream_ while streaming text!');
  191. try {
  192. // Clear MediaSource's buffered text, so that the new text stream will
  193. // properly replace the old buffered text.
  194. // TODO: Should this happen in unloadTextStream() instead?
  195. await this.playerInterface_.mediaSourceEngine.clear(ContentType.TEXT);
  196. } catch (error) {
  197. if (this.playerInterface_) {
  198. this.playerInterface_.onError(error);
  199. }
  200. }
  201. const mimeType = shaka.util.MimeUtils.getFullType(
  202. stream.mimeType, stream.codecs);
  203. this.playerInterface_.mediaSourceEngine.reinitText(
  204. mimeType, this.manifest_.sequenceMode);
  205. const textDisplayer =
  206. this.playerInterface_.mediaSourceEngine.getTextDisplayer();
  207. const streamText =
  208. textDisplayer.isTextVisible() || this.config_.alwaysStreamText;
  209. if (streamText) {
  210. const state = this.createMediaState_(stream);
  211. this.mediaStates_.set(ContentType.TEXT, state);
  212. this.scheduleUpdate_(state, 0);
  213. }
  214. }
  215. /**
  216. * Stop fetching text stream when the user chooses to hide the captions.
  217. */
  218. unloadTextStream() {
  219. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  220. const state = this.mediaStates_.get(ContentType.TEXT);
  221. if (state) {
  222. this.cancelUpdate_(state);
  223. this.abortOperations_(state).catch(() => {});
  224. this.mediaStates_.delete(ContentType.TEXT);
  225. }
  226. this.currentTextStream_ = null;
  227. }
  228. /**
  229. * Set trick play on or off.
  230. * If trick play is on, related trick play streams will be used when possible.
  231. * @param {boolean} on
  232. */
  233. setTrickPlay(on) {
  234. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  235. const mediaState = this.mediaStates_.get(ContentType.VIDEO);
  236. if (!mediaState) {
  237. return;
  238. }
  239. const stream = mediaState.stream;
  240. if (!stream) {
  241. return;
  242. }
  243. shaka.log.debug('setTrickPlay', on);
  244. if (on) {
  245. const trickModeVideo = stream.trickModeVideo;
  246. if (!trickModeVideo) {
  247. return; // Can't engage trick play.
  248. }
  249. const normalVideo = mediaState.restoreStreamAfterTrickPlay;
  250. if (normalVideo) {
  251. return; // Already in trick play.
  252. }
  253. shaka.log.debug('Engaging trick mode stream', trickModeVideo);
  254. this.switchInternal_(trickModeVideo, /* clearBuffer= */ false,
  255. /* safeMargin= */ 0, /* force= */ false);
  256. mediaState.restoreStreamAfterTrickPlay = stream;
  257. } else {
  258. const normalVideo = mediaState.restoreStreamAfterTrickPlay;
  259. if (!normalVideo) {
  260. return;
  261. }
  262. shaka.log.debug('Restoring non-trick-mode stream', normalVideo);
  263. mediaState.restoreStreamAfterTrickPlay = null;
  264. this.switchInternal_(normalVideo, /* clearBuffer= */ true,
  265. /* safeMargin= */ 0, /* force= */ false);
  266. }
  267. }
  268. /**
  269. * @param {shaka.extern.Variant} variant
  270. * @param {boolean=} clearBuffer
  271. * @param {number=} safeMargin
  272. * @param {boolean=} force
  273. * If true, reload the variant even if it did not change.
  274. * @param {boolean=} adaptation
  275. * If true, update the media state to indicate MediaSourceEngine should
  276. * reset the timestamp offset to ensure the new track segments are correctly
  277. * placed on the timeline.
  278. */
  279. switchVariant(
  280. variant, clearBuffer = false, safeMargin = 0, force = false,
  281. adaptation = false) {
  282. this.currentVariant_ = variant;
  283. if (!this.startupComplete_) {
  284. // The selected variant will be used in start().
  285. return;
  286. }
  287. if (variant.video) {
  288. this.switchInternal_(
  289. variant.video, /* clearBuffer= */ clearBuffer,
  290. /* safeMargin= */ safeMargin, /* force= */ force,
  291. /* adaptation= */ adaptation);
  292. }
  293. if (variant.audio) {
  294. this.switchInternal_(
  295. variant.audio, /* clearBuffer= */ clearBuffer,
  296. /* safeMargin= */ safeMargin, /* force= */ force,
  297. /* adaptation= */ adaptation);
  298. }
  299. }
  300. /**
  301. * @param {shaka.extern.Stream} textStream
  302. */
  303. switchTextStream(textStream) {
  304. this.currentTextStream_ = textStream;
  305. if (!this.startupComplete_) {
  306. // The selected text stream will be used in start().
  307. return;
  308. }
  309. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  310. goog.asserts.assert(textStream && textStream.type == ContentType.TEXT,
  311. 'Wrong stream type passed to switchTextStream!');
  312. this.switchInternal_(
  313. textStream, /* clearBuffer= */ true,
  314. /* safeMargin= */ 0, /* force= */ false);
  315. }
  316. /** Reload the current text stream. */
  317. reloadTextStream() {
  318. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  319. const mediaState = this.mediaStates_.get(ContentType.TEXT);
  320. if (mediaState) { // Don't reload if there's no text to begin with.
  321. this.switchInternal_(
  322. mediaState.stream, /* clearBuffer= */ true,
  323. /* safeMargin= */ 0, /* force= */ true);
  324. }
  325. }
  326. /**
  327. * Switches to the given Stream. |stream| may be from any Variant.
  328. *
  329. * @param {shaka.extern.Stream} stream
  330. * @param {boolean} clearBuffer
  331. * @param {number} safeMargin
  332. * @param {boolean} force
  333. * If true, reload the text stream even if it did not change.
  334. * @param {boolean=} adaptation
  335. * If true, update the media state to indicate MediaSourceEngine should
  336. * reset the timestamp offset to ensure the new track segments are correctly
  337. * placed on the timeline.
  338. * @private
  339. */
  340. switchInternal_(stream, clearBuffer, safeMargin, force, adaptation) {
  341. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  342. const type = /** @type {!ContentType} */(stream.type);
  343. const mediaState = this.mediaStates_.get(type);
  344. if (!mediaState && stream.type == ContentType.TEXT) {
  345. this.loadNewTextStream_(stream);
  346. return;
  347. }
  348. goog.asserts.assert(mediaState, 'switch: expected mediaState to exist');
  349. if (!mediaState) {
  350. return;
  351. }
  352. if (mediaState.restoreStreamAfterTrickPlay) {
  353. shaka.log.debug('switch during trick play mode', stream);
  354. // Already in trick play mode, so stick with trick mode tracks if
  355. // possible.
  356. if (stream.trickModeVideo) {
  357. // Use the trick mode stream, but revert to the new selection later.
  358. mediaState.restoreStreamAfterTrickPlay = stream;
  359. stream = stream.trickModeVideo;
  360. shaka.log.debug('switch found trick play stream', stream);
  361. } else {
  362. // There is no special trick mode video for this stream!
  363. mediaState.restoreStreamAfterTrickPlay = null;
  364. shaka.log.debug('switch found no special trick play stream');
  365. }
  366. }
  367. if (mediaState.stream == stream && !force) {
  368. const streamTag = shaka.media.StreamingEngine.logPrefix_(mediaState);
  369. shaka.log.debug('switch: Stream ' + streamTag + ' already active');
  370. return;
  371. }
  372. if (stream.type == ContentType.TEXT) {
  373. // Mime types are allowed to change for text streams.
  374. // Reinitialize the text parser, but only if we are going to fetch the
  375. // init segment again.
  376. const fullMimeType = shaka.util.MimeUtils.getFullType(
  377. stream.mimeType, stream.codecs);
  378. this.playerInterface_.mediaSourceEngine.reinitText(
  379. fullMimeType, this.manifest_.sequenceMode);
  380. }
  381. // Releases the segmentIndex of the old stream.
  382. if (mediaState.stream.closeSegmentIndex) {
  383. mediaState.stream.closeSegmentIndex();
  384. }
  385. mediaState.stream = stream;
  386. mediaState.segmentIterator = null;
  387. mediaState.adaptation = !!adaptation;
  388. const streamTag = shaka.media.StreamingEngine.logPrefix_(mediaState);
  389. shaka.log.debug('switch: switching to Stream ' + streamTag);
  390. if (clearBuffer) {
  391. if (mediaState.clearingBuffer) {
  392. // We are already going to clear the buffer, but make sure it is also
  393. // flushed.
  394. mediaState.waitingToFlushBuffer = true;
  395. } else if (mediaState.performingUpdate) {
  396. // We are performing an update, so we have to wait until it's finished.
  397. // onUpdate_() will call clearBuffer_() when the update has finished.
  398. // We need to save the safe margin because its value will be needed when
  399. // clearing the buffer after the update.
  400. mediaState.waitingToClearBuffer = true;
  401. mediaState.clearBufferSafeMargin = safeMargin;
  402. mediaState.waitingToFlushBuffer = true;
  403. } else {
  404. // Cancel the update timer, if any.
  405. this.cancelUpdate_(mediaState);
  406. // Clear right away.
  407. this.clearBuffer_(mediaState, /* flush= */ true, safeMargin)
  408. .catch((error) => {
  409. if (this.playerInterface_) {
  410. goog.asserts.assert(error instanceof shaka.util.Error,
  411. 'Wrong error type!');
  412. this.playerInterface_.onError(error);
  413. }
  414. });
  415. }
  416. }
  417. this.makeAbortDecision_(mediaState).catch((error) => {
  418. if (this.playerInterface_) {
  419. goog.asserts.assert(error instanceof shaka.util.Error,
  420. 'Wrong error type!');
  421. this.playerInterface_.onError(error);
  422. }
  423. });
  424. }
  425. /**
  426. * Decide if it makes sense to abort the current operation, and abort it if
  427. * so.
  428. *
  429. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  430. * @private
  431. */
  432. async makeAbortDecision_(mediaState) {
  433. // If the operation is completed, it will be set to null, and there's no
  434. // need to abort the request.
  435. if (!mediaState.operation) {
  436. return;
  437. }
  438. const originalStream = mediaState.stream;
  439. const originalOperation = mediaState.operation;
  440. if (!originalStream.segmentIndex) {
  441. // Create the new segment index so the time taken is accounted for when
  442. // deciding whether to abort.
  443. await originalStream.createSegmentIndex();
  444. }
  445. if (mediaState.operation != originalOperation) {
  446. // The original operation completed while we were getting a segment index,
  447. // so there's nothing to do now.
  448. return;
  449. }
  450. if (mediaState.stream != originalStream) {
  451. // The stream changed again while we were getting a segment index. We
  452. // can't carry out this check, since another one might be in progress by
  453. // now.
  454. return;
  455. }
  456. goog.asserts.assert(mediaState.stream.segmentIndex,
  457. 'Segment index should exist by now!');
  458. if (this.shouldAbortCurrentRequest_(mediaState)) {
  459. shaka.log.info('Aborting current segment request.');
  460. mediaState.operation.abort();
  461. }
  462. }
  463. /**
  464. * Returns whether we should abort the current request.
  465. *
  466. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  467. * @return {boolean}
  468. * @private
  469. */
  470. shouldAbortCurrentRequest_(mediaState) {
  471. goog.asserts.assert(mediaState.operation,
  472. 'Abort logic requires an ongoing operation!');
  473. goog.asserts.assert(mediaState.stream && mediaState.stream.segmentIndex,
  474. 'Abort logic requires a segment index');
  475. const presentationTime = this.playerInterface_.getPresentationTime();
  476. const bufferEnd =
  477. this.playerInterface_.mediaSourceEngine.bufferEnd(mediaState.type);
  478. // The next segment to append from the current stream. This doesn't
  479. // account for a pending network request and will likely be different from
  480. // that since we just switched.
  481. const timeNeeded = this.getTimeNeeded_(mediaState, presentationTime);
  482. const index = mediaState.stream.segmentIndex.find(timeNeeded);
  483. const newSegment =
  484. index == null ? null : mediaState.stream.segmentIndex.get(index);
  485. let newSegmentSize = newSegment ? newSegment.getSize() : null;
  486. if (newSegment && !newSegmentSize) {
  487. // compute approximate segment size using stream bandwidth
  488. const duration = newSegment.getEndTime() - newSegment.getStartTime();
  489. const bandwidth = mediaState.stream.bandwidth || 0;
  490. // bandwidth is in bits per second, and the size is in bytes
  491. newSegmentSize = duration * bandwidth / 8;
  492. }
  493. if (!newSegmentSize) {
  494. return false;
  495. }
  496. // When switching, we'll need to download the init segment.
  497. const init = newSegment.initSegmentReference;
  498. if (init) {
  499. newSegmentSize += init.getSize() || 0;
  500. }
  501. const bandwidthEstimate = this.playerInterface_.getBandwidthEstimate();
  502. // The estimate is in bits per second, and the size is in bytes. The time
  503. // remaining is in seconds after this calculation.
  504. const timeToFetchNewSegment = (newSegmentSize * 8) / bandwidthEstimate;
  505. // If the new segment can be finished in time without risking a buffer
  506. // underflow, we should abort the old one and switch.
  507. const bufferedAhead = (bufferEnd || 0) - presentationTime;
  508. const safetyBuffer = Math.max(
  509. this.manifest_.minBufferTime || 0,
  510. this.config_.rebufferingGoal);
  511. const safeBufferedAhead = bufferedAhead - safetyBuffer;
  512. if (timeToFetchNewSegment < safeBufferedAhead) {
  513. return true;
  514. }
  515. // If the thing we want to switch to will be done more quickly than what
  516. // we've got in progress, we should abort the old one and switch.
  517. const bytesRemaining = mediaState.operation.getBytesRemaining();
  518. if (bytesRemaining > newSegmentSize) {
  519. return true;
  520. }
  521. // Otherwise, complete the operation in progress.
  522. return false;
  523. }
  524. /**
  525. * Notifies the StreamingEngine that the playhead has moved to a valid time
  526. * within the presentation timeline.
  527. */
  528. seeked() {
  529. const presentationTime = this.playerInterface_.getPresentationTime();
  530. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  531. const newTimeIsBuffered = (type) => {
  532. return this.playerInterface_.mediaSourceEngine.isBuffered(
  533. type, presentationTime);
  534. };
  535. let streamCleared = false;
  536. for (const type of this.mediaStates_.keys()) {
  537. const mediaState = this.mediaStates_.get(type);
  538. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  539. // Always clear the iterator since we need to start streaming from the
  540. // new time. This also happens in clearBuffer_, but if we don't clear,
  541. // we still want to reset the iterator.
  542. mediaState.segmentIterator = null;
  543. if (!newTimeIsBuffered(type)) {
  544. const bufferEnd =
  545. this.playerInterface_.mediaSourceEngine.bufferEnd(type);
  546. const somethingBuffered = bufferEnd != null;
  547. // Don't clear the buffer unless something is buffered. This extra
  548. // check prevents extra, useless calls to clear the buffer.
  549. if (somethingBuffered || mediaState.performingUpdate) {
  550. this.forceClearBuffer_(mediaState);
  551. streamCleared = true;
  552. }
  553. // If there is an operation in progress, stop it now.
  554. if (mediaState.operation) {
  555. mediaState.operation.abort();
  556. shaka.log.debug(logPrefix, 'Aborting operation due to seek');
  557. mediaState.operation = null;
  558. }
  559. // The pts has shifted from the seek, invalidating captions currently
  560. // in the text buffer. Thus, clear and reset the caption parser.
  561. if (type === ContentType.TEXT) {
  562. this.playerInterface_.mediaSourceEngine.resetCaptionParser();
  563. }
  564. // Mark the media state as having seeked, so that the new buffers know
  565. // that they will need to be at a new position (for sequence mode).
  566. mediaState.seeked = true;
  567. }
  568. }
  569. if (!streamCleared) {
  570. shaka.log.debug(
  571. '(all): seeked: buffered seek: presentationTime=' + presentationTime);
  572. }
  573. }
  574. /**
  575. * Clear the buffer for a given stream. Unlike clearBuffer_, this will handle
  576. * cases where a MediaState is performing an update. After this runs, every
  577. * MediaState will have a pending update.
  578. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  579. * @private
  580. */
  581. forceClearBuffer_(mediaState) {
  582. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  583. if (mediaState.clearingBuffer) {
  584. // We're already clearing the buffer, so we don't need to clear the
  585. // buffer again.
  586. shaka.log.debug(logPrefix, 'clear: already clearing the buffer');
  587. return;
  588. }
  589. if (mediaState.waitingToClearBuffer) {
  590. // May not be performing an update, but an update will still happen.
  591. // See: https://github.com/shaka-project/shaka-player/issues/334
  592. shaka.log.debug(logPrefix, 'clear: already waiting');
  593. return;
  594. }
  595. if (mediaState.performingUpdate) {
  596. // We are performing an update, so we have to wait until it's finished.
  597. // onUpdate_() will call clearBuffer_() when the update has finished.
  598. shaka.log.debug(logPrefix, 'clear: currently updating');
  599. mediaState.waitingToClearBuffer = true;
  600. // We can set the offset to zero to remember that this was a call to
  601. // clearAllBuffers.
  602. mediaState.clearBufferSafeMargin = 0;
  603. return;
  604. }
  605. const type = mediaState.type;
  606. if (this.playerInterface_.mediaSourceEngine.bufferStart(type) == null) {
  607. // Nothing buffered.
  608. shaka.log.debug(logPrefix, 'clear: nothing buffered');
  609. if (mediaState.updateTimer == null) {
  610. // Note: an update cycle stops when we buffer to the end of the
  611. // presentation, or when we raise an error.
  612. this.scheduleUpdate_(mediaState, 0);
  613. }
  614. return;
  615. }
  616. // An update may be scheduled, but we can just cancel it and clear the
  617. // buffer right away. Note: clearBuffer_() will schedule the next update.
  618. shaka.log.debug(logPrefix, 'clear: handling right now');
  619. this.cancelUpdate_(mediaState);
  620. this.clearBuffer_(mediaState, /* flush= */ false, 0).catch((error) => {
  621. if (this.playerInterface_) {
  622. goog.asserts.assert(error instanceof shaka.util.Error,
  623. 'Wrong error type!');
  624. this.playerInterface_.onError(error);
  625. }
  626. });
  627. }
  628. /**
  629. * Initializes the initial streams and media states. This will schedule
  630. * updates for the given types.
  631. *
  632. * @return {!Promise}
  633. * @private
  634. */
  635. async initStreams_() {
  636. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  637. goog.asserts.assert(this.config_,
  638. 'StreamingEngine configure() must be called before init()!');
  639. if (!this.currentVariant_) {
  640. shaka.log.error('init: no Streams chosen');
  641. throw new shaka.util.Error(
  642. shaka.util.Error.Severity.CRITICAL,
  643. shaka.util.Error.Category.STREAMING,
  644. shaka.util.Error.Code.STREAMING_ENGINE_STARTUP_INVALID_STATE);
  645. }
  646. /**
  647. * @type {!Map.<shaka.util.ManifestParserUtils.ContentType,
  648. * shaka.extern.Stream>}
  649. */
  650. const streamsByType = new Map();
  651. /** @type {!Set.<shaka.extern.Stream>} */
  652. const streams = new Set();
  653. if (this.currentVariant_.audio) {
  654. streamsByType.set(ContentType.AUDIO, this.currentVariant_.audio);
  655. streams.add(this.currentVariant_.audio);
  656. }
  657. if (this.currentVariant_.video) {
  658. streamsByType.set(ContentType.VIDEO, this.currentVariant_.video);
  659. streams.add(this.currentVariant_.video);
  660. }
  661. if (this.currentTextStream_) {
  662. streamsByType.set(ContentType.TEXT, this.currentTextStream_);
  663. streams.add(this.currentTextStream_);
  664. }
  665. // Init MediaSourceEngine.
  666. const mediaSourceEngine = this.playerInterface_.mediaSourceEngine;
  667. const forceTransmuxTS = this.config_.forceTransmuxTS;
  668. await mediaSourceEngine.init(streamsByType, forceTransmuxTS,
  669. this.manifest_.sequenceMode);
  670. this.destroyer_.ensureNotDestroyed();
  671. this.updateDuration();
  672. for (const type of streamsByType.keys()) {
  673. const stream = streamsByType.get(type);
  674. if (!this.mediaStates_.has(type)) {
  675. const state = this.createMediaState_(stream);
  676. this.mediaStates_.set(type, state);
  677. this.scheduleUpdate_(state, 0);
  678. }
  679. }
  680. }
  681. /**
  682. * Creates a media state.
  683. *
  684. * @param {shaka.extern.Stream} stream
  685. * @return {shaka.media.StreamingEngine.MediaState_}
  686. * @private
  687. */
  688. createMediaState_(stream) {
  689. return /** @type {shaka.media.StreamingEngine.MediaState_} */ ({
  690. stream,
  691. type: stream.type,
  692. segmentIterator: null,
  693. lastSegmentReference: null,
  694. lastInitSegmentReference: null,
  695. lastTimestampOffset: null,
  696. lastAppendWindowStart: null,
  697. lastAppendWindowEnd: null,
  698. restoreStreamAfterTrickPlay: null,
  699. endOfStream: false,
  700. performingUpdate: false,
  701. updateTimer: null,
  702. waitingToClearBuffer: false,
  703. clearBufferSafeMargin: 0,
  704. waitingToFlushBuffer: false,
  705. clearingBuffer: false,
  706. // The playhead might be seeking on startup, if a start time is set, so
  707. // start "seeked" as true.
  708. seeked: true,
  709. recovering: false,
  710. hasError: false,
  711. operation: null,
  712. });
  713. }
  714. /**
  715. * Sets the MediaSource's duration.
  716. */
  717. updateDuration() {
  718. const duration = this.manifest_.presentationTimeline.getDuration();
  719. if (duration < Infinity) {
  720. this.playerInterface_.mediaSourceEngine.setDuration(duration);
  721. } else {
  722. // Not all platforms support infinite durations, so set a finite duration
  723. // so we can append segments and so the user agent can seek.
  724. this.playerInterface_.mediaSourceEngine.setDuration(Math.pow(2, 32));
  725. }
  726. }
  727. /**
  728. * Called when |mediaState|'s update timer has expired.
  729. *
  730. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  731. * @suppress {suspiciousCode} The compiler assumes that updateTimer can't
  732. * change during the await, and so complains about the null check.
  733. * @private
  734. */
  735. async onUpdate_(mediaState) {
  736. this.destroyer_.ensureNotDestroyed();
  737. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  738. // Sanity check.
  739. goog.asserts.assert(
  740. !mediaState.performingUpdate && (mediaState.updateTimer != null),
  741. logPrefix + ' unexpected call to onUpdate_()');
  742. if (mediaState.performingUpdate || (mediaState.updateTimer == null)) {
  743. return;
  744. }
  745. goog.asserts.assert(
  746. !mediaState.clearingBuffer, logPrefix +
  747. ' onUpdate_() should not be called when clearing the buffer');
  748. if (mediaState.clearingBuffer) {
  749. return;
  750. }
  751. mediaState.updateTimer = null;
  752. // Handle pending buffer clears.
  753. if (mediaState.waitingToClearBuffer) {
  754. // Note: clearBuffer_() will schedule the next update.
  755. shaka.log.debug(logPrefix, 'skipping update and clearing the buffer');
  756. await this.clearBuffer_(
  757. mediaState, mediaState.waitingToFlushBuffer,
  758. mediaState.clearBufferSafeMargin);
  759. return;
  760. }
  761. // Make sure the segment index exists. If not, create the segment index.
  762. if (!mediaState.stream.segmentIndex) {
  763. const thisStream = mediaState.stream;
  764. await mediaState.stream.createSegmentIndex();
  765. if (thisStream != mediaState.stream) {
  766. // We switched streams while in the middle of this async call to
  767. // createSegmentIndex. Abandon this update and schedule a new one if
  768. // there's not already one pending.
  769. // Releases the segmentIndex of the old stream.
  770. if (thisStream.closeSegmentIndex) {
  771. goog.asserts.assert(!mediaState.stream.segmentIndex,
  772. 'mediastate.stream should not have segmentIndex yet.');
  773. thisStream.closeSegmentIndex();
  774. }
  775. if (mediaState.updateTimer == null) {
  776. this.scheduleUpdate_(mediaState, 0);
  777. }
  778. return;
  779. }
  780. }
  781. // Update the MediaState.
  782. try {
  783. const delay = this.update_(mediaState);
  784. if (delay != null) {
  785. this.scheduleUpdate_(mediaState, delay);
  786. mediaState.hasError = false;
  787. }
  788. } catch (error) {
  789. await this.handleStreamingError_(error);
  790. return;
  791. }
  792. const mediaStates = Array.from(this.mediaStates_.values());
  793. // Check if we've buffered to the end of the presentation. We delay adding
  794. // the audio and video media states, so it is possible for the text stream
  795. // to be the only state and buffer to the end. So we need to wait until we
  796. // have completed startup to determine if we have reached the end.
  797. if (this.startupComplete_ &&
  798. mediaStates.every((ms) => ms.endOfStream)) {
  799. shaka.log.v1(logPrefix, 'calling endOfStream()...');
  800. await this.playerInterface_.mediaSourceEngine.endOfStream();
  801. this.destroyer_.ensureNotDestroyed();
  802. // If the media segments don't reach the end, then we need to update the
  803. // timeline duration to match the final media duration to avoid
  804. // buffering forever at the end.
  805. // We should only do this if the duration needs to shrink.
  806. // Growing it by less than 1ms can actually cause buffering on
  807. // replay, as in https://github.com/shaka-project/shaka-player/issues/979
  808. // On some platforms, this can spuriously be 0, so ignore this case.
  809. // https://github.com/shaka-project/shaka-player/issues/1967,
  810. const duration = this.playerInterface_.mediaSourceEngine.getDuration();
  811. if (duration != 0 &&
  812. duration < this.manifest_.presentationTimeline.getDuration()) {
  813. this.manifest_.presentationTimeline.setDuration(duration);
  814. }
  815. }
  816. }
  817. /**
  818. * Updates the given MediaState.
  819. *
  820. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  821. * @return {?number} The number of seconds to wait until updating again or
  822. * null if another update does not need to be scheduled.
  823. * @private
  824. */
  825. update_(mediaState) {
  826. goog.asserts.assert(this.manifest_, 'manifest_ should not be null');
  827. goog.asserts.assert(this.config_, 'config_ should not be null');
  828. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  829. // Do not schedule update for closed captions text mediastate, since closed
  830. // captions are embedded in video streams.
  831. if (shaka.media.StreamingEngine.isEmbeddedText_(mediaState)) {
  832. this.playerInterface_.mediaSourceEngine.setSelectedClosedCaptionId(
  833. mediaState.stream.originalId || '');
  834. return null;
  835. } else if (mediaState.type == ContentType.TEXT) {
  836. // Disable embedded captions if not desired (e.g. if transitioning from
  837. // embedded to not-embedded captions).
  838. this.playerInterface_.mediaSourceEngine.clearSelectedClosedCaptionId();
  839. }
  840. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  841. // Compute how far we've buffered ahead of the playhead.
  842. const presentationTime = this.playerInterface_.getPresentationTime();
  843. // Get the next timestamp we need.
  844. const timeNeeded = this.getTimeNeeded_(mediaState, presentationTime);
  845. shaka.log.v2(logPrefix, 'timeNeeded=' + timeNeeded);
  846. // Get the amount of content we have buffered, accounting for drift. This
  847. // is only used to determine if we have meet the buffering goal. This
  848. // should be the same method that PlayheadObserver uses.
  849. const bufferedAhead =
  850. this.playerInterface_.mediaSourceEngine.bufferedAheadOf(
  851. mediaState.type, presentationTime);
  852. shaka.log.v2(logPrefix,
  853. 'update_:',
  854. 'presentationTime=' + presentationTime,
  855. 'bufferedAhead=' + bufferedAhead);
  856. const unscaledBufferingGoal = Math.max(
  857. this.manifest_.minBufferTime || 0,
  858. this.config_.rebufferingGoal,
  859. this.config_.bufferingGoal);
  860. const scaledBufferingGoal =
  861. unscaledBufferingGoal * this.bufferingGoalScale_;
  862. // Check if we've buffered to the end of the presentation.
  863. const timeUntilEnd =
  864. this.manifest_.presentationTimeline.getDuration() - timeNeeded;
  865. const oneMicrosecond = 1e-6;
  866. const bufferEnd =
  867. this.playerInterface_.mediaSourceEngine.bufferEnd(mediaState.type);
  868. if (timeUntilEnd < oneMicrosecond && !!bufferEnd) {
  869. // We shouldn't rebuffer if the playhead is close to the end of the
  870. // presentation.
  871. shaka.log.debug(logPrefix, 'buffered to end of presentation');
  872. mediaState.endOfStream = true;
  873. if (mediaState.type == ContentType.VIDEO) {
  874. // Since the text stream of CEA closed captions doesn't have update
  875. // timer, we have to set the text endOfStream based on the video
  876. // stream's endOfStream state.
  877. const textState = this.mediaStates_.get(ContentType.TEXT);
  878. if (textState &&
  879. shaka.media.StreamingEngine.isEmbeddedText_(textState)) {
  880. textState.endOfStream = true;
  881. }
  882. }
  883. return null;
  884. }
  885. mediaState.endOfStream = false;
  886. // If we've buffered to the buffering goal then schedule an update.
  887. if (bufferedAhead >= scaledBufferingGoal) {
  888. shaka.log.v2(logPrefix, 'buffering goal met');
  889. // Do not try to predict the next update. Just poll according to
  890. // configuration (seconds). The playback rate can change at any time, so
  891. // any prediction we make now could be terribly invalid soon.
  892. return this.config_.updateIntervalSeconds / 2;
  893. }
  894. const reference = this.getSegmentReferenceNeeded_(
  895. mediaState, presentationTime, bufferEnd);
  896. if (!reference) {
  897. // The segment could not be found, does not exist, or is not available.
  898. // In any case just try again... if the manifest is incomplete or is not
  899. // being updated then we'll idle forever; otherwise, we'll end up getting
  900. // a SegmentReference eventually.
  901. return this.config_.updateIntervalSeconds;
  902. }
  903. // Do not let any one stream get far ahead of any other.
  904. let minTimeNeeded = Infinity;
  905. const mediaStates = Array.from(this.mediaStates_.values());
  906. for (const otherState of mediaStates) {
  907. // Do not consider embedded captions in this calculation. It could lead
  908. // to hangs in streaming.
  909. if (shaka.media.StreamingEngine.isEmbeddedText_(otherState)) {
  910. continue;
  911. }
  912. // If there is no next segment, ignore this stream. This happens with
  913. // text when there's a Period with no text in it.
  914. if (otherState.segmentIterator && !otherState.segmentIterator.current()) {
  915. continue;
  916. }
  917. const timeNeeded = this.getTimeNeeded_(otherState, presentationTime);
  918. minTimeNeeded = Math.min(minTimeNeeded, timeNeeded);
  919. }
  920. const maxSegmentDuration =
  921. this.manifest_.presentationTimeline.getMaxSegmentDuration();
  922. const maxRunAhead = maxSegmentDuration *
  923. shaka.media.StreamingEngine.MAX_RUN_AHEAD_SEGMENTS_;
  924. if (timeNeeded >= minTimeNeeded + maxRunAhead) {
  925. // Wait and give other media types time to catch up to this one.
  926. // For example, let video buffering catch up to audio buffering before
  927. // fetching another audio segment.
  928. shaka.log.v2(logPrefix, 'waiting for other streams to buffer');
  929. return this.config_.updateIntervalSeconds;
  930. }
  931. const p = this.fetchAndAppend_(mediaState, presentationTime, reference);
  932. p.catch(() => {}); // TODO(#1993): Handle asynchronous errors.
  933. return null;
  934. }
  935. /**
  936. * Gets the next timestamp needed. Returns the playhead's position if the
  937. * buffer is empty; otherwise, returns the time at which the last segment
  938. * appended ends.
  939. *
  940. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  941. * @param {number} presentationTime
  942. * @return {number} The next timestamp needed.
  943. * @private
  944. */
  945. getTimeNeeded_(mediaState, presentationTime) {
  946. // Get the next timestamp we need. We must use |lastSegmentReference|
  947. // to determine this and not the actual buffer for two reasons:
  948. // 1. Actual segments end slightly before their advertised end times, so
  949. // the next timestamp we need is actually larger than |bufferEnd|.
  950. // 2. There may be drift (the timestamps in the segments are ahead/behind
  951. // of the timestamps in the manifest), but we need drift-free times
  952. // when comparing times against the presentation timeline.
  953. if (!mediaState.lastSegmentReference) {
  954. return presentationTime;
  955. }
  956. return mediaState.lastSegmentReference.endTime;
  957. }
  958. /**
  959. * Gets the SegmentReference of the next segment needed.
  960. *
  961. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  962. * @param {number} presentationTime
  963. * @param {?number} bufferEnd
  964. * @return {shaka.media.SegmentReference} The SegmentReference of the
  965. * next segment needed. Returns null if a segment could not be found, does
  966. * not exist, or is not available.
  967. * @private
  968. */
  969. getSegmentReferenceNeeded_(mediaState, presentationTime, bufferEnd) {
  970. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  971. goog.asserts.assert(
  972. mediaState.stream.segmentIndex,
  973. 'segment index should have been generated already');
  974. if (mediaState.segmentIterator) {
  975. // Something is buffered from the same Stream. Use the current position
  976. // in the segment index. This is updated via next() after each segment is
  977. // appended.
  978. return mediaState.segmentIterator.current();
  979. } else if (mediaState.lastSegmentReference || bufferEnd) {
  980. // Something is buffered from another Stream.
  981. const time = mediaState.lastSegmentReference ?
  982. mediaState.lastSegmentReference.endTime :
  983. bufferEnd;
  984. goog.asserts.assert(time != null, 'Should have a time to search');
  985. shaka.log.v1(
  986. logPrefix, 'looking up segment from new stream endTime:', time);
  987. mediaState.segmentIterator =
  988. mediaState.stream.segmentIndex.getIteratorForTime(time);
  989. const ref = mediaState.segmentIterator &&
  990. mediaState.segmentIterator.next().value;
  991. if (ref == null) {
  992. shaka.log.warning(logPrefix, 'cannot find segment', 'endTime:', time);
  993. }
  994. return ref;
  995. } else {
  996. // Nothing is buffered. Start at the playhead time.
  997. // If there's positive drift then we need to adjust the lookup time, and
  998. // may wind up requesting the previous segment to be safe.
  999. // inaccurateManifestTolerance should be 0 for low latency streaming.
  1000. const inaccurateTolerance = this.config_.inaccurateManifestTolerance;
  1001. const lookupTime = Math.max(presentationTime - inaccurateTolerance, 0);
  1002. shaka.log.v1(logPrefix, 'looking up segment',
  1003. 'lookupTime:', lookupTime,
  1004. 'presentationTime:', presentationTime);
  1005. let ref = null;
  1006. if (inaccurateTolerance) {
  1007. mediaState.segmentIterator =
  1008. mediaState.stream.segmentIndex.getIteratorForTime(lookupTime);
  1009. ref = mediaState.segmentIterator &&
  1010. mediaState.segmentIterator.next().value;
  1011. }
  1012. if (!ref) {
  1013. // If we can't find a valid segment with the drifted time, look for a
  1014. // segment with the presentation time.
  1015. mediaState.segmentIterator =
  1016. mediaState.stream.segmentIndex.getIteratorForTime(presentationTime);
  1017. ref = mediaState.segmentIterator &&
  1018. mediaState.segmentIterator.next().value;
  1019. }
  1020. if (ref == null) {
  1021. shaka.log.warning(logPrefix, 'cannot find segment',
  1022. 'lookupTime:', lookupTime,
  1023. 'presentationTime:', presentationTime);
  1024. }
  1025. return ref;
  1026. }
  1027. }
  1028. /**
  1029. * Fetches and appends the given segment. Sets up the given MediaState's
  1030. * associated SourceBuffer and evicts segments if either are required
  1031. * beforehand. Schedules another update after completing successfully.
  1032. *
  1033. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  1034. * @param {number} presentationTime
  1035. * @param {!shaka.media.SegmentReference} reference
  1036. * @private
  1037. */
  1038. async fetchAndAppend_(mediaState, presentationTime, reference) {
  1039. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  1040. const StreamingEngine = shaka.media.StreamingEngine;
  1041. const logPrefix = StreamingEngine.logPrefix_(mediaState);
  1042. shaka.log.v1(logPrefix,
  1043. 'fetchAndAppend_:',
  1044. 'presentationTime=' + presentationTime,
  1045. 'reference.startTime=' + reference.startTime,
  1046. 'reference.endTime=' + reference.endTime);
  1047. // Subtlety: The playhead may move while asynchronous update operations are
  1048. // in progress, so we should avoid calling playhead.getTime() in any
  1049. // callbacks. Furthermore, switch() or seeked() may be called at any time,
  1050. // so we store the old iterator. This allows the mediaState to change and
  1051. // we'll update the old iterator.
  1052. const stream = mediaState.stream;
  1053. const iter = mediaState.segmentIterator;
  1054. mediaState.performingUpdate = true;
  1055. try {
  1056. if (reference.getStatus() ==
  1057. shaka.media.SegmentReference.Status.MISSING) {
  1058. throw new shaka.util.Error(
  1059. shaka.util.Error.Severity.RECOVERABLE,
  1060. shaka.util.Error.Category.NETWORK,
  1061. shaka.util.Error.Code.SEGMENT_MISSING);
  1062. }
  1063. await this.initSourceBuffer_(mediaState, reference);
  1064. this.destroyer_.ensureNotDestroyed();
  1065. if (this.fatalError_) {
  1066. return;
  1067. }
  1068. shaka.log.v2(logPrefix, 'fetching segment');
  1069. const isMP4 = stream.mimeType == 'video/mp4' ||
  1070. stream.mimeType == 'audio/mp4';
  1071. const isReadableStreamSupported = window.ReadableStream;
  1072. // Enable MP4 low latency streaming with ReadableStream chunked data.
  1073. if (this.config_.lowLatencyMode && isReadableStreamSupported && isMP4) {
  1074. let remaining = new Uint8Array(0);
  1075. let processingResult = false;
  1076. let callbackCalled = false;
  1077. const streamDataCallback = async (data) => {
  1078. if (processingResult) {
  1079. // If the fallback result processing was triggered, don't also
  1080. // append the buffer here. In theory this should never happen,
  1081. // but it does on some older TVs.
  1082. return;
  1083. }
  1084. callbackCalled = true;
  1085. this.destroyer_.ensureNotDestroyed();
  1086. if (this.fatalError_) {
  1087. return;
  1088. }
  1089. // Append the data with complete boxes.
  1090. // Every time streamDataCallback gets called, append the new data to
  1091. // the remaining data.
  1092. // Find the last fully completed Mdat box, and slice the data into two
  1093. // parts: the first part with completed Mdat boxes, and the second
  1094. // part with an incomplete box.
  1095. // Append the first part, and save the second part as remaining data,
  1096. // and handle it with the next streamDataCallback call.
  1097. remaining = this.concatArray_(remaining, data);
  1098. let sawMDAT = false;
  1099. let offset = 0;
  1100. new shaka.util.Mp4Parser()
  1101. .box('mdat', (box) => {
  1102. offset = box.size + box.start;
  1103. sawMDAT = true;
  1104. })
  1105. .parse(remaining, /* partialOkay= */ false,
  1106. /* isChunkedData= */ true);
  1107. if (sawMDAT) {
  1108. const dataToAppend = remaining.subarray(0, offset);
  1109. remaining = remaining.subarray(offset);
  1110. await this.append_(
  1111. mediaState, presentationTime, stream, reference, dataToAppend);
  1112. }
  1113. };
  1114. const result =
  1115. await this.fetch_(mediaState, reference, streamDataCallback);
  1116. if (!callbackCalled) {
  1117. // In some environments, we might be forced to use network plugins
  1118. // that don't support streamDataCallback. In those cases, as a
  1119. // fallback, append the buffer here.
  1120. processingResult = true;
  1121. this.destroyer_.ensureNotDestroyed();
  1122. if (this.fatalError_) {
  1123. return;
  1124. }
  1125. // If the text stream gets switched between fetch_() and append_(),
  1126. // the new text parser is initialized, but the new init segment is
  1127. // not fetched yet. That would cause an error in
  1128. // TextParser.parseMedia().
  1129. // See http://b/168253400
  1130. if (mediaState.waitingToClearBuffer) {
  1131. shaka.log.info(logPrefix, 'waitingToClearBuffer, skip append');
  1132. mediaState.performingUpdate = false;
  1133. this.scheduleUpdate_(mediaState, 0);
  1134. return;
  1135. }
  1136. await this.append_(
  1137. mediaState, presentationTime, stream, reference, result);
  1138. }
  1139. } else {
  1140. if (this.config_.lowLatencyMode && !isReadableStreamSupported) {
  1141. shaka.log.warning('Low latency streaming mode is enabled, but ' +
  1142. 'ReadableStream is not supported by the browser.');
  1143. }
  1144. const fetchSegment = this.fetch_(mediaState, reference);
  1145. const result = await fetchSegment;
  1146. this.destroyer_.ensureNotDestroyed();
  1147. if (this.fatalError_) {
  1148. return;
  1149. }
  1150. // If the text stream gets switched between fetch_() and append_(), the
  1151. // new text parser is initialized, but the new init segment is not
  1152. // fetched yet. That would cause an error in TextParser.parseMedia().
  1153. // See http://b/168253400
  1154. if (mediaState.waitingToClearBuffer) {
  1155. shaka.log.info(logPrefix, 'waitingToClearBuffer, skip append');
  1156. mediaState.performingUpdate = false;
  1157. this.scheduleUpdate_(mediaState, 0);
  1158. return;
  1159. }
  1160. await this.append_(
  1161. mediaState, presentationTime, stream, reference, result);
  1162. }
  1163. this.destroyer_.ensureNotDestroyed();
  1164. if (this.fatalError_) {
  1165. return;
  1166. }
  1167. // move to next segment after appending the current segment.
  1168. mediaState.lastSegmentReference = reference;
  1169. const newRef = iter.next().value;
  1170. shaka.log.v2(logPrefix, 'advancing to next segment', newRef);
  1171. mediaState.performingUpdate = false;
  1172. mediaState.recovering = false;
  1173. const info = this.playerInterface_.mediaSourceEngine.getBufferedInfo();
  1174. const buffered = info[mediaState.type];
  1175. // Convert the buffered object to a string capture its properties on
  1176. // WebOS.
  1177. shaka.log.v1(logPrefix, 'finished fetch and append',
  1178. JSON.stringify(buffered));
  1179. if (!mediaState.waitingToClearBuffer) {
  1180. this.playerInterface_.onSegmentAppended();
  1181. }
  1182. // Update right away.
  1183. this.scheduleUpdate_(mediaState, 0);
  1184. } catch (error) {
  1185. this.destroyer_.ensureNotDestroyed(error);
  1186. if (this.fatalError_) {
  1187. return;
  1188. }
  1189. goog.asserts.assert(error instanceof shaka.util.Error,
  1190. 'Should only receive a Shaka error');
  1191. mediaState.performingUpdate = false;
  1192. if (error.code == shaka.util.Error.Code.OPERATION_ABORTED) {
  1193. // If the network slows down, abort the current fetch request and start
  1194. // a new one, and ignore the error message.
  1195. mediaState.performingUpdate = false;
  1196. mediaState.updateTimer = null;
  1197. this.scheduleUpdate_(mediaState, 0);
  1198. } else if (mediaState.type == ContentType.TEXT &&
  1199. this.config_.ignoreTextStreamFailures) {
  1200. if (error.code == shaka.util.Error.Code.BAD_HTTP_STATUS) {
  1201. shaka.log.warning(logPrefix,
  1202. 'Text stream failed to download. Proceeding without it.');
  1203. } else {
  1204. shaka.log.warning(logPrefix,
  1205. 'Text stream failed to parse. Proceeding without it.');
  1206. }
  1207. this.mediaStates_.delete(ContentType.TEXT);
  1208. } else if (error.code == shaka.util.Error.Code.QUOTA_EXCEEDED_ERROR) {
  1209. this.handleQuotaExceeded_(mediaState, error);
  1210. } else {
  1211. shaka.log.error(logPrefix, 'failed fetch and append: code=' +
  1212. error.code);
  1213. mediaState.hasError = true;
  1214. error.severity = shaka.util.Error.Severity.CRITICAL;
  1215. await this.handleStreamingError_(error);
  1216. }
  1217. }
  1218. }
  1219. /**
  1220. * Clear per-stream error states and retry any failed streams.
  1221. * @param {number} delaySeconds
  1222. * @return {boolean} False if unable to retry.
  1223. */
  1224. retry(delaySeconds) {
  1225. if (this.destroyer_.destroyed()) {
  1226. shaka.log.error('Unable to retry after StreamingEngine is destroyed!');
  1227. return false;
  1228. }
  1229. if (this.fatalError_) {
  1230. shaka.log.error('Unable to retry after StreamingEngine encountered a ' +
  1231. 'fatal error!');
  1232. return false;
  1233. }
  1234. for (const mediaState of this.mediaStates_.values()) {
  1235. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1236. if (mediaState.hasError) {
  1237. shaka.log.info(logPrefix, 'Retrying after failure...');
  1238. mediaState.hasError = false;
  1239. this.scheduleUpdate_(mediaState, delaySeconds);
  1240. }
  1241. }
  1242. return true;
  1243. }
  1244. /**
  1245. * Append the data to the remaining data.
  1246. * @param {!Uint8Array} remaining
  1247. * @param {!Uint8Array} data
  1248. * @return {!Uint8Array}
  1249. * @private
  1250. */
  1251. concatArray_(remaining, data) {
  1252. const result = new Uint8Array(remaining.length + data.length);
  1253. result.set(remaining);
  1254. result.set(data, remaining.length);
  1255. return result;
  1256. }
  1257. /**
  1258. * Handles a QUOTA_EXCEEDED_ERROR.
  1259. *
  1260. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1261. * @param {!shaka.util.Error} error
  1262. * @private
  1263. */
  1264. handleQuotaExceeded_(mediaState, error) {
  1265. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1266. // The segment cannot fit into the SourceBuffer. Ideally, MediaSource would
  1267. // have evicted old data to accommodate the segment; however, it may have
  1268. // failed to do this if the segment is very large, or if it could not find
  1269. // a suitable time range to remove.
  1270. //
  1271. // We can overcome the latter by trying to append the segment again;
  1272. // however, to avoid continuous QuotaExceededErrors we must reduce the size
  1273. // of the buffer going forward.
  1274. //
  1275. // If we've recently reduced the buffering goals, wait until the stream
  1276. // which caused the first QuotaExceededError recovers. Doing this ensures
  1277. // we don't reduce the buffering goals too quickly.
  1278. const mediaStates = Array.from(this.mediaStates_.values());
  1279. const waitingForAnotherStreamToRecover = mediaStates.some((ms) => {
  1280. return ms != mediaState && ms.recovering;
  1281. });
  1282. if (!waitingForAnotherStreamToRecover) {
  1283. // Reduction schedule: 80%, 60%, 40%, 20%, 16%, 12%, 8%, 4%, fail.
  1284. // Note: percentages are used for comparisons to avoid rounding errors.
  1285. const percentBefore = Math.round(100 * this.bufferingGoalScale_);
  1286. if (percentBefore > 20) {
  1287. this.bufferingGoalScale_ -= 0.2;
  1288. } else if (percentBefore > 4) {
  1289. this.bufferingGoalScale_ -= 0.04;
  1290. } else {
  1291. shaka.log.error(
  1292. logPrefix, 'MediaSource threw QuotaExceededError too many times');
  1293. mediaState.hasError = true;
  1294. this.fatalError_ = true;
  1295. this.playerInterface_.onError(error);
  1296. return;
  1297. }
  1298. const percentAfter = Math.round(100 * this.bufferingGoalScale_);
  1299. shaka.log.warning(
  1300. logPrefix,
  1301. 'MediaSource threw QuotaExceededError:',
  1302. 'reducing buffering goals by ' + (100 - percentAfter) + '%');
  1303. mediaState.recovering = true;
  1304. } else {
  1305. shaka.log.debug(
  1306. logPrefix,
  1307. 'MediaSource threw QuotaExceededError:',
  1308. 'waiting for another stream to recover...');
  1309. }
  1310. // QuotaExceededError gets thrown if evication didn't help to make room
  1311. // for a segment. We want to wait for a while (4 seconds is just an
  1312. // arbitrary number) before updating to give the playhead a chance to
  1313. // advance, so we don't immidiately throw again.
  1314. this.scheduleUpdate_(mediaState, 4);
  1315. }
  1316. /**
  1317. * Sets the given MediaState's associated SourceBuffer's timestamp offset,
  1318. * append window, and init segment if they have changed. If an error occurs
  1319. * then neither the timestamp offset or init segment are unset, since another
  1320. * call to switch() will end up superseding them.
  1321. *
  1322. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1323. * @param {!shaka.media.SegmentReference} reference
  1324. * @return {!Promise}
  1325. * @private
  1326. */
  1327. async initSourceBuffer_(mediaState, reference) {
  1328. const StreamingEngine = shaka.media.StreamingEngine;
  1329. const logPrefix = StreamingEngine.logPrefix_(mediaState);
  1330. /** @type {!Array.<!Promise>} */
  1331. const operations = [];
  1332. // Rounding issues can cause us to remove the first frame of a Period, so
  1333. // reduce the window start time slightly.
  1334. const appendWindowStart = Math.max(0,
  1335. reference.appendWindowStart -
  1336. StreamingEngine.APPEND_WINDOW_START_FUDGE_);
  1337. const appendWindowEnd =
  1338. reference.appendWindowEnd + StreamingEngine.APPEND_WINDOW_END_FUDGE_;
  1339. goog.asserts.assert(
  1340. reference.startTime <= appendWindowEnd,
  1341. logPrefix + ' segment should start before append window end');
  1342. const timestampOffset = reference.timestampOffset;
  1343. if (timestampOffset != mediaState.lastTimestampOffset ||
  1344. appendWindowStart != mediaState.lastAppendWindowStart ||
  1345. appendWindowEnd != mediaState.lastAppendWindowEnd) {
  1346. shaka.log.v1(logPrefix, 'setting timestamp offset to ' + timestampOffset);
  1347. shaka.log.v1(logPrefix,
  1348. 'setting append window start to ' + appendWindowStart);
  1349. shaka.log.v1(logPrefix,
  1350. 'setting append window end to ' + appendWindowEnd);
  1351. const setProperties = async () => {
  1352. try {
  1353. mediaState.lastAppendWindowStart = appendWindowStart;
  1354. mediaState.lastAppendWindowEnd = appendWindowEnd;
  1355. mediaState.lastTimestampOffset = timestampOffset;
  1356. await this.playerInterface_.mediaSourceEngine.setStreamProperties(
  1357. mediaState.type, timestampOffset, appendWindowStart,
  1358. appendWindowEnd, this.manifest_.sequenceMode);
  1359. } catch (error) {
  1360. mediaState.lastAppendWindowStart = null;
  1361. mediaState.lastAppendWindowEnd = null;
  1362. mediaState.lastTimestampOffset = null;
  1363. throw error;
  1364. }
  1365. };
  1366. operations.push(setProperties());
  1367. }
  1368. if (!shaka.media.InitSegmentReference.equal(
  1369. reference.initSegmentReference, mediaState.lastInitSegmentReference)) {
  1370. mediaState.lastInitSegmentReference = reference.initSegmentReference;
  1371. if (reference.initSegmentReference) {
  1372. shaka.log.v1(logPrefix, 'fetching init segment');
  1373. const fetchInit =
  1374. this.fetch_(mediaState, reference.initSegmentReference);
  1375. const append = async () => {
  1376. try {
  1377. const initSegment = await fetchInit;
  1378. this.destroyer_.ensureNotDestroyed();
  1379. shaka.log.v1(logPrefix, 'appending init segment');
  1380. const hasClosedCaptions = mediaState.stream.closedCaptions &&
  1381. mediaState.stream.closedCaptions.size > 0;
  1382. await this.playerInterface_.beforeAppendSegment(
  1383. mediaState.type, initSegment);
  1384. await this.playerInterface_.mediaSourceEngine.appendBuffer(
  1385. mediaState.type, initSegment, /* reference= */ null,
  1386. hasClosedCaptions);
  1387. } catch (error) {
  1388. mediaState.lastInitSegmentReference = null;
  1389. throw error;
  1390. }
  1391. };
  1392. this.playerInterface_.onInitSegmentAppended(
  1393. reference.startTime, reference.initSegmentReference);
  1394. operations.push(append());
  1395. }
  1396. }
  1397. await Promise.all(operations);
  1398. }
  1399. /**
  1400. * Appends the given segment and evicts content if required to append.
  1401. *
  1402. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  1403. * @param {number} presentationTime
  1404. * @param {shaka.extern.Stream} stream
  1405. * @param {!shaka.media.SegmentReference} reference
  1406. * @param {BufferSource} segment
  1407. * @return {!Promise}
  1408. * @private
  1409. */
  1410. async append_(mediaState, presentationTime, stream, reference, segment) {
  1411. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1412. const hasClosedCaptions = stream.closedCaptions &&
  1413. stream.closedCaptions.size > 0;
  1414. if ((stream.emsgSchemeIdUris != null &&
  1415. stream.emsgSchemeIdUris.length > 0) ||
  1416. this.config_.dispatchAllEmsgBoxes) {
  1417. new shaka.util.Mp4Parser()
  1418. .fullBox(
  1419. 'emsg',
  1420. (box) => this.parseEMSG_(
  1421. reference, stream.emsgSchemeIdUris, box))
  1422. .parse(segment);
  1423. }
  1424. await this.evict_(mediaState, presentationTime);
  1425. this.destroyer_.ensureNotDestroyed();
  1426. shaka.log.v1(logPrefix, 'appending media segment at',
  1427. (reference.syncTime == null ? 'unknown' : reference.syncTime));
  1428. // 'seeked' or 'adaptation' triggered logic applies only to this
  1429. // appendBuffer() call.
  1430. const seeked = mediaState.seeked;
  1431. mediaState.seeked = false;
  1432. const adaptation = mediaState.adaptation;
  1433. mediaState.adaptation = false;
  1434. await this.playerInterface_.beforeAppendSegment(mediaState.type, segment);
  1435. await this.playerInterface_.mediaSourceEngine.appendBuffer(
  1436. mediaState.type,
  1437. segment,
  1438. reference,
  1439. hasClosedCaptions,
  1440. seeked,
  1441. adaptation);
  1442. this.destroyer_.ensureNotDestroyed();
  1443. shaka.log.v2(logPrefix, 'appended media segment');
  1444. }
  1445. /**
  1446. * Parse the EMSG box from a MP4 container.
  1447. *
  1448. * @param {!shaka.media.SegmentReference} reference
  1449. * @param {?Array.<string>} emsgSchemeIdUris Array of emsg
  1450. * scheme_id_uri for which emsg boxes should be parsed.
  1451. * @param {!shaka.extern.ParsedBox} box
  1452. * @private
  1453. * https://dashif-documents.azurewebsites.net/Events/master/event.html#emsg-format
  1454. * aligned(8) class DASHEventMessageBox
  1455. * extends FullBox(‘emsg’, version, flags = 0){
  1456. * if (version==0) {
  1457. * string scheme_id_uri;
  1458. * string value;
  1459. * unsigned int(32) timescale;
  1460. * unsigned int(32) presentation_time_delta;
  1461. * unsigned int(32) event_duration;
  1462. * unsigned int(32) id;
  1463. * } else if (version==1) {
  1464. * unsigned int(32) timescale;
  1465. * unsigned int(64) presentation_time;
  1466. * unsigned int(32) event_duration;
  1467. * unsigned int(32) id;
  1468. * string scheme_id_uri;
  1469. * string value;
  1470. * }
  1471. * unsigned int(8) message_data[];
  1472. */
  1473. parseEMSG_(reference, emsgSchemeIdUris, box) {
  1474. let timescale;
  1475. let id;
  1476. let eventDuration;
  1477. let schemeId;
  1478. let startTime;
  1479. let presentationTimeDelta;
  1480. let value;
  1481. if (box.version === 0) {
  1482. schemeId = box.reader.readTerminatedString();
  1483. value = box.reader.readTerminatedString();
  1484. timescale = box.reader.readUint32();
  1485. presentationTimeDelta = box.reader.readUint32();
  1486. eventDuration = box.reader.readUint32();
  1487. id = box.reader.readUint32();
  1488. startTime = reference.startTime + (presentationTimeDelta / timescale);
  1489. } else {
  1490. timescale = box.reader.readUint32();
  1491. const pts = box.reader.readUint64();
  1492. startTime = (pts / timescale) + reference.timestampOffset;
  1493. presentationTimeDelta = startTime - reference.startTime;
  1494. eventDuration = box.reader.readUint32();
  1495. id = box.reader.readUint32();
  1496. schemeId = box.reader.readTerminatedString();
  1497. value = box.reader.readTerminatedString();
  1498. }
  1499. const messageData = box.reader.readBytes(
  1500. box.reader.getLength() - box.reader.getPosition());
  1501. // See DASH sec. 5.10.3.3.1
  1502. // If a DASH client detects an event message box with a scheme that is not
  1503. // defined in MPD, the client is expected to ignore it.
  1504. if ((emsgSchemeIdUris && emsgSchemeIdUris.includes(schemeId)) ||
  1505. this.config_.dispatchAllEmsgBoxes) {
  1506. // See DASH sec. 5.10.4.1
  1507. // A special scheme in DASH used to signal manifest updates.
  1508. if (schemeId == 'urn:mpeg:dash:event:2012') {
  1509. this.playerInterface_.onManifestUpdate();
  1510. } else {
  1511. /** @type {shaka.extern.EmsgInfo} */
  1512. const emsg = {
  1513. startTime: startTime,
  1514. endTime: startTime + (eventDuration / timescale),
  1515. schemeIdUri: schemeId,
  1516. value: value,
  1517. timescale: timescale,
  1518. presentationTimeDelta: presentationTimeDelta,
  1519. eventDuration: eventDuration,
  1520. id: id,
  1521. messageData: messageData,
  1522. };
  1523. // Dispatch an event to notify the application about the emsg box.
  1524. const eventName = shaka.util.FakeEvent.EventName.Emsg;
  1525. const data = (new Map()).set('detail', emsg);
  1526. const event = new shaka.util.FakeEvent(eventName, data);
  1527. this.playerInterface_.onEvent(event);
  1528. }
  1529. }
  1530. }
  1531. /**
  1532. * Evicts media to meet the max buffer behind limit.
  1533. *
  1534. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1535. * @param {number} presentationTime
  1536. * @private
  1537. */
  1538. async evict_(mediaState, presentationTime) {
  1539. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1540. shaka.log.v2(logPrefix, 'checking buffer length');
  1541. // Use the max segment duration, if it is longer than the bufferBehind, to
  1542. // avoid accidentally clearing too much data when dealing with a manifest
  1543. // with a long keyframe interval.
  1544. const bufferBehind = Math.max(this.config_.bufferBehind,
  1545. this.manifest_.presentationTimeline.getMaxSegmentDuration());
  1546. const startTime =
  1547. this.playerInterface_.mediaSourceEngine.bufferStart(mediaState.type);
  1548. if (startTime == null) {
  1549. shaka.log.v2(logPrefix,
  1550. 'buffer behind okay because nothing buffered:',
  1551. 'presentationTime=' + presentationTime,
  1552. 'bufferBehind=' + bufferBehind);
  1553. return;
  1554. }
  1555. const bufferedBehind = presentationTime - startTime;
  1556. const overflow = bufferedBehind - bufferBehind;
  1557. // See: https://github.com/shaka-project/shaka-player/issues/2982
  1558. if (overflow <= 0.01) {
  1559. shaka.log.v2(logPrefix,
  1560. 'buffer behind okay:',
  1561. 'presentationTime=' + presentationTime,
  1562. 'bufferedBehind=' + bufferedBehind,
  1563. 'bufferBehind=' + bufferBehind,
  1564. 'underflow=' + Math.abs(overflow));
  1565. return;
  1566. }
  1567. shaka.log.v1(logPrefix,
  1568. 'buffer behind too large:',
  1569. 'presentationTime=' + presentationTime,
  1570. 'bufferedBehind=' + bufferedBehind,
  1571. 'bufferBehind=' + bufferBehind,
  1572. 'overflow=' + overflow);
  1573. await this.playerInterface_.mediaSourceEngine.remove(mediaState.type,
  1574. startTime, startTime + overflow);
  1575. this.destroyer_.ensureNotDestroyed();
  1576. shaka.log.v1(logPrefix, 'evicted ' + overflow + ' seconds');
  1577. }
  1578. /**
  1579. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1580. * @return {boolean}
  1581. * @private
  1582. */
  1583. static isEmbeddedText_(mediaState) {
  1584. const MimeUtils = shaka.util.MimeUtils;
  1585. const CEA608_MIME = MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE;
  1586. const CEA708_MIME = MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE;
  1587. return mediaState &&
  1588. mediaState.type == shaka.util.ManifestParserUtils.ContentType.TEXT &&
  1589. (mediaState.stream.mimeType == CEA608_MIME ||
  1590. mediaState.stream.mimeType == CEA708_MIME);
  1591. }
  1592. /**
  1593. * Fetches the given segment.
  1594. *
  1595. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  1596. * @param {(!shaka.media.InitSegmentReference|!shaka.media.SegmentReference)}
  1597. * reference
  1598. * @param {?function(BufferSource):!Promise=} streamDataCallback
  1599. *
  1600. * @return {!Promise.<BufferSource>}
  1601. * @private
  1602. * @suppress {strictMissingProperties}
  1603. */
  1604. async fetch_(mediaState, reference, streamDataCallback) {
  1605. const requestType = shaka.net.NetworkingEngine.RequestType.SEGMENT;
  1606. const request = shaka.util.Networking.createSegmentRequest(
  1607. reference.getUris(),
  1608. reference.startByte,
  1609. reference.endByte,
  1610. this.config_.retryParameters,
  1611. streamDataCallback);
  1612. shaka.log.v2('fetching: reference=', reference);
  1613. const stream = mediaState.stream;
  1614. this.playerInterface_.modifySegmentRequest(
  1615. request,
  1616. {
  1617. type: stream.type,
  1618. init: reference instanceof shaka.media.InitSegmentReference,
  1619. duration: reference.endTime - reference.startTime,
  1620. mimeType: stream.mimeType,
  1621. codecs: stream.codecs,
  1622. bandwidth: stream.bandwidth,
  1623. },
  1624. );
  1625. const op = this.playerInterface_.netEngine.request(requestType, request);
  1626. mediaState.operation = op;
  1627. const response = await op.promise;
  1628. mediaState.operation = null;
  1629. return response.data;
  1630. }
  1631. /**
  1632. * Clears the buffer and schedules another update.
  1633. * The optional parameter safeMargin allows to retain a certain amount
  1634. * of buffer, which can help avoiding rebuffering events.
  1635. * The value of the safe margin should be provided by the ABR manager.
  1636. *
  1637. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  1638. * @param {boolean} flush
  1639. * @param {number} safeMargin
  1640. * @private
  1641. */
  1642. async clearBuffer_(mediaState, flush, safeMargin) {
  1643. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1644. goog.asserts.assert(
  1645. !mediaState.performingUpdate && (mediaState.updateTimer == null),
  1646. logPrefix + ' unexpected call to clearBuffer_()');
  1647. mediaState.waitingToClearBuffer = false;
  1648. mediaState.waitingToFlushBuffer = false;
  1649. mediaState.clearBufferSafeMargin = 0;
  1650. mediaState.clearingBuffer = true;
  1651. mediaState.lastSegmentReference = null;
  1652. mediaState.lastInitSegmentReference = null;
  1653. mediaState.segmentIterator = null;
  1654. shaka.log.debug(logPrefix, 'clearing buffer');
  1655. if (safeMargin) {
  1656. const presentationTime = this.playerInterface_.getPresentationTime();
  1657. const duration = this.playerInterface_.mediaSourceEngine.getDuration();
  1658. await this.playerInterface_.mediaSourceEngine.remove(
  1659. mediaState.type, presentationTime + safeMargin, duration);
  1660. } else {
  1661. await this.playerInterface_.mediaSourceEngine.clear(mediaState.type);
  1662. this.destroyer_.ensureNotDestroyed();
  1663. if (flush) {
  1664. await this.playerInterface_.mediaSourceEngine.flush(
  1665. mediaState.type);
  1666. }
  1667. }
  1668. this.destroyer_.ensureNotDestroyed();
  1669. shaka.log.debug(logPrefix, 'cleared buffer');
  1670. mediaState.clearingBuffer = false;
  1671. mediaState.endOfStream = false;
  1672. this.scheduleUpdate_(mediaState, 0);
  1673. }
  1674. /**
  1675. * Schedules |mediaState|'s next update.
  1676. *
  1677. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
  1678. * @param {number} delay The delay in seconds.
  1679. * @private
  1680. */
  1681. scheduleUpdate_(mediaState, delay) {
  1682. const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
  1683. // If the text's update is canceled and its mediaState is deleted, stop
  1684. // scheduling another update.
  1685. const type = mediaState.type;
  1686. if (type == shaka.util.ManifestParserUtils.ContentType.TEXT &&
  1687. !this.mediaStates_.has(type)) {
  1688. shaka.log.v1(logPrefix, 'Text stream is unloaded. No update is needed.');
  1689. return;
  1690. }
  1691. shaka.log.v2(logPrefix, 'updating in ' + delay + ' seconds');
  1692. goog.asserts.assert(mediaState.updateTimer == null,
  1693. logPrefix + ' did not expect update to be scheduled');
  1694. mediaState.updateTimer = new shaka.util.DelayedTick(async () => {
  1695. try {
  1696. await this.onUpdate_(mediaState);
  1697. } catch (error) {
  1698. if (this.playerInterface_) {
  1699. this.playerInterface_.onError(error);
  1700. }
  1701. }
  1702. }).tickAfter(delay);
  1703. }
  1704. /**
  1705. * If |mediaState| is scheduled to update, stop it.
  1706. *
  1707. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1708. * @private
  1709. */
  1710. cancelUpdate_(mediaState) {
  1711. if (mediaState.updateTimer == null) {
  1712. return;
  1713. }
  1714. mediaState.updateTimer.stop();
  1715. mediaState.updateTimer = null;
  1716. }
  1717. /**
  1718. * If |mediaState| holds any in-progress operations, abort them.
  1719. *
  1720. * @return {!Promise}
  1721. * @private
  1722. */
  1723. async abortOperations_(mediaState) {
  1724. if (mediaState.operation) {
  1725. await mediaState.operation.abort();
  1726. }
  1727. }
  1728. /**
  1729. * Handle streaming errors by delaying, then notifying the application by
  1730. * error callback and by streaming failure callback.
  1731. *
  1732. * @param {!shaka.util.Error} error
  1733. * @return {!Promise}
  1734. * @private
  1735. */
  1736. async handleStreamingError_(error) {
  1737. // If we invoke the callback right away, the application could trigger a
  1738. // rapid retry cycle that could be very unkind to the server. Instead,
  1739. // use the backoff system to delay and backoff the error handling.
  1740. await this.failureCallbackBackoff_.attempt();
  1741. this.destroyer_.ensureNotDestroyed();
  1742. // First fire an error event.
  1743. this.playerInterface_.onError(error);
  1744. // If the error was not handled by the application, call the failure
  1745. // callback.
  1746. if (!error.handled) {
  1747. this.config_.failureCallback(error);
  1748. }
  1749. }
  1750. /**
  1751. * @param {shaka.media.StreamingEngine.MediaState_} mediaState
  1752. * @return {string} A log prefix of the form ($CONTENT_TYPE:$STREAM_ID), e.g.,
  1753. * "(audio:5)" or "(video:hd)".
  1754. * @private
  1755. */
  1756. static logPrefix_(mediaState) {
  1757. return '(' + mediaState.type + ':' + mediaState.stream.id + ')';
  1758. }
  1759. };
  1760. /**
  1761. * @typedef {{
  1762. * getPresentationTime: function():number,
  1763. * getBandwidthEstimate: function():number,
  1764. * modifySegmentRequest: function(shaka.extern.Request,
  1765. * shaka.util.CmcdManager.SegmentInfo),
  1766. * mediaSourceEngine: !shaka.media.MediaSourceEngine,
  1767. * netEngine: shaka.net.NetworkingEngine,
  1768. * onError: function(!shaka.util.Error),
  1769. * onEvent: function(!Event),
  1770. * onManifestUpdate: function(),
  1771. * onSegmentAppended: function(),
  1772. * onInitSegmentAppended: function(!number,!shaka.media.InitSegmentReference),
  1773. * beforeAppendSegment: function(
  1774. * shaka.util.ManifestParserUtils.ContentType,!BufferSource):Promise
  1775. * }}
  1776. *
  1777. * @property {function():number} getPresentationTime
  1778. * Get the position in the presentation (in seconds) of the content that the
  1779. * viewer is seeing on screen right now.
  1780. * @property {function():number} getBandwidthEstimate
  1781. * Get the estimated bandwidth in bits per second.
  1782. * @property {function(shaka.extern.Request,
  1783. * shaka.extern.Cmcd.SegmentInfo)} modifySegmentRequest
  1784. * The request modifier
  1785. * @property {!shaka.media.MediaSourceEngine} mediaSourceEngine
  1786. * The MediaSourceEngine. The caller retains ownership.
  1787. * @property {shaka.net.NetworkingEngine} netEngine
  1788. * The NetworkingEngine instance to use. The caller retains ownership.
  1789. * @property {function(!shaka.util.Error)} onError
  1790. * Called when an error occurs. If the error is recoverable (see
  1791. * {@link shaka.util.Error}) then the caller may invoke either
  1792. * StreamingEngine.switch*() or StreamingEngine.seeked() to attempt recovery.
  1793. * @property {function(!Event)} onEvent
  1794. * Called when an event occurs that should be sent to the app.
  1795. * @property {function()} onManifestUpdate
  1796. * Called when an embedded 'emsg' box should trigger a manifest update.
  1797. * @property {function()} onSegmentAppended
  1798. * Called after a segment is successfully appended to a MediaSource.
  1799. * @property
  1800. * {function(!number, !shaka.media.InitSegmentReference)} onInitSegmentAppended
  1801. * Called when an init segment is appended to a MediaSource.
  1802. * @property {!function(shaka.util.ManifestParserUtils.ContentType,
  1803. * !BufferSource):Promise} beforeAppendSegment
  1804. * A function called just before appending to the source buffer.
  1805. */
  1806. shaka.media.StreamingEngine.PlayerInterface;
  1807. /**
  1808. * @typedef {{
  1809. * type: shaka.util.ManifestParserUtils.ContentType,
  1810. * stream: shaka.extern.Stream,
  1811. * segmentIterator: shaka.media.SegmentIterator,
  1812. * lastSegmentReference: shaka.media.SegmentReference,
  1813. * lastInitSegmentReference: shaka.media.InitSegmentReference,
  1814. * lastTimestampOffset: ?number,
  1815. * lastAppendWindowStart: ?number,
  1816. * lastAppendWindowEnd: ?number,
  1817. * restoreStreamAfterTrickPlay: ?shaka.extern.Stream,
  1818. * endOfStream: boolean,
  1819. * performingUpdate: boolean,
  1820. * updateTimer: shaka.util.DelayedTick,
  1821. * waitingToClearBuffer: boolean,
  1822. * waitingToFlushBuffer: boolean,
  1823. * clearBufferSafeMargin: number,
  1824. * clearingBuffer: boolean,
  1825. * seeked: boolean,
  1826. * adaptation: boolean,
  1827. * recovering: boolean,
  1828. * hasError: boolean,
  1829. * operation: shaka.net.NetworkingEngine.PendingRequest
  1830. * }}
  1831. *
  1832. * @description
  1833. * Contains the state of a logical stream, i.e., a sequence of segmented data
  1834. * for a particular content type. At any given time there is a Stream object
  1835. * associated with the state of the logical stream.
  1836. *
  1837. * @property {shaka.util.ManifestParserUtils.ContentType} type
  1838. * The stream's content type, e.g., 'audio', 'video', or 'text'.
  1839. * @property {shaka.extern.Stream} stream
  1840. * The current Stream.
  1841. * @property {shaka.media.SegmentIndexIterator} segmentIterator
  1842. * An iterator through the segments of |stream|.
  1843. * @property {shaka.media.SegmentReference} lastSegmentReference
  1844. * The SegmentReference of the last segment that was appended.
  1845. * @property {shaka.media.InitSegmentReference} lastInitSegmentReference
  1846. * The InitSegmentReference of the last init segment that was appended.
  1847. * @property {?number} lastTimestampOffset
  1848. * The last timestamp offset given to MediaSourceEngine for this type.
  1849. * @property {?number} lastAppendWindowStart
  1850. * The last append window start given to MediaSourceEngine for this type.
  1851. * @property {?number} lastAppendWindowEnd
  1852. * The last append window end given to MediaSourceEngine for this type.
  1853. * @property {?shaka.extern.Stream} restoreStreamAfterTrickPlay
  1854. * The Stream to restore after trick play mode is turned off.
  1855. * @property {boolean} endOfStream
  1856. * True indicates that the end of the buffer has hit the end of the
  1857. * presentation.
  1858. * @property {boolean} performingUpdate
  1859. * True indicates that an update is in progress.
  1860. * @property {shaka.util.DelayedTick} updateTimer
  1861. * A timer used to update the media state.
  1862. * @property {boolean} waitingToClearBuffer
  1863. * True indicates that the buffer must be cleared after the current update
  1864. * finishes.
  1865. * @property {boolean} waitingToFlushBuffer
  1866. * True indicates that the buffer must be flushed after it is cleared.
  1867. * @property {number} clearBufferSafeMargin
  1868. * The amount of buffer to retain when clearing the buffer after the update.
  1869. * @property {boolean} clearingBuffer
  1870. * True indicates that the buffer is being cleared.
  1871. * @property {boolean} seeked
  1872. * True indicates that the presentation just seeked.
  1873. * @property {boolean} adaptation
  1874. * True indicates that the presentation just automatically switched variants.
  1875. * @property {boolean} recovering
  1876. * True indicates that the last segment was not appended because it could not
  1877. * fit in the buffer.
  1878. * @property {boolean} hasError
  1879. * True indicates that the stream has encountered an error and has stopped
  1880. * updating.
  1881. * @property {shaka.net.NetworkingEngine.PendingRequest} operation
  1882. * Operation with the number of bytes to be downloaded.
  1883. */
  1884. shaka.media.StreamingEngine.MediaState_;
  1885. /**
  1886. * The fudge factor for appendWindowStart. By adjusting the window backward, we
  1887. * avoid rounding errors that could cause us to remove the keyframe at the start
  1888. * of the Period.
  1889. *
  1890. * NOTE: This was increased as part of the solution to
  1891. * https://github.com/shaka-project/shaka-player/issues/1281
  1892. *
  1893. * @const {number}
  1894. * @private
  1895. */
  1896. shaka.media.StreamingEngine.APPEND_WINDOW_START_FUDGE_ = 0.1;
  1897. /**
  1898. * The fudge factor for appendWindowEnd. By adjusting the window backward, we
  1899. * avoid rounding errors that could cause us to remove the last few samples of
  1900. * the Period. This rounding error could then create an artificial gap and a
  1901. * stutter when the gap-jumping logic takes over.
  1902. *
  1903. * https://github.com/shaka-project/shaka-player/issues/1597
  1904. *
  1905. * @const {number}
  1906. * @private
  1907. */
  1908. shaka.media.StreamingEngine.APPEND_WINDOW_END_FUDGE_ = 0.01;
  1909. /**
  1910. * The maximum number of segments by which a stream can get ahead of other
  1911. * streams.
  1912. *
  1913. * Introduced to keep StreamingEngine from letting one media type get too far
  1914. * ahead of another. For example, audio segments are typically much smaller
  1915. * than video segments, so in the time it takes to fetch one video segment, we
  1916. * could fetch many audio segments. This doesn't help with buffering, though,
  1917. * since the intersection of the two buffered ranges is what counts.
  1918. *
  1919. * @const {number}
  1920. * @private
  1921. */
  1922. shaka.media.StreamingEngine.MAX_RUN_AHEAD_SEGMENTS_ = 1;