Bug 744896 - Part 4: Enable track interfaces for media elements that are consuming media-resource. r=roc.

content/media/AbstractMediaDecoder.h

@@ -93,6 +93,8 @@ public:
   virtual void MetadataLoaded(MediaInfo* aInfo, MetadataTags* aTags) = 0;
   virtual void QueueMetadata(int64_t aTime, MediaInfo* aInfo, MetadataTags* aTags) = 0;
 
+  virtual void RemoveMediaTracks() = 0;
+
   // Set the media end time in microseconds
   virtual void SetMediaEndTime(int64_t aTime) = 0;
 
@@ -161,6 +163,24 @@ class MetadataEventRunner : public nsRunnable
   MetadataTags* mTags;
 };
 
+class RemoveMediaTracksEventRunner : public nsRunnable
+{
+public:
+  RemoveMediaTracksEventRunner(AbstractMediaDecoder* aDecoder)
+    : mDecoder(aDecoder)
+  {}
+
+  NS_IMETHOD Run() MOZ_OVERRIDE
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+
+    mDecoder->RemoveMediaTracks();
+    return NS_OK;
+  }
+
+private:
+  nsRefPtr<AbstractMediaDecoder> mDecoder;
+};
 
 }
 

content/media/BufferDecoder.cpp

@@ -158,6 +158,12 @@ BufferDecoder::QueueMetadata(int64_t aTime, MediaInfo* aInfo, MetadataTags* aTag
   // ignore
 }
 
+void
+BufferDecoder::RemoveMediaTracks()
+{
+  // ignore
+}
+
 void
 BufferDecoder::SetMediaEndTime(int64_t aTime)
 {

content/media/BufferDecoder.h

@@ -63,6 +63,8 @@ public:
   virtual void MetadataLoaded(MediaInfo* aInfo, MetadataTags* aTags) MOZ_OVERRIDE;
   virtual void QueueMetadata(int64_t aTime, MediaInfo* aInfo, MetadataTags* aTags) MOZ_OVERRIDE;
 
+  virtual void RemoveMediaTracks() MOZ_OVERRIDE;
+
   virtual void SetMediaEndTime(int64_t aTime) MOZ_OVERRIDE;
 
   virtual void UpdatePlaybackPosition(int64_t aTime) MOZ_OVERRIDE;

content/media/MediaDecoder.cpp

@@ -24,6 +24,11 @@
 #include <algorithm>
 #include "MediaShutdownManager.h"
 #include "AudioChannelService.h"
+#include "mozilla/dom/AudioTrack.h"
+#include "mozilla/dom/AudioTrackList.h"
+#include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/dom/VideoTrack.h"
+#include "mozilla/dom/VideoTrackList.h"
 
 #ifdef MOZ_WMF
 #include "WMFDecoder.h"
@@ -433,7 +438,8 @@ MediaDecoder::MediaDecoder() :
   mPinnedForSeek(false),
   mShuttingDown(false),
   mPausedForPlaybackRateNull(false),
-  mMinimizePreroll(false)
+  mMinimizePreroll(false),
+  mMediaTracksConstructed(false)
 {
   MOZ_COUNT_CTOR(MediaDecoder);
   MOZ_ASSERT(NS_IsMainThread());
@@ -693,6 +699,9 @@ void MediaDecoder::MetadataLoaded(MediaInfo* aInfo, MetadataTags* aTags)
     SetInfinite(true);
   }
 
+  mInfo = aInfo;
+  ConstructMediaTracks();
+
   if (mOwner) {
     // Make sure the element and the frame (if any) are told about
     // our new size.
@@ -1150,6 +1159,12 @@ void MediaDecoder::ChangeState(PlayState aState)
   }
   mPlayState = aState;
 
+  if (mPlayState == PLAY_STATE_PLAYING) {
+    ConstructMediaTracks();
+  } else if (mPlayState == PLAY_STATE_ENDED) {
+    RemoveMediaTracks();
+  }
+
   ApplyStateToStateMachine(mPlayState);
 
   if (aState!= PLAY_STATE_LOADING) {
@@ -1774,6 +1789,73 @@ MediaDecoder::GetOwner()
   return mOwner;
 }
 
+void
+MediaDecoder::ConstructMediaTracks()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (mMediaTracksConstructed) {
+    return;
+  }
+
+  if (!mOwner || !mInfo) {
+    return;
+  }
+
+  HTMLMediaElement* element = mOwner->GetMediaElement();
+  if (!element) {
+    return;
+  }
+
+  mMediaTracksConstructed = true;
+
+  AudioTrackList* audioList = element->AudioTracks();
+  if (audioList && mInfo->HasAudio()) {
+    TrackInfo info = mInfo->mAudio.mTrackInfo;
+    nsRefPtr<AudioTrack> track = MediaTrackList::CreateAudioTrack(
+    info.mId, info.mKind, info.mLabel, info.mLanguage, info.mEnabled);
+
+    audioList->AddTrack(track);
+  }
+
+  VideoTrackList* videoList = element->VideoTracks();
+  if (videoList && mInfo->HasVideo()) {
+    TrackInfo info = mInfo->mVideo.mTrackInfo;
+    nsRefPtr<VideoTrack> track = MediaTrackList::CreateVideoTrack(
+    info.mId, info.mKind, info.mLabel, info.mLanguage);
+
+    videoList->AddTrack(track);
+    track->SetEnabledInternal(info.mEnabled, MediaTrack::FIRE_NO_EVENTS);
+  }
+}
+
+void
+MediaDecoder::RemoveMediaTracks()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mOwner) {
+    return;
+  }
+
+  HTMLMediaElement* element = mOwner->GetMediaElement();
+  if (!element) {
+    return;
+  }
+
+  AudioTrackList* audioList = element->AudioTracks();
+  if (audioList) {
+    audioList->RemoveTracks();
+  }
+
+  VideoTrackList* videoList = element->VideoTracks();
+  if (videoList) {
+    videoList->RemoveTracks();
+  }
+
+  mMediaTracksConstructed = false;
+}
+
 MediaMemoryTracker::MediaMemoryTracker()
 {
 }

content/media/MediaDecoder.h

@@ -777,6 +777,15 @@ public:
   virtual void MetadataLoaded(MediaInfo* aInfo,
                               MetadataTags* aTags);
 
+  // Called from MetadataLoaded(). Creates audio tracks and adds them to its
+  // owner's audio track list, and implies to video tracks respectively.
+  // Call on the main thread only.
+  void ConstructMediaTracks();
+
+  // Removes all audio tracks and video tracks that are previously added into
+  // the track list. Call on the main thread only.
+  virtual void RemoveMediaTracks() MOZ_OVERRIDE;
+
   // Called when the first frame has been loaded.
   // Call on the main thread only.
   void FirstFrameLoaded();
@@ -1209,6 +1218,14 @@ protected:
   // to minimize preroll, as we assume the user is likely to keep playing,
   // or play the media again.
   bool mMinimizePreroll;
+
+  // True if audio tracks and video tracks are constructed and added into the
+  // track list, false if all tracks are removed from the track list.
+  bool mMediaTracksConstructed;
+
+  // Stores media info, including info of audio tracks and video tracks, should
+  // only be accessed from main thread.
+  nsAutoPtr<MediaInfo> mInfo;
 };
 
 } // namespace mozilla

content/media/MediaInfo.h

@@ -9,9 +9,31 @@
 #include "nsSize.h"
 #include "nsRect.h"
 #include "ImageTypes.h"
+#include "nsString.h"
 
 namespace mozilla {
 
+struct TrackInfo {
+  void Init(const nsAString& aId,
+            const nsAString& aKind,
+            const nsAString& aLabel,
+            const nsAString& aLanguage,
+            bool aEnabled)
+  {
+    mId = aId;
+    mKind = aKind;
+    mLabel = aLabel;
+    mLanguage = aLanguage;
+    mEnabled = aEnabled;
+  }
+
+  nsString mId;
+  nsString mKind;
+  nsString mLabel;
+  nsString mLanguage;
+  bool mEnabled;
+};
+
 // Stores info relevant to presenting media frames.
 class VideoInfo {
 public:
@@ -19,7 +41,12 @@ public:
     : mDisplay(0,0)
     , mStereoMode(StereoMode::MONO)
     , mHasVideo(false)
-  {}
+  {
+    // TODO: TrackInfo should be initialized by its specific codec decoder.
+    // This following call should be removed once we have that implemented.
+    mTrackInfo.Init(NS_LITERAL_STRING("2"), NS_LITERAL_STRING("main"),
+    EmptyString(), EmptyString(), true);
+  }
 
   // Size in pixels at which the video is rendered. This is after it has
   // been scaled by its aspect ratio.
@@ -30,6 +57,8 @@ public:
 
   // True if we have an active video bitstream.
   bool mHasVideo;
+
+  TrackInfo mTrackInfo;
 };
 
 class AudioInfo {
@@ -38,7 +67,12 @@ public:
     : mRate(44100)
     , mChannels(2)
     , mHasAudio(false)
-  {}
+  {
+    // TODO: TrackInfo should be initialized by its specific codec decoder.
+    // This following call should be removed once we have that implemented.
+    mTrackInfo.Init(NS_LITERAL_STRING("1"), NS_LITERAL_STRING("main"),
+    EmptyString(), EmptyString(), true);
+  }
 
   // Sample rate.
   uint32_t mRate;
@@ -48,6 +82,8 @@ public:
 
   // True if we have an active audio bitstream.
   bool mHasAudio;
+
+  TrackInfo mTrackInfo;
 };
 
 class MediaInfo {
@@ -67,6 +103,7 @@ public:
     return HasVideo() || HasAudio();
   }
 
+  // TODO: Store VideoInfo and AudioIndo in arrays to support multi-tracks.
   VideoInfo mVideo;
   AudioInfo mAudio;
 };

content/media/MediaMetadataManager.h

@@ -46,6 +46,11 @@ namespace mozilla {
       void DispatchMetadataIfNeeded(AbstractMediaDecoder* aDecoder, double aCurrentTime) {
         TimedMetadata* metadata = mMetadataQueue.getFirst();
         while (metadata && aCurrentTime >= static_cast<double>(metadata->mPublishTime) / USECS_PER_S) {
+          // Remove all media tracks from the list first.
+          nsCOMPtr<nsIRunnable> removeTracksEvent =
+            new RemoveMediaTracksEventRunner(aDecoder);
+          NS_DispatchToMainThread(removeTracksEvent);
+
           nsCOMPtr<nsIRunnable> metadataUpdatedEvent =
             new MetadataEventRunner(aDecoder,
                                     metadata->mInfo.forget(),

content/media/MediaTrackList.cpp

@@ -105,6 +105,15 @@ MediaTrackList::RemoveTrack(const nsRefPtr<MediaTrack>& aTrack)
   CreateAndDispatchTrackEventRunner(aTrack, NS_LITERAL_STRING("removetrack"));
 }
 
+void
+MediaTrackList::RemoveTracks()
+{
+  while (!mTracks.IsEmpty()) {
+    nsRefPtr<MediaTrack> track = mTracks.LastElement();
+    RemoveTrack(track);
+  }
+}
+
 already_AddRefed<AudioTrack>
 MediaTrackList::CreateAudioTrack(const nsAString& aId,
                                  const nsAString& aKind,

content/media/MediaTrackList.h

@@ -79,6 +79,8 @@ public:
 
   void RemoveTrack(const nsRefPtr<MediaTrack>& aTrack);
 
+  void RemoveTracks();
+
   static already_AddRefed<AudioTrack>
   CreateAudioTrack(const nsAString& aId,
                    const nsAString& aKind,

content/media/test/mochitest.ini

@@ -380,7 +380,9 @@ skip-if = buildapp == 'b2g' || toolkit == 'android' # mimetype check, bug 969289
 [test_mediarecorder_reload_crash.html]
 [test_mediarecorder_unsupported_src.html]
 [test_mediarecorder_record_getdata_afterstart.html]
+[test_mediatrack_consuming_mediaresource.html]
 [test_mediatrack_events_and_consuming_ms.html]
+[test_mediatrack_replay_from_end.html]
 [test_metadata.html]
 [test_mixed_principals.html]
 skip-if = true # bug 567954 and intermittent leaks

content/media/test/test_mediatrack_consuming_mediaresource.html

@@ -0,0 +1,181 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test track interfaces when consuming media resources</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+  <script type="text/javascript" src="manifest.js"></script>
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+var manager = new MediaTestManager;
+
+function startTest(test, token) {
+  var elemType = getMajorMimeType(test.type);
+  var element = document.createElement(elemType);
+
+  var audioOnchange = 0;
+  var audioOnaddtrack = 0;
+  var audioOnremovetrack = 0;
+  var videoOnchange = 0;
+  var videoOnaddtrack = 0;
+  var videoOnremovetrack = 0;
+  var isPlaying = false;
+
+  isnot(element.audioTracks, undefined,
+        'HTMLMediaElement::AudioTracks() property should be available.');
+  isnot(element.videoTracks, undefined,
+        'HTMLMediaElement::VideoTracks() property should be available.');
+
+  element.audioTracks.onaddtrack = function(e) {
+    audioOnaddtrack++;
+  }
+
+  element.audioTracks.onremovetrack = function(e) {
+    audioOnremovetrack++;
+  }
+
+  element.audioTracks.onchange = function(e) {
+    audioOnchange++;
+  }
+
+  element.videoTracks.onaddtrack = function(e) {
+    videoOnaddtrack++;
+  }
+
+  element.videoTracks.onremovetrack = function(e) {
+    videoOnremovetrack++;
+  }
+
+  element.videoTracks.onchange = function(e) {
+    videoOnchange++;
+  }
+
+  function checkTrackRemoved() {
+    if (isPlaying) {
+      if (test.hasAudio) {
+        is(audioOnremovetrack, 1, 'Calls of onremovetrack on audioTracks should be 1.');
+        is(element.audioTracks.length, 0, 'The length of audioTracks should be 0.');
+      }
+      if (test.hasVideo) {
+        is(videoOnremovetrack, 1, 'Calls of onremovetrack on videoTracks should be 1.');
+        is(element.videoTracks.length, 0, 'The length of videoTracks should be 0.');
+      }
+    }
+  }
+
+  function onended() {
+    ok(true, 'Event ended is expected to be fired on element.');
+    checkTrackRemoved();
+    element.onended = null;
+    element.onplaying = null;
+    element.onpause = null;
+    manager.finished(element.token);
+  }
+
+  function checkTrackAdded() {
+    isPlaying = true;
+    if (test.hasAudio) {
+      is(audioOnaddtrack, 1, 'Calls of onaddtrack on audioTracks should be 1.');
+      is(element.audioTracks.length, 1, 'The length of audioTracks should be 1.');
+      ok(element.audioTracks[0].enabled, 'Audio track should be enabled as default.');
+    }
+    if (test.hasVideo) {
+      is(videoOnaddtrack, 1, 'Calls of onaddtrack on videoTracks should be 1.');
+      is(element.videoTracks.length, 1, 'The length of videoTracks should be 1.');
+      is(element.videoTracks.selectedIndex, 0,
+         'The first video track is set selected as default.');
+    }
+  }
+
+  function setTrackEnabled(enabled) {
+    if (test.hasAudio) {
+      element.audioTracks[0].enabled = enabled;
+    }
+    if (test.hasVideo) {
+      element.videoTracks[0].selected = enabled;
+    }
+  }
+
+  function checkTrackChanged(calls, enabled) {
+    if (test.hasAudio) {
+      is(audioOnchange, calls, 'Calls of onchange on audioTracks should be '+calls);
+      is(element.audioTracks[0].enabled, enabled,
+         'Enabled value of the audio track should be ' +enabled);
+    }
+    if (test.hasVideo) {
+      is(videoOnchange, calls, 'Calls of onchange on videoTracks should be '+calls);
+      is(element.videoTracks[0].selected, enabled,
+         'Selected value of the video track should be ' +enabled);
+      var index = enabled ? 0 : -1;
+      is(element.videoTracks.selectedIndex, index,
+         'SelectedIndex of video tracks should be ' +index);
+    }
+  }
+
+  function onpause() {
+    element.onpause = null;
+    if (element.ended) {
+      return;
+    }
+    if (steps == 1) {
+      setTrackEnabled(false);
+      element.onplaying = onplaying;
+      element.play();
+      steps++;
+    } else if (steps == 2) {
+      setTrackEnabled(true);
+      element.onplaying = onplaying;
+      element.play();
+      steps++;
+    }
+  }
+
+  function onplaying() {
+    element.onplaying = null;
+    if (element.ended) {
+      return;
+    }
+    if (steps == 1) {
+      element.onpause = onpause;
+      element.pause();
+      checkTrackAdded();
+    } else if (steps == 2) {
+      element.onpause = onpause;
+      element.pause();
+      checkTrackChanged(1, false);
+    } else if (steps == 3) {
+      checkTrackChanged(2, true);
+    }
+  }
+
+  var steps = 0;
+
+  element.token = token;
+  manager.started(token);
+
+  element.src = test.name;
+  element.test = test;
+  element.onplaying = onplaying;
+  element.onended = onended;
+  element.play();
+  steps++;
+}
+
+SimpleTest.waitForExplicitFinish();
+SpecialPowers.pushPrefEnv(
+  {
+    "set": [
+      ["media.track.enabled", true]
+    ]
+  },
+  function() {
+    manager.runTests(gTrackTests, startTest);
+  });
+
+</script>
+</pre>
+</body>
+</html>

content/media/test/test_mediatrack_replay_from_end.html

@@ -0,0 +1,146 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test media tracks if replay after playback has ended</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+  <script type="text/javascript" src="manifest.js"></script>
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+var manager = new MediaTestManager;
+
+function startTest(test, token) {
+  // Scenario to test:
+  // 1. Audio tracks and video tracks should be added to the track list when
+  //    playing, and all tracks should be removed from the list after we seek
+  //    to the end.
+  // 2. All tracks should be added back to the list if we replay from the end,
+  //    and all tracks should be removed from the list after we seek to the end.
+  // 3. After seek to the middle from end of playback, all tracks should be
+  //    added back to the list if we play from here, and all tracks should be
+  //    removed from the list after we seek to the end.
+
+  var elemType = getMajorMimeType(test.type);
+  var element = document.createElement(elemType);
+
+  var audioOnchange = 0;
+  var audioOnaddtrack = 0;
+  var audioOnremovetrack = 0;
+  var videoOnchange = 0;
+  var videoOnaddtrack = 0;
+  var videoOnremovetrack = 0;
+  var isPlaying = false;
+  var steps = 0;
+
+  element.audioTracks.onaddtrack = function(e) {
+    audioOnaddtrack++;
+  }
+
+  element.audioTracks.onremovetrack = function(e) {
+    audioOnremovetrack++;
+  }
+
+  element.videoTracks.onaddtrack = function(e) {
+    videoOnaddtrack++;
+  }
+
+  element.videoTracks.onremovetrack = function(e) {
+    videoOnremovetrack++;
+  }
+
+  function testTrackEventCalls(expectedCalls) {
+    if (test.hasAudio) {
+      is(audioOnaddtrack, expectedCalls,
+         'Calls of onaddtrack on audioTracks should be '+expectedCalls+' times.');
+      is(audioOnremovetrack, expectedCalls,
+         'Calls of onremovetrack on audioTracks should be '+expectedCalls+' times.');
+    }
+    if (test.hasVideo) {
+      is(videoOnaddtrack, expectedCalls,
+         'Calls of onaddtrack on videoTracks should be '+expectedCalls+' times.');
+      is(videoOnremovetrack, expectedCalls,
+         'Calls of onremovetrack on videoTracks should be '+expectedCalls+' times.');
+    }
+  }
+
+  function finishTesting() {
+    element.onpause = null;
+    element.onseeked = null;
+    element.onplaying = null;
+    element.onended = null;
+    manager.finished(element.token);
+  }
+
+  function onended() {
+    if (isPlaying) {
+      switch(steps) {
+        case 1:
+          testTrackEventCalls(1);
+          element.onplaying = onplaying;
+          element.play();
+          steps++;
+          break;
+        case 2:
+          testTrackEventCalls(2);
+          element.currentTime = element.duration * 0.5;
+          element.onplaying = onplaying;
+          element.play();
+          steps++;
+          break;
+        case 3:
+          testTrackEventCalls(3);
+          finishTesting();
+          break;
+      }
+    } else {
+      ok(true, 'Finish the test anyway if ended is fired before other events.');
+      finishTesting();
+    }
+  }
+
+  function seekToEnd() {
+    element.onpause = null;
+    element.currentTime = element.duration * 1.1;
+  }
+
+  function onseeked() {
+    element.onseeked = null;
+    element.onpause = seekToEnd;
+    element.pause();
+  }
+
+  function onplaying() {
+    isPlaying = true;
+    element.onplaying = null;
+    element.onseeked = onseeked;
+  }
+
+  element.token = token;
+  manager.started(token);
+
+  element.src = test.name;
+  element.test = test;
+  element.onplaying = onplaying;
+  element.onended = onended;
+  element.play();
+  steps++;
+}
+
+SimpleTest.waitForExplicitFinish();
+SpecialPowers.pushPrefEnv(
+  {
+    "set": [
+      ["media.track.enabled", true]
+    ]
+  },
+  function() {
+    manager.runTests(gTrackTests, startTest);
+  });
+
+</script>
+</pre>
+</body>
+</html>