wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity
d71d6904a9
gecko/content/media/gstreamer/GStreamerReader.cpp

1178 lines
38 KiB
C++
Raw Blame History

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsError.h"
#include "nsMimeTypes.h"
#include "MediaDecoderStateMachine.h"
#include "AbstractMediaDecoder.h"
#include "MediaResource.h"
#include "GStreamerReader.h"
#include "GStreamerFormatHelper.h"
#include "GStreamerMozVideoBuffer.h"
#include "VideoUtils.h"
#include "mozilla/dom/TimeRanges.h"
#include "mozilla/Preferences.h"
#include "GStreamerLoader.h"
namespace mozilla {
using namespace layers;
// Un-comment to enable logging of seek bisections.
//#define SEEK_LOGGING
#ifdef PR_LOGGING
extern PRLogModuleInfo* gMediaDecoderLog;
#define LOG(type, msg) PR_LOG(gMediaDecoderLog, type, msg)
#else
#define LOG(type, msg)
#endif
extern bool
IsYV12Format(const VideoData::YCbCrBuffer::Plane& aYPlane,
const VideoData::YCbCrBuffer::Plane& aCbPlane,
const VideoData::YCbCrBuffer::Plane& aCrPlane);
static const unsigned int MAX_CHANNELS = 4;
// Let the demuxer work in pull mode for short files
static const int SHORT_FILE_SIZE = 1024 * 1024;
// The default resource->Read() size when working in push mode
static const int DEFAULT_SOURCE_READ_SIZE = 50 * 1024;
typedef enum {
GST_PLAY_FLAG_VIDEO = (1 << 0),
GST_PLAY_FLAG_AUDIO = (1 << 1),
GST_PLAY_FLAG_TEXT = (1 << 2),
GST_PLAY_FLAG_VIS = (1 << 3),
GST_PLAY_FLAG_SOFT_VOLUME = (1 << 4),
GST_PLAY_FLAG_NATIVE_AUDIO = (1 << 5),
GST_PLAY_FLAG_NATIVE_VIDEO = (1 << 6),
GST_PLAY_FLAG_DOWNLOAD = (1 << 7),
GST_PLAY_FLAG_BUFFERING = (1 << 8),
GST_PLAY_FLAG_DEINTERLACE = (1 << 9),
GST_PLAY_FLAG_SOFT_COLORBALANCE = (1 << 10)
} PlayFlags;
GStreamerReader::GStreamerReader(AbstractMediaDecoder* aDecoder)
: MediaDecoderReader(aDecoder),
mMP3FrameParser(aDecoder->GetResource()->GetLength()),
mUseParserDuration(false),
mPlayBin(nullptr),
mBus(nullptr),
mSource(nullptr),
mVideoSink(nullptr),
mVideoAppSink(nullptr),
mAudioSink(nullptr),
mAudioAppSink(nullptr),
mFormat(GST_VIDEO_FORMAT_UNKNOWN),
mVideoSinkBufferCount(0),
mAudioSinkBufferCount(0),
mGstThreadsMonitor("media.gst.threads"),
mReachedEos(false),
fpsNum(0),
fpsDen(0)
{
MOZ_COUNT_CTOR(GStreamerReader);
mSrcCallbacks.need_data = GStreamerReader::NeedDataCb;
mSrcCallbacks.enough_data = GStreamerReader::EnoughDataCb;
mSrcCallbacks.seek_data = GStreamerReader::SeekDataCb;
mSinkCallbacks.eos = GStreamerReader::EosCb;
mSinkCallbacks.new_preroll = GStreamerReader::NewPrerollCb;
mSinkCallbacks.new_buffer = GStreamerReader::NewBufferCb;
mSinkCallbacks.new_buffer_list = nullptr;
gst_segment_init(&mVideoSegment, GST_FORMAT_UNDEFINED);
gst_segment_init(&mAudioSegment, GST_FORMAT_UNDEFINED);
}
GStreamerReader::~GStreamerReader()
{
MOZ_COUNT_DTOR(GStreamerReader);
ResetDecode();
if (mPlayBin) {
gst_app_src_end_of_stream(mSource);
if (mSource)
gst_object_unref(mSource);
gst_element_set_state(mPlayBin, GST_STATE_NULL);
gst_object_unref(mPlayBin);
mPlayBin = nullptr;
mVideoSink = nullptr;
mVideoAppSink = nullptr;
mAudioSink = nullptr;
mAudioAppSink = nullptr;
gst_object_unref(mBus);
mBus = nullptr;
}
}
nsresult GStreamerReader::Init(MediaDecoderReader* aCloneDonor)
{
GError* error = nullptr;
if (!gst_init_check(0, 0, &error)) {
LOG(PR_LOG_ERROR, ("gst initialization failed: %s", error->message));
g_error_free(error);
return NS_ERROR_FAILURE;
}
mPlayBin = gst_element_factory_make("playbin2", nullptr);
if (!mPlayBin) {
LOG(PR_LOG_ERROR, ("couldn't create playbin2"));
return NS_ERROR_FAILURE;
}
g_object_set(mPlayBin, "buffer-size", 0, nullptr);
mBus = gst_pipeline_get_bus(GST_PIPELINE(mPlayBin));
mVideoSink = gst_parse_bin_from_description("capsfilter name=filter ! "
"appsink name=videosink sync=true max-buffers=1 "
"caps=video/x-raw-yuv,format=(fourcc)I420"
, TRUE, nullptr);
mVideoAppSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(mVideoSink),
"videosink"));
gst_app_sink_set_callbacks(mVideoAppSink, &mSinkCallbacks,
(gpointer) this, nullptr);
GstPad* sinkpad = gst_element_get_pad(GST_ELEMENT(mVideoAppSink), "sink");
gst_pad_add_event_probe(sinkpad,
G_CALLBACK(&GStreamerReader::EventProbeCb), this);
gst_object_unref(sinkpad);
gst_pad_set_bufferalloc_function(sinkpad, GStreamerReader::AllocateVideoBufferCb);
gst_pad_set_element_private(sinkpad, this);
mAudioSink = gst_parse_bin_from_description("capsfilter name=filter ! "
#ifdef MOZ_SAMPLE_TYPE_FLOAT32
"appsink name=audiosink max-buffers=2 sync=false caps=audio/x-raw-float,"
#ifdef IS_LITTLE_ENDIAN
"channels={1,2},width=32,endianness=1234", TRUE, nullptr);
#else
"channels={1,2},width=32,endianness=4321", TRUE, nullptr);
#endif
#else
"appsink name=audiosink max-buffers=2 sync=false caps=audio/x-raw-int,"
#ifdef IS_LITTLE_ENDIAN
"channels={1,2},width=16,endianness=1234", TRUE, nullptr);
#else
"channels={1,2},width=16,endianness=4321", TRUE, nullptr);
#endif
#endif
mAudioAppSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(mAudioSink),
"audiosink"));
gst_app_sink_set_callbacks(mAudioAppSink, &mSinkCallbacks,
(gpointer) this, nullptr);
sinkpad = gst_element_get_pad(GST_ELEMENT(mAudioAppSink), "sink");
gst_pad_add_event_probe(sinkpad,
G_CALLBACK(&GStreamerReader::EventProbeCb), this);
gst_object_unref(sinkpad);
g_object_set(mPlayBin, "uri", "appsrc://",
"video-sink", mVideoSink,
"audio-sink", mAudioSink,
nullptr);
g_signal_connect(G_OBJECT(mPlayBin), "notify::source",
G_CALLBACK(GStreamerReader::PlayBinSourceSetupCb), this);
g_signal_connect(G_OBJECT(mPlayBin), "element-added",
G_CALLBACK(GStreamerReader::PlayElementAddedCb), this);
return NS_OK;
}
GstBusSyncReply
GStreamerReader::ErrorCb(GstBus *aBus, GstMessage *aMessage, gpointer aUserData)
{
return static_cast<GStreamerReader*>(aUserData)->Error(aBus, aMessage);
}
GstBusSyncReply
GStreamerReader::Error(GstBus *aBus, GstMessage *aMessage)
{
if (GST_MESSAGE_TYPE(aMessage) == GST_MESSAGE_ERROR) {
Eos();
}
return GST_BUS_PASS;
}
void GStreamerReader::PlayBinSourceSetupCb(GstElement* aPlayBin,
GParamSpec* pspec,
gpointer aUserData)
{
GstElement *source;
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
g_object_get(aPlayBin, "source", &source, nullptr);
reader->PlayBinSourceSetup(GST_APP_SRC(source));
}
void GStreamerReader::PlayBinSourceSetup(GstAppSrc* aSource)
{
mSource = GST_APP_SRC(aSource);
gst_app_src_set_callbacks(mSource, &mSrcCallbacks, (gpointer) this, nullptr);
MediaResource* resource = mDecoder->GetResource();
/* do a short read to trigger a network request so that GetLength() below
* returns something meaningful and not -1
*/
char buf[512];
unsigned int size = 0;
resource->Read(buf, sizeof(buf), &size);
resource->Seek(SEEK_SET, 0);
/* now we should have a length */
int64_t resourceLength = resource->GetLength();
gst_app_src_set_size(mSource, resourceLength);
if (resource->IsDataCachedToEndOfResource(0) ||
(resourceLength != -1 && resourceLength <= SHORT_FILE_SIZE)) {
/* let the demuxer work in pull mode for local files (or very short files)
* so that we get optimal seeking accuracy/performance
*/
LOG(PR_LOG_DEBUG, ("configuring random access, len %lld", resourceLength));
gst_app_src_set_stream_type(mSource, GST_APP_STREAM_TYPE_RANDOM_ACCESS);
} else {
/* make the demuxer work in push mode so that seeking is kept to a minimum
*/
LOG(PR_LOG_DEBUG, ("configuring push mode, len %lld", resourceLength));
gst_app_src_set_stream_type(mSource, GST_APP_STREAM_TYPE_SEEKABLE);
}
// Set the source MIME type to stop typefind trying every. single. format.
GstCaps *caps =
GStreamerFormatHelper::ConvertFormatsToCaps(mDecoder->GetResource()->GetContentType().get(),
nullptr);
gst_app_src_set_caps(aSource, caps);
gst_caps_unref(caps);
}
/**
* If this stream is an MP3, we want to parse the headers to estimate the
* stream duration.
*/
nsresult GStreamerReader::ParseMP3Headers()
{
MediaResource *resource = mDecoder->GetResource();
const uint32_t MAX_READ_BYTES = 4096;
uint64_t offset = 0;
char bytes[MAX_READ_BYTES];
uint32_t bytesRead;
do {
nsresult rv = resource->ReadAt(offset, bytes, MAX_READ_BYTES, &bytesRead);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(bytesRead, NS_ERROR_FAILURE);
mMP3FrameParser.Parse(bytes, bytesRead, offset);
offset += bytesRead;
} while (!mMP3FrameParser.ParsedHeaders());
if (mMP3FrameParser.IsMP3()) {
mLastParserDuration = mMP3FrameParser.GetDuration();
}
return NS_OK;
}
nsresult GStreamerReader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
nsresult ret = NS_OK;
/* We do 3 attempts here: decoding audio and video, decoding video only,
* decoding audio only. This allows us to play streams that have one broken
* stream but that are otherwise decodeable.
*/
guint flags[3] = {GST_PLAY_FLAG_VIDEO|GST_PLAY_FLAG_AUDIO,
static_cast<guint>(~GST_PLAY_FLAG_AUDIO), static_cast<guint>(~GST_PLAY_FLAG_VIDEO)};
guint default_flags, current_flags;
g_object_get(mPlayBin, "flags", &default_flags, nullptr);
GstMessage* message = nullptr;
for (unsigned int i = 0; i < G_N_ELEMENTS(flags); i++) {
current_flags = default_flags & flags[i];
g_object_set(G_OBJECT(mPlayBin), "flags", current_flags, nullptr);
/* reset filter caps to ANY */
GstCaps* caps = gst_caps_new_any();
GstElement* filter = gst_bin_get_by_name(GST_BIN(mAudioSink), "filter");
g_object_set(filter, "caps", caps, nullptr);
gst_object_unref(filter);
filter = gst_bin_get_by_name(GST_BIN(mVideoSink), "filter");
g_object_set(filter, "caps", caps, nullptr);
gst_object_unref(filter);
gst_caps_unref(caps);
filter = nullptr;
if (!(current_flags & GST_PLAY_FLAG_AUDIO))
filter = gst_bin_get_by_name(GST_BIN(mAudioSink), "filter");
else if (!(current_flags & GST_PLAY_FLAG_VIDEO))
filter = gst_bin_get_by_name(GST_BIN(mVideoSink), "filter");
if (filter) {
/* Little trick: set the target caps to "skip" so that playbin2 fails to
* find a decoder for the stream we want to skip.
*/
GstCaps* filterCaps = gst_caps_new_simple ("skip", nullptr);
g_object_set(filter, "caps", filterCaps, nullptr);
gst_caps_unref(filterCaps);
gst_object_unref(filter);
}
/* start the pipeline */
gst_element_set_state(mPlayBin, GST_STATE_PAUSED);
/* Wait for ASYNC_DONE, which is emitted when the pipeline is built,
* prerolled and ready to play. Also watch for errors.
*/
message = gst_bus_timed_pop_filtered(mBus, GST_CLOCK_TIME_NONE,
(GstMessageType)(GST_MESSAGE_ASYNC_DONE | GST_MESSAGE_ERROR));
if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) {
GError* error;
gchar* debug;
gst_message_parse_error(message, &error, &debug);
LOG(PR_LOG_ERROR, ("read metadata error: %s: %s", error->message,
debug));
g_error_free(error);
g_free(debug);
gst_element_set_state(mPlayBin, GST_STATE_NULL);
gst_message_unref(message);
ret = NS_ERROR_FAILURE;
} else {
gst_message_unref(message);
ret = NS_OK;
break;
}
}
if (NS_SUCCEEDED(ret))
ret = CheckSupportedFormats();
if (NS_FAILED(ret))
/* we couldn't get this to play */
return ret;
/* FIXME: workaround for a bug in matroskademux. This seek makes matroskademux
* parse the index */
if (gst_element_seek_simple(mPlayBin, GST_FORMAT_TIME,
GST_SEEK_FLAG_FLUSH, 0)) {
/* after a seek we need to wait again for ASYNC_DONE */
message = gst_bus_timed_pop_filtered(mBus, GST_CLOCK_TIME_NONE,
(GstMessageType)(GST_MESSAGE_ASYNC_DONE | GST_MESSAGE_ERROR));
if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) {
gst_element_set_state(mPlayBin, GST_STATE_NULL);
gst_message_unref(message);
return NS_ERROR_FAILURE;
}
}
bool isMP3 = mDecoder->GetResource()->GetContentType().EqualsASCII(AUDIO_MP3);
if (isMP3) {
ParseMP3Headers();
}
/* report the duration */
gint64 duration;
GstFormat format = GST_FORMAT_TIME;
if (isMP3 && mMP3FrameParser.IsMP3()) {
// The MP3FrameParser has reported a duration; use that over the gstreamer
// reported duration for inter-platform consistency.
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mUseParserDuration = true;
mLastParserDuration = mMP3FrameParser.GetDuration();
mDecoder->SetMediaDuration(mLastParserDuration);
} else if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
&format, &duration) && format == GST_FORMAT_TIME) {
// Otherwise use the gstreamer duration.
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
LOG(PR_LOG_DEBUG, ("returning duration %" GST_TIME_FORMAT,
GST_TIME_ARGS (duration)));
duration = GST_TIME_AS_USECONDS (duration);
mDecoder->SetMediaDuration(duration);
} else {
mDecoder->SetMediaSeekable(false);
}
int n_video = 0, n_audio = 0;
g_object_get(mPlayBin, "n-video", &n_video, "n-audio", &n_audio, nullptr);
mInfo.mVideo.mHasVideo = n_video != 0;
mInfo.mAudio.mHasAudio = n_audio != 0;
*aInfo = mInfo;
*aTags = nullptr;
// Watch the pipeline for fatal errors
gst_bus_set_sync_handler(mBus, GStreamerReader::ErrorCb, this);
/* set the pipeline to PLAYING so that it starts decoding and queueing data in
* the appsinks */
gst_element_set_state(mPlayBin, GST_STATE_PLAYING);
return NS_OK;
}
nsresult GStreamerReader::CheckSupportedFormats()
{
bool done = false;
bool unsupported = false;
GstIterator *it = gst_bin_iterate_recurse(GST_BIN(mPlayBin));
while (!done) {
GstElement* element;
GstIteratorResult res = gst_iterator_next(it, (void **)&element);
switch(res) {
case GST_ITERATOR_OK:
{
GstElementFactory* factory = gst_element_get_factory(element);
if (factory) {
const char* klass = gst_element_factory_get_klass(factory);
GstPad* pad = gst_element_get_pad(element, "sink");
if (pad) {
GstCaps* caps = gst_pad_get_negotiated_caps(pad);
if (caps) {
/* check for demuxers but ignore elements like id3demux */
if (strstr (klass, "Demuxer") && !strstr(klass, "Metadata"))
unsupported = !GStreamerFormatHelper::Instance()->CanHandleContainerCaps(caps);
else if (strstr (klass, "Decoder") && !strstr(klass, "Generic"))
unsupported = !GStreamerFormatHelper::Instance()->CanHandleCodecCaps(caps);
gst_caps_unref(caps);
}
gst_object_unref(pad);
}
}
gst_object_unref(element);
done = unsupported;
break;
}
case GST_ITERATOR_RESYNC:
unsupported = false;
done = false;
break;
case GST_ITERATOR_ERROR:
done = true;
break;
case GST_ITERATOR_DONE:
done = true;
break;
}
}
return unsupported ? NS_ERROR_FAILURE : NS_OK;
}
nsresult GStreamerReader::ResetDecode()
{
nsresult res = NS_OK;
if (NS_FAILED(MediaDecoderReader::ResetDecode())) {
res = NS_ERROR_FAILURE;
}
mVideoQueue.Reset();
mAudioQueue.Reset();
mVideoSinkBufferCount = 0;
mAudioSinkBufferCount = 0;
mReachedEos = false;
return res;
}
bool GStreamerReader::DecodeAudioData()
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
GstBuffer *buffer = nullptr;
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
if (mReachedEos) {
return false;
}
/* Wait something to be decoded before return or continue */
if (!mAudioSinkBufferCount) {
if(!mVideoSinkBufferCount) {
/* We have nothing decoded so it makes no sense to return to the state machine
* as it will call us back immediately, we'll return again and so on, wasting
* CPU cycles for no job done. So, block here until there is either video or
* audio data available
*/
mon.Wait();
if (!mAudioSinkBufferCount) {
/* There is still no audio data available, so either there is video data or
* something else has happened (Eos, etc...). Return to the state machine
* to process it.
*/
return true;
}
}
else {
return true;
}
}
buffer = gst_app_sink_pull_buffer(mAudioAppSink);
mAudioSinkBufferCount--;
}
int64_t timestamp = GST_BUFFER_TIMESTAMP(buffer);
timestamp = gst_segment_to_stream_time(&mAudioSegment,
GST_FORMAT_TIME, timestamp);
timestamp = GST_TIME_AS_USECONDS(timestamp);
int64_t duration = 0;
if (GST_CLOCK_TIME_IS_VALID(GST_BUFFER_DURATION(buffer)))
duration = GST_TIME_AS_USECONDS(GST_BUFFER_DURATION(buffer));
int64_t offset = GST_BUFFER_OFFSET(buffer);
unsigned int size = GST_BUFFER_SIZE(buffer);
int32_t frames = (size / sizeof(AudioDataValue)) / mInfo.mAudio.mChannels;
ssize_t outSize = static_cast<size_t>(size / sizeof(AudioDataValue));
nsAutoArrayPtr<AudioDataValue> data(new AudioDataValue[outSize]);
memcpy(data, GST_BUFFER_DATA(buffer), GST_BUFFER_SIZE(buffer));
AudioData* audio = new AudioData(offset, timestamp, duration,
frames, data.forget(), mInfo.mAudio.mChannels);
mAudioQueue.Push(audio);
gst_buffer_unref(buffer);
return true;
}
bool GStreamerReader::DecodeVideoFrame(bool &aKeyFrameSkip,
int64_t aTimeThreshold)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
GstBuffer *buffer = nullptr;
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
if (mReachedEos) {
return false;
}
/* Wait something to be decoded before return or continue */
if (!mVideoSinkBufferCount) {
if (!mAudioSinkBufferCount) {
/* We have nothing decoded so it makes no sense to return to the state machine
* as it will call us back immediately, we'll return again and so on, wasting
* CPU cycles for no job done. So, block here until there is either video or
* audio data available
*/
mon.Wait();
if (!mVideoSinkBufferCount) {
/* There is still no video data available, so either there is audio data or
* something else has happened (Eos, etc...). Return to the state machine
* to process it
*/
return true;
}
}
else {
return true;
}
}
mDecoder->NotifyDecodedFrames(0, 1);
buffer = gst_app_sink_pull_buffer(mVideoAppSink);
mVideoSinkBufferCount--;
}
bool isKeyframe = !GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DISCONT);
if ((aKeyFrameSkip && !isKeyframe)) {
gst_buffer_unref(buffer);
return true;
}
int64_t timestamp = GST_BUFFER_TIMESTAMP(buffer);
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
timestamp = gst_segment_to_stream_time(&mVideoSegment,
GST_FORMAT_TIME, timestamp);
}
NS_ASSERTION(GST_CLOCK_TIME_IS_VALID(timestamp),
"frame has invalid timestamp");
timestamp = GST_TIME_AS_USECONDS(timestamp);
if (timestamp < aTimeThreshold) {
LOG(PR_LOG_DEBUG, ("skipping frame %" GST_TIME_FORMAT
" threshold %" GST_TIME_FORMAT,
GST_TIME_ARGS(timestamp), GST_TIME_ARGS(aTimeThreshold)));
gst_buffer_unref(buffer);
return true;
}
if (!buffer)
/* no more frames */
return false;
int64_t duration = 0;
if (GST_CLOCK_TIME_IS_VALID(GST_BUFFER_DURATION(buffer)))
duration = GST_TIME_AS_USECONDS(GST_BUFFER_DURATION(buffer));
else if (fpsNum && fpsDen)
/* 1-frame duration */
duration = gst_util_uint64_scale(GST_USECOND, fpsNum, fpsDen);
nsRefPtr<PlanarYCbCrImage> image;
GstMozVideoBufferData* bufferdata = reinterpret_cast<GstMozVideoBufferData*>
GST_IS_MOZ_VIDEO_BUFFER(buffer)?gst_moz_video_buffer_get_data(GST_MOZ_VIDEO_BUFFER(buffer)):nullptr;
if(bufferdata)
image = bufferdata->mImage;
if (!image) {
/* Ugh, upstream is not calling gst_pad_alloc_buffer(). Fallback to
* allocating a PlanarYCbCrImage backed GstBuffer here and memcpy.
*/
GstBuffer* tmp = nullptr;
AllocateVideoBufferFull(nullptr, GST_BUFFER_OFFSET(buffer),
GST_BUFFER_SIZE(buffer), nullptr, &tmp, image);
/* copy */
gst_buffer_copy_metadata(tmp, buffer, (GstBufferCopyFlags)GST_BUFFER_COPY_ALL);
memcpy(GST_BUFFER_DATA(tmp), GST_BUFFER_DATA(buffer),
GST_BUFFER_SIZE(tmp));
gst_buffer_unref(buffer);
buffer = tmp;
}
guint8* data = GST_BUFFER_DATA(buffer);
int width = mPicture.width;
int height = mPicture.height;
GstVideoFormat format = mFormat;
VideoData::YCbCrBuffer b;
for(int i = 0; i < 3; i++) {
b.mPlanes[i].mData = data + gst_video_format_get_component_offset(format, i,
width, height);
b.mPlanes[i].mStride = gst_video_format_get_row_stride(format, i, width);
b.mPlanes[i].mHeight = gst_video_format_get_component_height(format,
i, height);
b.mPlanes[i].mWidth = gst_video_format_get_component_width(format,
i, width);
b.mPlanes[i].mOffset = 0;
b.mPlanes[i].mSkip = 0;
}
isKeyframe = !GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
int64_t offset = mDecoder->GetResource()->Tell(); // Estimate location in media.
VideoData* video = VideoData::Create(mInfo.mVideo, image, offset,
timestamp, duration, b,
isKeyframe, -1, mPicture);
mVideoQueue.Push(video);
gst_buffer_unref(buffer);
return true;
}
nsresult GStreamerReader::Seek(int64_t aTarget,
int64_t aStartTime,
int64_t aEndTime,
int64_t aCurrentTime)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
gint64 seekPos = aTarget * GST_USECOND;
LOG(PR_LOG_DEBUG, ("%p About to seek to %" GST_TIME_FORMAT,
mDecoder, GST_TIME_ARGS(seekPos)));
if (!gst_element_seek_simple(mPlayBin, GST_FORMAT_TIME,
static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE), seekPos)) {
LOG(PR_LOG_ERROR, ("seek failed"));
return NS_ERROR_FAILURE;
}
LOG(PR_LOG_DEBUG, ("seek succeeded"));
return DecodeToTarget(aTarget);
}
nsresult GStreamerReader::GetBuffered(dom::TimeRanges* aBuffered,
int64_t aStartTime)
{
if (!mInfo.HasValidMedia()) {
return NS_OK;
}
GstFormat format = GST_FORMAT_TIME;
MediaResource* resource = mDecoder->GetResource();
nsTArray<MediaByteRange> ranges;
resource->GetCachedRanges(ranges);
if (resource->IsDataCachedToEndOfResource(0)) {
/* fast path for local or completely cached files */
gint64 duration = 0;
duration = QueryDuration();
double end = (double) duration / GST_MSECOND;
LOG(PR_LOG_DEBUG, ("complete range [0, %f] for [0, %li]",
end, resource->GetLength()));
aBuffered->Add(0, end);
return NS_OK;
}
for(uint32_t index = 0; index < ranges.Length(); index++) {
int64_t startOffset = ranges[index].mStart;
int64_t endOffset = ranges[index].mEnd;
gint64 startTime, endTime;
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
startOffset, &format, &startTime) || format != GST_FORMAT_TIME)
continue;
if (!gst_element_query_convert(GST_ELEMENT(mPlayBin), GST_FORMAT_BYTES,
endOffset, &format, &endTime) || format != GST_FORMAT_TIME)
continue;
double start = (double) GST_TIME_AS_USECONDS (startTime) / GST_MSECOND;
double end = (double) GST_TIME_AS_USECONDS (endTime) / GST_MSECOND;
LOG(PR_LOG_DEBUG, ("adding range [%f, %f] for [%li %li] size %li",
start, end, startOffset, endOffset, resource->GetLength()));
aBuffered->Add(start, end);
}
return NS_OK;
}
void GStreamerReader::ReadAndPushData(guint aLength)
{
MediaResource* resource = mDecoder->GetResource();
NS_ASSERTION(resource, "Decoder has no media resource");
nsresult rv = NS_OK;
GstBuffer* buffer = gst_buffer_new_and_alloc(aLength);
guint8* data = GST_BUFFER_DATA(buffer);
uint32_t size = 0, bytesRead = 0;
while(bytesRead < aLength) {
rv = resource->Read(reinterpret_cast<char*>(data + bytesRead),
aLength - bytesRead, &size);
if (NS_FAILED(rv) || size == 0)
break;
bytesRead += size;
}
GST_BUFFER_SIZE(buffer) = bytesRead;
GstFlowReturn ret = gst_app_src_push_buffer(mSource, gst_buffer_ref(buffer));
if (ret != GST_FLOW_OK) {
LOG(PR_LOG_ERROR, ("ReadAndPushData push ret %s", gst_flow_get_name(ret)));
}
if (GST_BUFFER_SIZE (buffer) < aLength) {
/* If we read less than what we wanted, we reached the end */
gst_app_src_end_of_stream(mSource);
}
gst_buffer_unref(buffer);
}
int64_t GStreamerReader::QueryDuration()
{
gint64 duration = 0;
GstFormat format = GST_FORMAT_TIME;
if (gst_element_query_duration(GST_ELEMENT(mPlayBin),
&format, &duration)) {
if (format == GST_FORMAT_TIME) {
LOG(PR_LOG_DEBUG, ("pipeline duration %" GST_TIME_FORMAT,
GST_TIME_ARGS (duration)));
duration = GST_TIME_AS_USECONDS (duration);
}
}
{
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
int64_t media_duration = mDecoder->GetMediaDuration();
if (media_duration != -1 && media_duration > duration) {
// We decoded more than the reported duration (which could be estimated)
LOG(PR_LOG_DEBUG, ("decoded duration > estimated duration"));
duration = media_duration;
}
}
return duration;
}
void GStreamerReader::NeedDataCb(GstAppSrc* aSrc,
guint aLength,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->NeedData(aSrc, aLength);
}
void GStreamerReader::NeedData(GstAppSrc* aSrc, guint aLength)
{
if (aLength == static_cast<guint>(-1))
aLength = DEFAULT_SOURCE_READ_SIZE;
ReadAndPushData(aLength);
}
void GStreamerReader::EnoughDataCb(GstAppSrc* aSrc, gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->EnoughData(aSrc);
}
void GStreamerReader::EnoughData(GstAppSrc* aSrc)
{
}
gboolean GStreamerReader::SeekDataCb(GstAppSrc* aSrc,
guint64 aOffset,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
return reader->SeekData(aSrc, aOffset);
}
gboolean GStreamerReader::SeekData(GstAppSrc* aSrc, guint64 aOffset)
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
MediaResource* resource = mDecoder->GetResource();
int64_t resourceLength = resource->GetLength();
if (gst_app_src_get_size(mSource) == -1) {
/* It's possible that we didn't know the length when we initialized mSource
* but maybe we do now
*/
gst_app_src_set_size(mSource, resourceLength);
}
nsresult rv = NS_ERROR_FAILURE;
if (aOffset < static_cast<guint64>(resourceLength)) {
rv = resource->Seek(SEEK_SET, aOffset);
}
if (NS_FAILED(rv)) {
LOG(PR_LOG_ERROR, ("seek at %lu failed", aOffset));
}
return NS_SUCCEEDED(rv);
}
gboolean GStreamerReader::EventProbeCb(GstPad* aPad,
GstEvent* aEvent,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
return reader->EventProbe(aPad, aEvent);
}
gboolean GStreamerReader::EventProbe(GstPad* aPad, GstEvent* aEvent)
{
GstElement* parent = GST_ELEMENT(gst_pad_get_parent(aPad));
switch(GST_EVENT_TYPE(aEvent)) {
case GST_EVENT_NEWSEGMENT:
{
gboolean update;
gdouble rate;
GstFormat format;
gint64 start, stop, position;
GstSegment* segment;
/* Store the segments so we can convert timestamps to stream time, which
* is what the upper layers sync on.
*/
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
gst_event_parse_new_segment(aEvent, &update, &rate, &format,
&start, &stop, &position);
if (parent == GST_ELEMENT(mVideoAppSink))
segment = &mVideoSegment;
else
segment = &mAudioSegment;
gst_segment_set_newsegment(segment, update, rate, format,
start, stop, position);
break;
}
case GST_EVENT_FLUSH_STOP:
/* Reset on seeks */
ResetDecode();
break;
default:
break;
}
gst_object_unref(parent);
return TRUE;
}
GstFlowReturn GStreamerReader::AllocateVideoBufferFull(GstPad* aPad,
guint64 aOffset,
guint aSize,
GstCaps* aCaps,
GstBuffer** aBuf,
nsRefPtr<PlanarYCbCrImage>& aImage)
{
/* allocate an image using the container */
ImageContainer* container = mDecoder->GetImageContainer();
if (!container) {
// We don't have an ImageContainer. We probably belong to an <audio>
// element.
return GST_FLOW_NOT_SUPPORTED;
}
PlanarYCbCrImage* img =
reinterpret_cast<PlanarYCbCrImage*>(
container->CreateImage(ImageFormat::PLANAR_YCBCR).get());
nsRefPtr<PlanarYCbCrImage> image = dont_AddRef(img);
/* prepare a GstBuffer pointing to the underlying PlanarYCbCrImage buffer */
GstBuffer* buf = GST_BUFFER(gst_moz_video_buffer_new());
GST_BUFFER_SIZE(buf) = aSize;
/* allocate the actual YUV buffer */
GST_BUFFER_DATA(buf) = image->AllocateAndGetNewBuffer(aSize);
aImage = image;
/* create a GstMozVideoBufferData to hold the image */
GstMozVideoBufferData* bufferdata = new GstMozVideoBufferData(image);
/* Attach bufferdata to our GstMozVideoBuffer, it will take care to free it */
gst_moz_video_buffer_set_data(GST_MOZ_VIDEO_BUFFER(buf), bufferdata);
*aBuf = buf;
return GST_FLOW_OK;
}
GstFlowReturn GStreamerReader::AllocateVideoBufferCb(GstPad* aPad,
guint64 aOffset,
guint aSize,
GstCaps* aCaps,
GstBuffer** aBuf)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(gst_pad_get_element_private(aPad));
return reader->AllocateVideoBuffer(aPad, aOffset, aSize, aCaps, aBuf);
}
GstFlowReturn GStreamerReader::AllocateVideoBuffer(GstPad* aPad,
guint64 aOffset,
guint aSize,
GstCaps* aCaps,
GstBuffer** aBuf)
{
nsRefPtr<PlanarYCbCrImage> image;
return AllocateVideoBufferFull(aPad, aOffset, aSize, aCaps, aBuf, image);
}
GstFlowReturn GStreamerReader::NewPrerollCb(GstAppSink* aSink,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
if (aSink == reader->mVideoAppSink)
reader->VideoPreroll();
else
reader->AudioPreroll();
return GST_FLOW_OK;
}
void GStreamerReader::AudioPreroll()
{
/* The first audio buffer has reached the audio sink. Get rate and channels */
LOG(PR_LOG_DEBUG, ("Audio preroll"));
GstPad* sinkpad = gst_element_get_pad(GST_ELEMENT(mAudioAppSink), "sink");
GstCaps* caps = gst_pad_get_negotiated_caps(sinkpad);
GstStructure* s = gst_caps_get_structure(caps, 0);
mInfo.mAudio.mRate = mInfo.mAudio.mChannels = 0;
gst_structure_get_int(s, "rate", (gint*) &mInfo.mAudio.mRate);
gst_structure_get_int(s, "channels", (gint*) &mInfo.mAudio.mChannels);
NS_ASSERTION(mInfo.mAudio.mRate != 0, ("audio rate is zero"));
NS_ASSERTION(mInfo.mAudio.mChannels != 0, ("audio channels is zero"));
NS_ASSERTION(mInfo.mAudio.mChannels > 0 && mInfo.mAudio.mChannels <= MAX_CHANNELS,
"invalid audio channels number");
mInfo.mAudio.mHasAudio = true;
gst_caps_unref(caps);
gst_object_unref(sinkpad);
}
void GStreamerReader::VideoPreroll()
{
/* The first video buffer has reached the video sink. Get width and height */
LOG(PR_LOG_DEBUG, ("Video preroll"));
GstPad* sinkpad = gst_element_get_pad(GST_ELEMENT(mVideoAppSink), "sink");
GstCaps* caps = gst_pad_get_negotiated_caps(sinkpad);
gst_video_format_parse_caps(caps, &mFormat, &mPicture.width, &mPicture.height);
GstStructure* structure = gst_caps_get_structure(caps, 0);
gst_structure_get_fraction(structure, "framerate", &fpsNum, &fpsDen);
NS_ASSERTION(mPicture.width && mPicture.height, "invalid video resolution");
mInfo.mVideo.mDisplay = nsIntSize(mPicture.width, mPicture.height);
mInfo.mVideo.mHasVideo = true;
gst_caps_unref(caps);
gst_object_unref(sinkpad);
}
GstFlowReturn GStreamerReader::NewBufferCb(GstAppSink* aSink,
gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
if (aSink == reader->mVideoAppSink)
reader->NewVideoBuffer();
else
reader->NewAudioBuffer();
return GST_FLOW_OK;
}
void GStreamerReader::NewVideoBuffer()
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* We have a new video buffer queued in the video sink. Increment the counter
* and notify the decode thread potentially blocked in DecodeVideoFrame
*/
mDecoder->NotifyDecodedFrames(1, 0);
mVideoSinkBufferCount++;
mon.NotifyAll();
}
void GStreamerReader::NewAudioBuffer()
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* We have a new audio buffer queued in the audio sink. Increment the counter
* and notify the decode thread potentially blocked in DecodeAudioData
*/
mAudioSinkBufferCount++;
mon.NotifyAll();
}
void GStreamerReader::EosCb(GstAppSink* aSink, gpointer aUserData)
{
GStreamerReader* reader = reinterpret_cast<GStreamerReader*>(aUserData);
reader->Eos();
}
void GStreamerReader::Eos()
{
/* We reached the end of the stream */
{
ReentrantMonitorAutoEnter mon(mGstThreadsMonitor);
/* Potentially unblock DecodeVideoFrame and DecodeAudioData */
mReachedEos = true;
mon.NotifyAll();
}
{
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
/* Potentially unblock the decode thread in ::DecodeLoop */
mon.NotifyAll();
}
}
/**
* This callback is called while the pipeline is automatically built, after a
* new element has been added to the pipeline. We use it to find the
* uridecodebin instance used by playbin and connect to it to apply our
* whitelist.
*/
void
GStreamerReader::PlayElementAddedCb(GstBin *aBin, GstElement *aElement,
gpointer *aUserData)
{
const static char sUriDecodeBinPrefix[] = "uridecodebin";
gchar *name = gst_element_get_name(aElement);
// Attach this callback to uridecodebin, child of playbin.
if (!strncmp(name, sUriDecodeBinPrefix, sizeof(sUriDecodeBinPrefix) - 1)) {
g_signal_connect(G_OBJECT(aElement), "autoplug-sort",
G_CALLBACK(GStreamerReader::AutoplugSortCb), aUserData);
}
g_free(name);
}
bool
GStreamerReader::ShouldAutoplugFactory(GstElementFactory* aFactory, GstCaps* aCaps)
{
bool autoplug;
const gchar *klass = gst_element_factory_get_klass(aFactory);
if (strstr(klass, "Demuxer") && !strstr(klass, "Metadata")) {
autoplug = GStreamerFormatHelper::Instance()->CanHandleContainerCaps(aCaps);
} else if (strstr(klass, "Decoder") && !strstr(klass, "Generic")) {
autoplug = GStreamerFormatHelper::Instance()->CanHandleCodecCaps(aCaps);
} else {
/* we only filter demuxers and decoders, let everything else be autoplugged */
autoplug = true;
}
return autoplug;
}
/**
* This is called by uridecodebin (running inside playbin), after it has found
* candidate factories to continue decoding the stream. We apply the whitelist
* here, allowing only demuxers and decoders that output the formats we want to
* support.
*/
GValueArray*
GStreamerReader::AutoplugSortCb(GstElement* aElement, GstPad* aPad,
GstCaps* aCaps, GValueArray* aFactories)
{
if (!aFactories->n_values) {
return nullptr;
}
/* aFactories[0] is the element factory that is going to be used to
* create the next element needed to demux or decode the stream.
*/
GstElementFactory *factory = (GstElementFactory*) g_value_get_object(g_value_array_get_nth(aFactories, 0));
if (!ShouldAutoplugFactory(factory, aCaps)) {
/* We don't support this factory. Return an empty array to signal that we
* don't want to continue decoding this (sub)stream.
*/
return g_value_array_new(0);
}
/* nullptr means that we're ok with the candidates and don't need to apply any
* sorting/filtering.
*/
return nullptr;
}
/**
* If this is an MP3 stream, pass any new data we get to the MP3 frame parser
* for duration estimation.
*/
void GStreamerReader::NotifyDataArrived(const char *aBuffer,
uint32_t aLength,
int64_t aOffset)
{
MOZ_ASSERT(NS_IsMainThread());
if (HasVideo()) {
return;
}
if (!mMP3FrameParser.NeedsData()) {
return;
}
mMP3FrameParser.Parse(aBuffer, aLength, aOffset);
int64_t duration = mMP3FrameParser.GetDuration();
if (duration != mLastParserDuration && mUseParserDuration) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mLastParserDuration = duration;
mDecoder->UpdateEstimatedMediaDuration(mLastParserDuration);
}
}
} // namespace mozilla
Reference in New Issue View Git Blame Copy Permalink