From 730949b4e176fb059b51f4f6a05dcc115eeed52c Mon Sep 17 00:00:00 2001 From: Tatiana Meshkova Date: Tue, 3 Apr 2012 14:57:22 -0700 Subject: [PATCH] Bug 486918. Part 2: Add the ability to pre-downscale using a high-quality scaler on a separate thread. r=joe,jlebar --HG-- extra : rebase_source : fa22f42c832ae57b8e221ee03c68a45d33a929d3 --- gfx/thebes/gfx2DGlue.h | 17 ++ image/decoders/nsBMPDecoder.cpp | 2 +- image/decoders/nsJPEGDecoder.cpp | 2 +- image/decoders/nsPNGDecoder.cpp | 2 +- image/src/RasterImage.cpp | 295 ++++++++++++++++++++++++++++++- image/src/RasterImage.h | 107 +++++++++++ 6 files changed, 414 insertions(+), 11 deletions(-) diff --git a/gfx/thebes/gfx2DGlue.h b/gfx/thebes/gfx2DGlue.h index eaa307ddd0e..522ad547c25 100644 --- a/gfx/thebes/gfx2DGlue.h +++ b/gfx/thebes/gfx2DGlue.h @@ -196,6 +196,23 @@ inline gfxASurface::gfxImageFormat SurfaceFormatToImageFormat(SurfaceFormat aFor } } +inline SurfaceFormat ImageFormatToSurfaceFormat(gfxASurface::gfxImageFormat aFormat) +{ + switch (aFormat) { + case gfxASurface::ImageFormatARGB32: + return FORMAT_B8G8R8A8; + case gfxASurface::ImageFormatRGB24: + return FORMAT_B8G8R8X8; + case gfxASurface::ImageFormatRGB16_565: + return FORMAT_R5G6B5; + case gfxASurface::ImageFormatA8: + return FORMAT_A8; + default: + case gfxASurface::ImageFormatUnknown: + return FORMAT_B8G8R8A8; + } +} + inline gfxASurface::gfxContentType ContentForFormat(const SurfaceFormat &aFormat) { switch (aFormat) { diff --git a/image/decoders/nsBMPDecoder.cpp b/image/decoders/nsBMPDecoder.cpp index d6d9377a289..586b4f30fea 100644 --- a/image/decoders/nsBMPDecoder.cpp +++ b/image/decoders/nsBMPDecoder.cpp @@ -9,13 +9,13 @@ #include +#include "ImageLogging.h" #include "EndianMacros.h" #include "nsBMPDecoder.h" #include "nsIInputStream.h" #include "RasterImage.h" #include "imgIContainerObserver.h" -#include "ImageLogging.h" namespace mozilla { namespace image { diff --git a/image/decoders/nsJPEGDecoder.cpp b/image/decoders/nsJPEGDecoder.cpp index 8fa8200f217..749103413aa 100644 --- a/image/decoders/nsJPEGDecoder.cpp +++ b/image/decoders/nsJPEGDecoder.cpp @@ -4,8 +4,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "nsJPEGDecoder.h" #include "ImageLogging.h" +#include "nsJPEGDecoder.h" #include "imgIContainerObserver.h" diff --git a/image/decoders/nsPNGDecoder.cpp b/image/decoders/nsPNGDecoder.cpp index 67839df6988..70b7466982a 100644 --- a/image/decoders/nsPNGDecoder.cpp +++ b/image/decoders/nsPNGDecoder.cpp @@ -4,8 +4,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "nsPNGDecoder.h" #include "ImageLogging.h" +#include "nsPNGDecoder.h" #include "nsMemory.h" #include "nsRect.h" diff --git a/image/src/RasterImage.cpp b/image/src/RasterImage.cpp index f86b07009a7..d061f2138f5 100644 --- a/image/src/RasterImage.cpp +++ b/image/src/RasterImage.cpp @@ -4,6 +4,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "base/histogram.h" +#include "ImageLogging.h" #include "nsComponentManagerUtils.h" #include "imgIContainerObserver.h" #include "nsError.h" @@ -16,7 +17,6 @@ #include "nsStringStream.h" #include "prmem.h" #include "prenv.h" -#include "ImageLogging.h" #include "ImageContainer.h" #include "Layers.h" @@ -28,12 +28,66 @@ #include "nsIconDecoder.h" #include "gfxContext.h" +#include "gfx2DGlue.h" #include "mozilla/Preferences.h" #include "mozilla/StandardInteger.h" #include "mozilla/Telemetry.h" #include "mozilla/TimeStamp.h" #include "mozilla/ClearOnShutdown.h" +#include "mozilla/gfx/Scale.h" + +// The high-quality scaler requires Skia. +#ifdef MOZ_ENABLE_SKIA + +static bool +ScaleFrameImage(imgFrame *aSrcFrame, imgFrame *aDstFrame, + const gfxSize &aScaleFactors) +{ + if (aScaleFactors.width <= 0 || aScaleFactors.height <= 0) + return false; + + imgFrame *srcFrame = aSrcFrame; + nsIntRect srcRect = srcFrame->GetRect(); + uint32_t dstWidth = NSToIntRoundUp(srcRect.width * aScaleFactors.width); + uint32_t dstHeight = NSToIntRoundUp(srcRect.height * aScaleFactors.height); + + // Destination is unconditionally ARGB32 because that's what the scaler + // outputs. + nsresult rv = aDstFrame->Init(0, 0, dstWidth, dstHeight, + gfxASurface::ImageFormatARGB32); + if (!NS_FAILED(rv)) { + uint8_t* srcData; + uint32_t srcDataLength; + // Source frame data is locked/unlocked on the main thread. + srcFrame->GetImageData(&srcData, &srcDataLength); + NS_ASSERTION(srcData != nullptr, "Source data is unavailable! Is it locked?"); + + uint8_t* dstData; + uint32_t dstDataLength; + aDstFrame->LockImageData(); + aDstFrame->GetImageData(&dstData, &dstDataLength); + + // This returns an SkBitmap backed by dstData; since it wrote to dstData, + // we don't need to look at that SkBitmap. + mozilla::gfx::Scale(srcData, srcRect.width, srcRect.height, aSrcFrame->GetImageBytesPerRow(), + dstData, dstWidth, dstHeight, aDstFrame->GetImageBytesPerRow(), + mozilla::gfx::ImageFormatToSurfaceFormat(aSrcFrame->GetFormat())); + + aDstFrame->UnlockImageData(); + return true; + } + + return false; +} +#else // MOZ_ENABLE_SKIA +static bool +ScaleFrameImage(imgFrame *aSrcFrame, imgFrame *aDstFrame, + const gfxSize &aScaleFactors) +{ + return false; +} +#endif // MOZ_ENABLE_SKIA using namespace mozilla; using namespace mozilla::image; @@ -136,6 +190,12 @@ namespace image { /* static */ StaticRefPtr RasterImage::DecodeWorker::sSingleton; +#define PRE_DOWNSCALE_MIN_FACTOR 0.9 + +/* static */ nsRefPtr RasterImage::ScaleWorker::sSingleton; +/* static */ nsRefPtr RasterImage::DrawWorker::sSingleton; +static nsCOMPtr sScaleWorkerThread = nullptr; + #ifndef DEBUG NS_IMPL_ISUPPORTS3(RasterImage, imgIContainer, nsIProperties, nsISupportsWeakReference) @@ -170,7 +230,8 @@ RasterImage::RasterImage(imgStatusTracker* aStatusTracker) : mInDecoder(false), mAnimationFinished(false), mFinishing(false), - mInUpdateImageContainer(false) + mInUpdateImageContainer(false), + mScaleRequest(this) { // Set up the discard tracker node. mDiscardTrackerNode.img = this; @@ -184,6 +245,8 @@ RasterImage::RasterImage(imgStatusTracker* aStatusTracker) : //****************************************************************************** RasterImage::~RasterImage() { + ScaleRequest::Stop(mScaleRequest.image); + delete mAnim; for (unsigned int i = 0; i < mFrames.Length(); ++i) @@ -226,6 +289,8 @@ RasterImage::Initialize() // Create our singletons now, so we don't have to worry about what thread // they're created on. DecodeWorker::Singleton(); + DrawWorker::Singleton(); + ScaleWorker::Singleton(); } nsresult @@ -2584,6 +2649,219 @@ RasterImage::SyncDecode() return mError ? NS_ERROR_FAILURE : NS_OK; } +/* static */ RasterImage::ScaleWorker* +RasterImage::ScaleWorker::Singleton() +{ + if (!sSingleton) { + sSingleton = new ScaleWorker(); + ClearOnShutdown(&sSingleton); + } + + return sSingleton; +} + +nsresult +RasterImage::ScaleWorker::Run() +{ + if (!mInitialized) { + PR_SetCurrentThreadName("Image Scaler"); + mInitialized = true; + } + + ScaleRequest* request; + gfxSize scale; + imgFrame* frame; + { + MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex); + request = mScaleRequests.popFirst(); + if (!request) + return NS_OK; + + scale = request->scale; + frame = request->srcFrame; + } + + nsAutoPtr scaledFrame(new imgFrame()); + bool scaled = ScaleFrameImage(frame, scaledFrame, scale); + + // OK, we've got a new scaled image. Let's get the main thread to unlock and + // redraw it. + { + MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex); + if (scaled && scale == request->scale && !request->isInList()) { + request->dstFrame = scaledFrame; + request->done = true; + } + + DrawWorker::Singleton()->RequestDraw(request->image); + } + return NS_OK; +} + +// Note: you MUST call RequestScale with the ScaleWorker mutex held. +void +RasterImage::ScaleWorker::RequestScale(RasterImage* aImg) +{ + mRequestsMutex.AssertCurrentThreadOwns(); + + ScaleRequest* request = &aImg->mScaleRequest; + if (request->isInList()) + return; + + mScaleRequests.insertBack(request); + + if (!sScaleWorkerThread) { + NS_NewThread(getter_AddRefs(sScaleWorkerThread), this, NS_DISPATCH_NORMAL); + ClearOnShutdown(&sScaleWorkerThread); + } + else { + sScaleWorkerThread->Dispatch(this, NS_DISPATCH_NORMAL); + } +} + +/* static */ RasterImage::DrawWorker* +RasterImage::DrawWorker::Singleton() +{ + if (!sSingleton) { + sSingleton = new DrawWorker(); + ClearOnShutdown(&sSingleton); + } + + return sSingleton; +} + +nsresult +RasterImage::DrawWorker::Run() +{ + ScaleRequest* request; + { + MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex); + request = mDrawRequests.popFirst(); + } + if (request) { + // ScaleWorker is finished with this request, so we can unlock the data now. + request->UnlockSourceData(); + // We have to reset dstFrame if request was stopped while ScaleWorker was scaling. + if (request->stopped) { + ScaleRequest::Stop(request->image); + } + nsCOMPtr observer(do_QueryReferent(request->image->mObserver)); + if (request->done && observer) { + imgFrame *scaledFrame = request->dstFrame.get(); + scaledFrame->ImageUpdated(scaledFrame->GetRect()); + nsIntRect frameRect = request->srcFrame->GetRect(); + observer->FrameChanged(nullptr, request->image, &frameRect); + } + } + + return NS_OK; +} + +void +RasterImage::DrawWorker::RequestDraw(RasterImage* aImg) +{ + ScaleRequest* request = &aImg->mScaleRequest; + mDrawRequests.insertBack(request); + NS_DispatchToMainThread(this, NS_DISPATCH_NORMAL); +} + +void +RasterImage::ScaleRequest::Stop(RasterImage* aImg) +{ + ScaleRequest* request = &aImg->mScaleRequest; + // It's safe to unlock source image data only if request is in the list. + // Otherwise we may be reading from the source while performing scaling + // and can't interrupt immediately. + if (request->isInList()) { + request->remove(); + request->UnlockSourceData(); + } + // We have to check if request is finished before dropping the destination + // frame. Otherwise we may be writing to the dest while performing scaling. + if (request->done) { + request->done = false; + request->dstFrame = nullptr; + request->scale.width = 0; + request->scale.height = 0; + } + request->stopped = true; +} + +bool +RasterImage::CanScale(gfxPattern::GraphicsFilter aFilter, + gfxSize aScale) +{ +// The high-quality scaler requires Skia. +#ifdef MOZ_ENABLE_SKIA + return (aFilter == gfxPattern::FILTER_GOOD) && + !mAnim && mDecoded && + (aScale.width <= 1.0 && aScale.height <= 1.0) && + (aScale.width < PRE_DOWNSCALE_MIN_FACTOR || + aScale.height < PRE_DOWNSCALE_MIN_FACTOR); +#else + return false; +#endif +} + +void +RasterImage::DrawWithPreDownscaleIfNeeded(imgFrame *aFrame, + gfxContext *aContext, + gfxPattern::GraphicsFilter aFilter, + const gfxMatrix &aUserSpaceToImageSpace, + const gfxRect &aFill, + const nsIntRect &aSubimage) +{ + imgFrame *frame = aFrame; + nsIntRect framerect = frame->GetRect(); + gfxMatrix userSpaceToImageSpace = aUserSpaceToImageSpace; + gfxMatrix imageSpaceToUserSpace = aUserSpaceToImageSpace; + imageSpaceToUserSpace.Invert(); + gfxSize scale = imageSpaceToUserSpace.ScaleFactors(true); + nsIntRect subimage = aSubimage; + + if (CanScale(aFilter, scale)) { + MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex); + // If scale factor is still the same that we scaled for and + // ScaleWorker has done it's job, then we can use pre-downscaled frame. + // If scale factor has changed, order new request. + if (mScaleRequest.scale == scale) { + if (mScaleRequest.done) { + frame = mScaleRequest.dstFrame.get(); + userSpaceToImageSpace.Multiply(gfxMatrix().Scale(scale.width, scale.height)); + + // Since we're switching to a scaled image, we we need to transform the + // area of the subimage to draw accordingly, since imgFrame::Draw() + // doesn't know about scaled frames. + subimage.ScaleRoundOut(scale.width, scale.height); + } + } else { + // FIXME: Current implementation doesn't support pre-downscale + // mechanism for multiple images from same src, since we cache + // pre-downscaled frame only for the latest requested scale. + // The solution is to cache more than one scaled image frame + // for each RasterImage. + int scaling = mScaleRequest.srcDataLocked ? 1 : 0; + if (mLockCount - scaling == 1) { + ScaleRequest::Stop(this); + mScaleRequest.srcFrame = frame; + mScaleRequest.scale = scale; + mScaleRequest.stopped = false; + + // We need to make sure that source data is available before asking to scale. + if (mScaleRequest.LockSourceData()) { + ScaleWorker::Singleton()->RequestScale(this); + } + } + } + } + + nsIntMargin padding(framerect.x, framerect.y, + mSize.width - framerect.XMost(), + mSize.height - framerect.YMost()); + + frame->Draw(aContext, aFilter, userSpaceToImageSpace, aFill, padding, subimage); +} + //****************************************************************************** /* [noscript] void draw(in gfxContext aContext, * in gfxGraphicsFilter aFilter, @@ -2655,12 +2933,7 @@ RasterImage::Draw(gfxContext *aContext, return NS_OK; // Getting the frame (above) touches the image and kicks off decoding } - nsIntRect framerect = frame->GetRect(); - nsIntMargin padding(framerect.x, framerect.y, - mSize.width - framerect.XMost(), - mSize.height - framerect.YMost()); - - frame->Draw(aContext, aFilter, aUserSpaceToImageSpace, aFill, padding, aSubimage, aFlags); + DrawWithPreDownscaleIfNeeded(frame, aContext, aFilter, aUserSpaceToImageSpace, aFill, aSubimage); if (mDecoded && !mDrawStartTime.IsNull()) { TimeDuration drawLatency = TimeStamp::Now() - mDrawStartTime; @@ -2668,6 +2941,7 @@ RasterImage::Draw(gfxContext *aContext, // clear the value of mDrawStartTime mDrawStartTime = TimeStamp(); } + return NS_OK; } @@ -2716,6 +2990,11 @@ RasterImage::UnlockImage() // Decrement our lock count mLockCount--; + if (ScaleWorker::sSingleton && mLockCount == 0) { + MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex); + ScaleRequest::Stop(this); + } + // If we've decoded this image once before, we're currently decoding again, // and our lock count is now zero (so nothing is forcing us to keep the // decoded data around), try to cancel the decode and throw away whatever diff --git a/image/src/RasterImage.h b/image/src/RasterImage.h index 28d04e55a4f..0d2c7dece6d 100644 --- a/image/src/RasterImage.h +++ b/image/src/RasterImage.h @@ -17,6 +17,7 @@ #ifndef mozilla_imagelib_RasterImage_h_ #define mozilla_imagelib_RasterImage_h_ +#include "mozilla/Mutex.h" #include "Image.h" #include "nsCOMArray.h" #include "nsCOMPtr.h" @@ -470,6 +471,109 @@ private: bool mPendingInEventLoop; }; + struct ScaleRequest : public LinkedListElement + { + ScaleRequest(RasterImage* aImage) + : image(aImage) + , srcFrame(nullptr) + , dstFrame(nullptr) + , scale(0, 0) + , done(false) + , stopped(false) + , srcDataLocked(false) + {}; + + bool LockSourceData() + { + if (!srcDataLocked) { + bool success = true; + success = success && NS_SUCCEEDED(image->LockImage()); + success = success && NS_SUCCEEDED(srcFrame->LockImageData()); + srcDataLocked = success; + } + return srcDataLocked; + } + + bool UnlockSourceData() + { + bool success = true; + if (srcDataLocked) { + success = success && NS_SUCCEEDED(image->UnlockImage()); + success = success && NS_SUCCEEDED(srcFrame->UnlockImageData()); + + // If unlocking fails, there's nothing we can do to make it work, so we + // claim that we're not locked regardless. + srcDataLocked = false; + } + return success; + } + + static void Stop(RasterImage* aImg); + + RasterImage* const image; + imgFrame *srcFrame; + nsAutoPtr dstFrame; + gfxSize scale; + bool done; + bool stopped; + bool srcDataLocked; + }; + + class ScaleWorker : public nsRunnable + { + public: + static ScaleWorker* Singleton(); + + NS_IMETHOD Run(); + + /* statics */ + static nsRefPtr sSingleton; + + private: /* methods */ + ScaleWorker() + : mRequestsMutex("RasterImage.ScaleWorker.mRequestsMutex") + , mInitialized(false) + {}; + + // Note: you MUST call RequestScale with the ScaleWorker mutex held. + void RequestScale(RasterImage* aImg); + + private: /* members */ + + friend class RasterImage; + LinkedList mScaleRequests; + Mutex mRequestsMutex; + bool mInitialized; + }; + + class DrawWorker : public nsRunnable + { + public: + static DrawWorker* Singleton(); + + NS_IMETHOD Run(); + + /* statics */ + static nsRefPtr sSingleton; + + private: /* methods */ + DrawWorker() {}; + + void RequestDraw(RasterImage* aImg); + + private: /* members */ + + friend class RasterImage; + LinkedList mDrawRequests; + }; + + void DrawWithPreDownscaleIfNeeded(imgFrame *aFrame, + gfxContext *aContext, + gfxPattern::GraphicsFilter aFilter, + const gfxMatrix &aUserSpaceToImageSpace, + const gfxRect &aFill, + const nsIntRect &aSubimage); + /** * Advances the animation. Typically, this will advance a single frame, but it * may advance multiple frames. This may happen if we have infrequently @@ -675,6 +779,9 @@ private: // data bool IsDecodeFinished(); TimeStamp mDrawStartTime; + inline bool CanScale(gfxPattern::GraphicsFilter aFilter, gfxSize aScale); + ScaleRequest mScaleRequest; + // Decoder shutdown enum eShutdownIntent { eShutdownIntent_Done = 0,