From 3820bf2c1acf00dace28ee43652c69c52bcba1b7 Mon Sep 17 00:00:00 2001 From: Michael Layzell Date: Tue, 22 Sep 2015 13:24:50 -0400 Subject: [PATCH 01/78] Bug 1199288 - Rename mApplyUserSelectStyle to mUserInitiated, r=smaug --- editor/libeditor/nsHTMLEditor.cpp | 2 +- layout/base/nsDocumentViewer.cpp | 2 +- layout/generic/Selection.h | 16 ++++++++-------- layout/generic/nsSelection.cpp | 15 ++++++--------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/editor/libeditor/nsHTMLEditor.cpp b/editor/libeditor/nsHTMLEditor.cpp index 9092ea10467..b1331488148 100644 --- a/editor/libeditor/nsHTMLEditor.cpp +++ b/editor/libeditor/nsHTMLEditor.cpp @@ -3531,7 +3531,7 @@ nsHTMLEditor::SelectAll() nsCOMPtr rootElement = do_QueryInterface(rootContent, &rv); NS_ENSURE_SUCCESS(rv, rv); - Maybe userSelection; + Maybe userSelection; if (!rootContent->IsEditable()) { userSelection.emplace(selection); } diff --git a/layout/base/nsDocumentViewer.cpp b/layout/base/nsDocumentViewer.cpp index 7b2dbfe703b..bf57c617fe8 100644 --- a/layout/base/nsDocumentViewer.cpp +++ b/layout/base/nsDocumentViewer.cpp @@ -2613,7 +2613,7 @@ NS_IMETHODIMP nsDocumentViewer::SelectAll() rv = selection->RemoveAllRanges(); if (NS_FAILED(rv)) return rv; - mozilla::dom::Selection::AutoApplyUserSelectStyle userSelection(selection); + mozilla::dom::Selection::AutoUserInitiated userSelection(selection); rv = selection->SelectAllChildren(bodyNode); return rv; } diff --git a/layout/generic/Selection.h b/layout/generic/Selection.h index 3e3500a14e6..c7d5c5920c2 100644 --- a/layout/generic/Selection.h +++ b/layout/generic/Selection.h @@ -103,7 +103,7 @@ public: nsresult SubtractRange(RangeData* aRange, nsRange* aSubtract, nsTArray* aOutput); /** - * AddItem adds aRange to this Selection. If mApplyUserSelectStyle is true, + * AddItem adds aRange to this Selection. If mUserInitiated is true, * then aRange is first scanned for -moz-user-select:none nodes and split up * into multiple ranges to exclude those before adding the resulting ranges * to this Selection. @@ -219,15 +219,15 @@ public: nsresult NotifySelectionListeners(); - friend struct AutoApplyUserSelectStyle; - struct MOZ_RAII AutoApplyUserSelectStyle + friend struct AutoUserInitiated; + struct MOZ_RAII AutoUserInitiated { - explicit AutoApplyUserSelectStyle(Selection* aSelection - MOZ_GUARD_OBJECT_NOTIFIER_PARAM) - : mSavedValue(aSelection->mApplyUserSelectStyle) + explicit AutoUserInitiated(Selection* aSelection + MOZ_GUARD_OBJECT_NOTIFIER_PARAM) + : mSavedValue(aSelection->mUserInitiated) { MOZ_GUARD_OBJECT_NOTIFIER_INIT; - aSelection->mApplyUserSelectStyle = true; + aSelection->mUserInitiated = true; } AutoRestore mSavedValue; MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER @@ -321,7 +321,7 @@ private: * It determines whether we exclude -moz-user-select:none nodes or not, * as well as whether selectstart events will be fired. */ - bool mApplyUserSelectStyle; + bool mUserInitiated; // Non-zero if we don't want any changes we make to the selection to be // visible to content. If non-zero, content won't be notified about changes. diff --git a/layout/generic/nsSelection.cpp b/layout/generic/nsSelection.cpp index 2d822ce1e71..51fd84d17ae 100644 --- a/layout/generic/nsSelection.cpp +++ b/layout/generic/nsSelection.cpp @@ -350,7 +350,7 @@ struct MOZ_RAII AutoPrepareFocusRange if (aSelection->mFrameSelection->IsUserSelectionReason()) { mUserSelect.emplace(aSelection); } - bool userSelection = aSelection->mApplyUserSelectStyle; + bool userSelection = aSelection->mUserInitiated; nsTArray& ranges = aSelection->mRanges; if (!userSelection || @@ -424,7 +424,7 @@ struct MOZ_RAII AutoPrepareFocusRange } } - Maybe mUserSelect; + Maybe mUserSelect; MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER }; @@ -1682,7 +1682,7 @@ nsFrameSelection::TakeFocus(nsIContent* aNewFocus, if (!mDomSelections[index]) return NS_ERROR_NULL_POINTER; - Maybe userSelect; + Maybe userSelect; if (IsUserSelectionReason()) { userSelect.emplace(mDomSelections[index]); } @@ -3331,7 +3331,7 @@ Selection::Selection() : mCachedOffsetForFrame(nullptr) , mDirection(eDirNext) , mType(nsISelectionController::SELECTION_NORMAL) - , mApplyUserSelectStyle(false) + , mUserInitiated(false) , mSelectionChangeBlockerCount(0) { } @@ -3341,7 +3341,7 @@ Selection::Selection(nsFrameSelection* aList) , mCachedOffsetForFrame(nullptr) , mDirection(eDirNext) , mType(nsISelectionController::SELECTION_NORMAL) - , mApplyUserSelectStyle(false) + , mUserInitiated(false) , mSelectionChangeBlockerCount(0) { } @@ -3700,10 +3700,7 @@ Selection::AddItem(nsRange* aItem, int32_t* aOutIndex, bool aNoStartSelect) NS_ASSERTION(aOutIndex, "aOutIndex can't be null"); - // XXX Rename mApplyUserSelectStyle? Not the best name (as it is also being - // used to detect here whether the event is user initiated for the purposes of - // dispatching the selectstart event). - if (mApplyUserSelectStyle) { + if (mUserInitiated) { nsAutoTArray, 4> rangesToAdd; *aOutIndex = -1; From a99cc91789091ace54c77c6533d00c5c60132ae0 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 23 Sep 2015 18:21:19 -0700 Subject: [PATCH 02/78] Bug 1205942 (part 2) - Fix "always true" warning in child_thread.cc. r=jld. The warning is "the address of NuwaMarkCurrentThread() will always evaluate to 'true'". --- ipc/chromium/src/chrome/common/child_thread.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/ipc/chromium/src/chrome/common/child_thread.cc b/ipc/chromium/src/chrome/common/child_thread.cc index cf2ae3574ad..ccf59da23a1 100644 --- a/ipc/chromium/src/chrome/common/child_thread.cc +++ b/ipc/chromium/src/chrome/common/child_thread.cc @@ -32,7 +32,6 @@ ChildThread::~ChildThread() { bool ChildThread::Run() { bool r = StartWithOptions(options_); #ifdef MOZ_NUWA_PROCESS - NS_ASSERTION(NuwaMarkCurrentThread, "NuwaMarkCurrentThread is not defined!"); if (IsNuwaProcess()) { message_loop()->PostTask(FROM_HERE, NewRunnableFunction(&ChildThread::MarkThread)); From aceeb1777d7067c0dc4ff78307f3304e37de80cd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 23 Sep 2015 11:51:31 -0700 Subject: [PATCH 03/78] Bug 1202561 - Mark no-longer-used constants in jsversion.h. r=jorendorff. --- js/src/jsversion.h | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/js/src/jsversion.h b/js/src/jsversion.h index 7bf677b874b..59fbc3815ce 100644 --- a/js/src/jsversion.h +++ b/js/src/jsversion.h @@ -10,29 +10,26 @@ /* * JS Capability Macros. */ -#define JS_HAS_STR_HTML_HELPERS 1 /* has str.anchor, str.bold, etc. */ +#define JS_HAS_STR_HTML_HELPERS 1 /* (no longer used) */ #define JS_HAS_OBJ_PROTO_PROP 1 /* has o.__proto__ etc. */ #define JS_HAS_OBJ_WATCHPOINT 1 /* has o.watch and o.unwatch */ #define JS_HAS_TOSOURCE 1 /* has Object/Array toSource method */ #define JS_HAS_CATCH_GUARD 1 /* has exception handling catch guard */ #define JS_HAS_UNEVAL 1 /* has uneval() top-level function */ -#define JS_HAS_CONST 1 /* has JS2 const as alternative var */ -#define JS_HAS_FUN_EXPR_STMT 1 /* has function expression statement */ +#define JS_HAS_CONST 1 /* (no longer used) */ +#define JS_HAS_FUN_EXPR_STMT 1 /* (no longer used) */ #define JS_HAS_NO_SUCH_METHOD 1 /* has o.__noSuchMethod__ handler */ #define JS_HAS_FOR_EACH_IN 1 /* has for each (lhs in iterable) */ -#define JS_HAS_GENERATORS 1 /* has yield in generator function */ -#define JS_HAS_BLOCK_SCOPE 1 /* has block scope via let/arraycomp */ -#define JS_HAS_DESTRUCTURING 2 /* has [a,b] = ... or {p:a,q:b} = ... */ +#define JS_HAS_GENERATORS 1 /* (no longer used) */ +#define JS_HAS_BLOCK_SCOPE 1 /* (no longer used) */ +#define JS_HAS_DESTRUCTURING 2 /* (no longer used) */ #define JS_HAS_GENERATOR_EXPRS 1 /* has (expr for (lhs in iterable)) */ #define JS_HAS_EXPR_CLOSURES 1 /* has function (formals) listexpr */ -/* Support for JS_NewGlobalObject. */ +/* (no longer used) */ #define JS_HAS_NEW_GLOBAL_OBJECT 1 -/* Support for JS_MakeSystemObject. */ -#define JS_HAS_MAKE_SYSTEM_OBJECT 1 - -/* Feature-test macro for evolving destructuring support. */ +/* (no longer used) */ #define JS_HAS_DESTRUCTURING_SHORTHAND (JS_HAS_DESTRUCTURING == 2) /* From 9e890d88fe59672fb076a18bb238ff866db45c55 Mon Sep 17 00:00:00 2001 From: Dragana Damjanovic Date: Wed, 23 Sep 2015 09:49:00 +0200 Subject: [PATCH 04/78] Bug 1152046 - Disable ClosingService for Firefox OS. r=mcmanus --- netwerk/base/ClosingService.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/netwerk/base/ClosingService.cpp b/netwerk/base/ClosingService.cpp index b8117dd79b3..81c83523926 100644 --- a/netwerk/base/ClosingService.cpp +++ b/netwerk/base/ClosingService.cpp @@ -26,6 +26,7 @@ public: namespace mozilla { namespace net { +#ifndef MOZ_WIDGET_GONK static PRIOMethods sTcpUdpPRCloseLayerMethods; static PRIOMethods *sTcpUdpPRCloseLayerMethodsPtr = nullptr; static PRDescIdentity sTcpUdpPRCloseLayerId; @@ -68,6 +69,7 @@ TcpUdpPRCloseLayerClose(PRFileDesc *aFd) delete closingLayerSecret; return status; } +#endif //MOZ_WIDGET_GONK ClosingService* ClosingService::sInstance = nullptr; @@ -83,6 +85,7 @@ ClosingService::ClosingService() void ClosingService::Start() { +#ifndef MOZ_WIDGET_GONK if (!sTcpUdpPRCloseLayerMethodsPtr) { sTcpUdpPRCloseLayerId = PR_GetUniqueIdentity("TCP and UDP PRClose layer"); PR_ASSERT(PR_INVALID_IO_LAYER != sTcpUdpPRCloseLayerId); @@ -101,6 +104,7 @@ ClosingService::Start() delete service; } } +#endif } nsresult @@ -119,6 +123,12 @@ ClosingService::StartInternal() nsresult ClosingService::AttachIOLayer(PRFileDesc *aFd) { +#ifdef MOZ_WIDGET_GONK + + return NS_OK; + +#else + if (!sTcpUdpPRCloseLayerMethodsPtr) { return NS_OK; } @@ -143,6 +153,8 @@ ClosingService::AttachIOLayer(PRFileDesc *aFd) PR_DELETE(layer); } return NS_OK; + +#endif //MOZ_WIDGET_GONK } void @@ -171,12 +183,14 @@ ClosingService::PostRequest(PRFileDesc *aFd) void ClosingService::Shutdown() { +#ifndef MOZ_WIDGET_GONK MOZ_ASSERT(NS_IsMainThread()); if (sInstance) { sInstance->ShutdownInternal(); NS_RELEASE(sInstance); } +#endif //MOZ_WIDGET_GONK } void From 86e2fcdbc002301eff570bcbd710ee616961c703 Mon Sep 17 00:00:00 2001 From: Stephanie Ouillon Date: Wed, 23 Sep 2015 07:41:00 +0200 Subject: [PATCH 05/78] Bug 1178533 - Add nsIInstallPackagedWebapp for registering permissions when navigating to signed packages. r=bholley, r=fabrice, r=valentin --- b2g/installer/package-manifest.in | 5 + .../preferences/in-content/advanced.js | 3 +- browser/components/preferences/translation.js | 5 +- .../sessionstore/SessionStorage.jsm | 3 +- browser/installer/package-manifest.in | 5 + caps/BasePrincipal.cpp | 22 ++++ caps/BasePrincipal.h | 1 + caps/nsIScriptSecurityManager.idl | 9 +- caps/nsScriptSecurityManager.cpp | 18 +++ caps/tests/unit/test_origin.js | 5 +- dom/apps/PermissionsInstaller.jsm | 9 +- dom/moz.build | 1 + dom/newapps/InstallPackagedWebapp.js | 63 +++++++++ dom/newapps/InstallPackagedWebapp.js.rej | 12 ++ dom/newapps/InstallPackagedWebapp.manifest | 2 + dom/newapps/interfaces/moz.build | 11 ++ .../interfaces/nsIInstallPackagedWebapp.idl | 13 ++ dom/newapps/moz.build | 14 ++ .../test/xpcshell/0001-1st-round-test.patch | 54 ++++++++ dom/newapps/test/xpcshell/test_install.js | 121 ++++++++++++++++++ dom/newapps/test/xpcshell/xpcshell.ini | 5 + dom/permission/PermissionSettings.jsm | 52 +++++--- dom/push/PushRecord.jsm | 4 +- dom/requestsync/RequestSyncService.jsm | 3 +- .../b2gdroid/installer/package-manifest.in | 4 + mobile/android/installer/package-manifest.in | 4 + netwerk/protocol/http/PackagedAppService.cpp | 43 ++++++- netwerk/protocol/http/PackagedAppService.h | 13 +- toolkit/modules/BrowserUtils.jsm | 24 ---- toolkit/modules/PermissionsUtils.jsm | 2 +- toolkit/modules/PermissionsUtils.jsm.rej | 10 ++ 31 files changed, 475 insertions(+), 65 deletions(-) create mode 100644 dom/newapps/InstallPackagedWebapp.js create mode 100644 dom/newapps/InstallPackagedWebapp.js.rej create mode 100644 dom/newapps/InstallPackagedWebapp.manifest create mode 100644 dom/newapps/interfaces/moz.build create mode 100644 dom/newapps/interfaces/nsIInstallPackagedWebapp.idl create mode 100644 dom/newapps/moz.build create mode 100644 dom/newapps/test/xpcshell/0001-1st-round-test.patch create mode 100644 dom/newapps/test/xpcshell/test_install.js create mode 100644 dom/newapps/test/xpcshell/xpcshell.ini create mode 100644 toolkit/modules/PermissionsUtils.jsm.rej diff --git a/b2g/installer/package-manifest.in b/b2g/installer/package-manifest.in index 7951bb470ff..26f980fd3e0 100644 --- a/b2g/installer/package-manifest.in +++ b/b2g/installer/package-manifest.in @@ -185,6 +185,7 @@ @RESPATH@/components/dom.xpt @RESPATH@/components/dom_activities.xpt @RESPATH@/components/dom_apps.xpt +@RESPATH@/components/dom_newapps.xpt @RESPATH@/components/dom_audiochannel.xpt @RESPATH@/components/dom_base.xpt @RESPATH@/components/dom_system.xpt @@ -727,6 +728,10 @@ @RESPATH@/components/@DLL_PREFIX@mozgnome@DLL_SUFFIX@ #endif +; Signed Packaged Content +@RESPATH@/components/InstallPackagedWebapp.manifest +@RESPATH@/components/InstallPackagedWebapp.js + ; ANGLE on Win32 #ifdef XP_WIN32 #ifndef HAVE_64BIT_BUILD diff --git a/browser/components/preferences/in-content/advanced.js b/browser/components/preferences/in-content/advanced.js index 7ec64ba4e6d..90949210aee 100644 --- a/browser/components/preferences/in-content/advanced.js +++ b/browser/components/preferences/in-content/advanced.js @@ -6,7 +6,6 @@ Components.utils.import("resource://gre/modules/DownloadUtils.jsm"); Components.utils.import("resource://gre/modules/LoadContextInfo.jsm"); Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); -Components.utils.import("resource://gre/modules/BrowserUtils.jsm"); var gAdvancedPane = { _inited: false, @@ -572,7 +571,7 @@ var gAdvancedPane = { var list = document.getElementById("offlineAppsList"); var item = list.selectedItem; var origin = item.getAttribute("origin"); - var principal = BrowserUtils.principalFromOrigin(origin); + var principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); var prompts = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] .getService(Components.interfaces.nsIPromptService); diff --git a/browser/components/preferences/translation.js b/browser/components/preferences/translation.js index b10baf761e3..8341740362a 100644 --- a/browser/components/preferences/translation.js +++ b/browser/components/preferences/translation.js @@ -9,7 +9,6 @@ const {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/BrowserUtils.jsm"); XPCOMUtils.defineLazyGetter(this, "gLangBundle", () => Services.strings.createBundle("chrome://global/locale/languageNames.properties")); @@ -190,7 +189,7 @@ var gTranslationExceptions = { onSiteDeleted: function() { let removedSites = this._siteTree.getSelectedItems(); for (let origin of removedSites) { - let principal = BrowserUtils.principalFromOrigin(origin); + let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); Services.perms.removeFromPrincipal(principal, kPermissionType); } }, @@ -203,7 +202,7 @@ var gTranslationExceptions = { this._siteTree.boxObject.rowCountChanged(0, -removedSites.length); for (let origin of removedSites) { - let principal = BrowserUtils.principalFromOrigin(origin); + let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); Services.perms.removeFromPrincipal(principal, kPermissionType); } diff --git a/browser/components/sessionstore/SessionStorage.jsm b/browser/components/sessionstore/SessionStorage.jsm index 5ed1dbdfa86..a2322e0181f 100644 --- a/browser/components/sessionstore/SessionStorage.jsm +++ b/browser/components/sessionstore/SessionStorage.jsm @@ -9,7 +9,6 @@ this.EXPORTED_SYMBOLS = ["SessionStorage"]; const Cu = Components.utils; const Ci = Components.interfaces; -Cu.import("resource://gre/modules/BrowserUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); @@ -105,7 +104,7 @@ var SessionStorageInternal = { restore: function (aDocShell, aStorageData) { for (let origin of Object.keys(aStorageData)) { let data = aStorageData[origin]; - let principal = BrowserUtils.principalFromOrigin(origin); + let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); let storageManager = aDocShell.QueryInterface(Ci.nsIDOMStorageManager); let window = aDocShell.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIDOMWindow); diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index aaf28ca4a54..3e19abd1d43 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -193,6 +193,7 @@ @RESPATH@/components/dom_messages.xpt #endif @RESPATH@/components/dom_apps.xpt +@RESPATH@/components/dom_newapps.xpt @RESPATH@/components/dom_base.xpt @RESPATH@/components/dom_system.xpt #ifdef MOZ_B2G_BT @@ -636,6 +637,10 @@ @RESPATH@/components/PrivateBrowsing.manifest @RESPATH@/components/PrivateBrowsingTrackingProtectionWhitelist.js +; Signed Packaged Content +@RESPATH@/components/InstallPackagedWebapp.manifest +@RESPATH@/components/InstallPackagedWebapp.js + ; ANGLE GLES-on-D3D rendering library #ifdef MOZ_ANGLE_RENDERER @BINPATH@/libEGL.dll diff --git a/caps/BasePrincipal.cpp b/caps/BasePrincipal.cpp index a864c85a234..4faa72190d8 100644 --- a/caps/BasePrincipal.cpp +++ b/caps/BasePrincipal.cpp @@ -403,6 +403,28 @@ BasePrincipal::CreateCodebasePrincipal(nsIURI* aURI, OriginAttributes& aAttrs) return codebase.forget(); } +already_AddRefed +BasePrincipal::CreateCodebasePrincipal(const nsACString& aOrigin) +{ + MOZ_ASSERT(!StringBeginsWith(aOrigin, NS_LITERAL_CSTRING("[")), + "CreateCodebasePrincipal does not support System and Expanded principals"); + + MOZ_ASSERT(!StringBeginsWith(aOrigin, NS_LITERAL_CSTRING(NS_NULLPRINCIPAL_SCHEME ":")), + "CreateCodebasePrincipal does not support nsNullPrincipal"); + + nsAutoCString originNoSuffix; + mozilla::OriginAttributes attrs; + if (!attrs.PopulateFromOrigin(aOrigin, originNoSuffix)) { + return nullptr; + } + + nsCOMPtr uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), originNoSuffix); + NS_ENSURE_SUCCESS(rv, nullptr); + + return BasePrincipal::CreateCodebasePrincipal(uri, attrs); +} + bool BasePrincipal::AddonAllowsLoad(nsIURI* aURI) { diff --git a/caps/BasePrincipal.h b/caps/BasePrincipal.h index 01ed3927fa8..67b4831de95 100644 --- a/caps/BasePrincipal.h +++ b/caps/BasePrincipal.h @@ -137,6 +137,7 @@ public: static BasePrincipal* Cast(nsIPrincipal* aPrin) { return static_cast(aPrin); } static already_AddRefed CreateCodebasePrincipal(nsIURI* aURI, OriginAttributes& aAttrs); + static already_AddRefed CreateCodebasePrincipal(const nsACString& aOrigin); const OriginAttributes& OriginAttributesRef() { return mOriginAttributes; } uint32_t AppId() const { return mOriginAttributes.mAppId; } diff --git a/caps/nsIScriptSecurityManager.idl b/caps/nsIScriptSecurityManager.idl index 916563d8587..cac4beec6a7 100644 --- a/caps/nsIScriptSecurityManager.idl +++ b/caps/nsIScriptSecurityManager.idl @@ -26,7 +26,7 @@ class DomainPolicyClone; [ptr] native JSObjectPtr(JSObject); [ptr] native DomainPolicyClonePtr(mozilla::dom::DomainPolicyClone); -[scriptable, uuid(6e8a4d1e-d9c6-4d86-bf53-d73f58f36148)] +[scriptable, uuid(b7ae2310-576e-11e5-a837-0800200c9a66)] interface nsIScriptSecurityManager : nsISupports { /** @@ -197,6 +197,13 @@ interface nsIScriptSecurityManager : nsISupports [implicit_jscontext] nsIPrincipal createCodebasePrincipal(in nsIURI uri, in jsval originAttributes); + /** + * Returns a principal whose origin is the one we pass in. + * See nsIPrincipal.idl for a description of origin attributes, and + * ChromeUtils.webidl for a list of origin attributes and their defaults. + */ + nsIPrincipal createCodebasePrincipalFromOrigin(in ACString origin); + /** * Returns a unique nonce principal with |originAttributes|. * See nsIPrincipal.idl for a description of origin attributes, and diff --git a/caps/nsScriptSecurityManager.cpp b/caps/nsScriptSecurityManager.cpp index b9d414b3b2f..1b2b825a95d 100644 --- a/caps/nsScriptSecurityManager.cpp +++ b/caps/nsScriptSecurityManager.cpp @@ -67,6 +67,7 @@ #include "nsContentUtils.h" #include "nsJSUtils.h" #include "nsILoadInfo.h" +#include "nsXPCOMStrings.h" // This should be probably defined on some other place... but I couldn't find it #define WEBAPPS_PERM_NAME "webapps-manage" @@ -1043,6 +1044,23 @@ nsScriptSecurityManager::CreateCodebasePrincipal(nsIURI* aURI, JS::Handle prin = BasePrincipal::CreateCodebasePrincipal(aOrigin); + prin.forget(aPrincipal); + return *aPrincipal ? NS_OK : NS_ERROR_FAILURE; +} + NS_IMETHODIMP nsScriptSecurityManager::CreateNullPrincipal(JS::Handle aOriginAttributes, JSContext* aCx, nsIPrincipal** aPrincipal) diff --git a/caps/tests/unit/test_origin.js b/caps/tests/unit/test_origin.js index 8a84102d2d8..ff2d2449ab5 100644 --- a/caps/tests/unit/test_origin.js +++ b/caps/tests/unit/test_origin.js @@ -3,7 +3,6 @@ const Ci = Components.interfaces; const Cu = Components.utils; Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/BrowserUtils.jsm"); var ssm = Services.scriptSecurityManager; function makeURI(uri) { return Services.io.newURI(uri, null, null); } @@ -28,9 +27,9 @@ function checkOriginAttributes(prin, attrs, suffix) { do_check_eq(prin.originAttributes.inBrowser, attrs.inBrowser || false); do_check_eq(prin.originSuffix, suffix || ''); if (!prin.isNullPrincipal && !prin.origin.startsWith('[')) { - do_check_true(BrowserUtils.principalFromOrigin(prin.origin).equals(prin)); + do_check_true(ssm.createCodebasePrincipalFromOrigin(prin.origin).equals(prin)); } else { - checkThrows(() => BrowserUtils.principalFromOrigin(prin.origin)); + checkThrows(() => ssm.createCodebasePrincipalFromOrigin(prin.origin)); } } diff --git a/dom/apps/PermissionsInstaller.jsm b/dom/apps/PermissionsInstaller.jsm index 208b681dfbf..e3ed3aca83d 100644 --- a/dom/apps/PermissionsInstaller.jsm +++ b/dom/apps/PermissionsInstaller.jsm @@ -164,7 +164,8 @@ this.PermissionsInstaller = { PermissionSettingsModule.getPermission(expandedPermNames[idx], aApp.manifestURL, aApp.origin, - false); + false, + aApp.isCachedPackage); if (permValue === "unknown") { permValue = PERM_TO_STRING[permission]; } @@ -192,7 +193,7 @@ this.PermissionsInstaller = { * The permission value. * @param object aApp * The just-installed app configuration. - * The properties used are manifestURL and origin. + * The properties used are manifestURL, origin, appId, isCachedPackage. * @returns void **/ _setPermission: function setPermission(aPermName, aPermValue, aApp) { @@ -201,7 +202,9 @@ this.PermissionsInstaller = { origin: aApp.origin, manifestURL: aApp.manifestURL, value: aPermValue, - browserFlag: false + browserFlag: false, + localId: aApp.localId, + isCachedPackage: aApp.isCachedPackage, }); } }; diff --git a/dom/moz.build b/dom/moz.build index 4279c8ff757..59c67682ceb 100644 --- a/dom/moz.build +++ b/dom/moz.build @@ -113,6 +113,7 @@ DIRS += [ 'resourcestats', 'manifest', 'vr', + 'newapps', ] if CONFIG['OS_ARCH'] == 'WINNT': diff --git a/dom/newapps/InstallPackagedWebapp.js b/dom/newapps/InstallPackagedWebapp.js new file mode 100644 index 00000000000..5577e23b0db --- /dev/null +++ b/dom/newapps/InstallPackagedWebapp.js @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr, Constructor: CC } = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/AppsUtils.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "PermissionsInstaller", + "resource://gre/modules/PermissionsInstaller.jsm"); + +function debug(aMsg) { + dump("-*-*- InstallPackagedWebapps.js : " + aMsg + "\n"); +} + +function InstallPackagedWebapp() { +} + +InstallPackagedWebapp.prototype = { + QueryInterface: XPCOMUtils.generateQI([Ci.nsIInstallPackagedWebapp]), + classID: Components.ID("{5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f}"), + + /** + * Install permissions for signed packaged web content + * @param string manifestContent + * The manifest content of the cached package. + * @param string aOrigin + * The package origin. + * @param string aManifestURL + * The manifest URL of the package. + * @returns boolean + **/ + + installPackagedWebapp: function(aManifestContent, aOrigin, aManifestURL) { + + try { + let isSuccess = true; + let manifest = JSON.parse(aManifestContent); + + PermissionsInstaller.installPermissions({ + manifest: manifest, + manifestURL: aManifestURL, + origin: aOrigin, + isPreinstalled: false, + isCachedPackage: true + }, false, function() { + Cu.reportError(ex); + }); + + // TODO Bug 1206058 - Register app handlers (system msg) on navigation + // to signed packages. + + return isSuccess; + } + catch(ex) { + Cu.reportError(ex); + return false; + } + }, +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([InstallPackagedWebapp]); diff --git a/dom/newapps/InstallPackagedWebapp.js.rej b/dom/newapps/InstallPackagedWebapp.js.rej new file mode 100644 index 00000000000..4f0861e6c6b --- /dev/null +++ b/dom/newapps/InstallPackagedWebapp.js.rej @@ -0,0 +1,12 @@ +diff a/dom/newapps/InstallPackagedWebapp.js b/dom/newapps/InstallPackagedWebapp.js (rejected hunks) +@@ -50,7 +50,9 @@ InstallPackagedWebapp.prototype = { + origin: aOrigin, + isPreinstalled: false, + isCachedPackage: true +- }, false); ++ }, false, function() { ++ Cu.reportError(ex); ++ }); + + // TODO: register app handlers (system msg) + diff --git a/dom/newapps/InstallPackagedWebapp.manifest b/dom/newapps/InstallPackagedWebapp.manifest new file mode 100644 index 00000000000..62699dc434f --- /dev/null +++ b/dom/newapps/InstallPackagedWebapp.manifest @@ -0,0 +1,2 @@ +component {5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f} InstallPackagedWebapp.js +contract @mozilla.org/newapps/installpackagedwebapp;1 {5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f} diff --git a/dom/newapps/interfaces/moz.build b/dom/newapps/interfaces/moz.build new file mode 100644 index 00000000000..f9eb7a739b0 --- /dev/null +++ b/dom/newapps/interfaces/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsIInstallPackagedWebapp.idl' +] + +XPIDL_MODULE = 'dom_newapps' diff --git a/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl b/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl new file mode 100644 index 00000000000..b61ae1ed9ef --- /dev/null +++ b/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +[scriptable, uuid(3b4b69a0-56dc-11e5-a837-0800200c9a66)] +interface nsIInstallPackagedWebapp : nsISupports +{ + boolean installPackagedWebapp(in string aManifestContent, + in string aOrigin, + in string aManifestURL); +}; diff --git a/dom/newapps/moz.build b/dom/newapps/moz.build new file mode 100644 index 00000000000..f213894e296 --- /dev/null +++ b/dom/newapps/moz.build @@ -0,0 +1,14 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += ['interfaces'] + +EXTRA_COMPONENTS += [ + 'InstallPackagedWebapp.js', + 'InstallPackagedWebapp.manifest', +] + +XPCSHELL_TESTS_MANIFESTS += ['test/xpcshell/xpcshell.ini'] diff --git a/dom/newapps/test/xpcshell/0001-1st-round-test.patch b/dom/newapps/test/xpcshell/0001-1st-round-test.patch new file mode 100644 index 00000000000..421951e9761 --- /dev/null +++ b/dom/newapps/test/xpcshell/0001-1st-round-test.patch @@ -0,0 +1,54 @@ +From d196359810da4b3fb11517b1b3d28dd0a9fcdf39 Mon Sep 17 00:00:00 2001 +From: Stephanie Ouillon +Date: Fri, 18 Sep 2015 15:22:47 +0200 +Subject: [PATCH] 1st round test + +--- + dom/newapps/test/xpcshell/test_install.js | 22 ++++++++++++++++------ + 1 file changed, 16 insertions(+), 6 deletions(-) + +diff --git a/dom/newapps/test/xpcshell/test_install.js b/dom/newapps/test/xpcshell/test_install.js +index b9073a0..53b4a26 100644 +--- a/dom/newapps/test/xpcshell/test_install.js ++++ b/dom/newapps/test/xpcshell/test_install.js +@@ -15,21 +15,31 @@ const mod = Cc['@mozilla.org/newapps/installpackagedwebapp;1'] + function run_test() { + + let manifest = { +- start_url: "start.html", +- launch_path: "other.html" +- }; ++ name: "Test App", ++ launch_path: "/index.html", ++ permissions: { ++ alarms: { }, ++ wifi-manage: { }, ++ tcp-socket: { }, ++ browser: { }, ++ desktop-notification: { }, ++ geolocation: { }, ++ }, ++ } + + //trigger error at install + let aOrigin = ""; + let aManifestURL = ""; + let manifestString = "boum" +- let appId = 123456789; +- let res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL, appId); ++ let res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL); + equal(res, false); + + aOrigin = "http://test.com"; + aManifestURL = "http://test.com/manifest.json"; + manifestString = JSON.stringify(manifest); +- res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL, appId); ++ res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL); + equal(res, true); ++ ++ //test permissions are set ++ //try modifying permissions r/w + } +-- +1.9.1 + diff --git a/dom/newapps/test/xpcshell/test_install.js b/dom/newapps/test/xpcshell/test_install.js new file mode 100644 index 00000000000..344120d99af --- /dev/null +++ b/dom/newapps/test/xpcshell/test_install.js @@ -0,0 +1,121 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +'use strict'; + +const { classes: Cc, interfaces: Ci, utils: Cu } = Components; + +Cu.import('resource://gre/modules/XPCOMUtils.jsm'); +Cu.import('resource://gre/modules/Services.jsm'); +Cu.import("resource://gre/modules/PermissionSettings.jsm"); +Cu.import("resource://gre/modules/PermissionsTable.jsm"); +Cu.import("resource://gre/modules/AppsUtils.jsm"); + +const mod = Cc['@mozilla.org/newapps/installpackagedwebapp;1'] + .getService(Ci.nsIInstallPackagedWebapp); + +XPCOMUtils.defineLazyServiceGetter(this, + "appsService", + "@mozilla.org/AppsService;1", + "nsIAppsService"); + +function run_test() { + + let manifestWithPerms = { + name: "Test App", + launch_path: "/index.html", + type: "privileged", + permissions: { + "alarms": { }, + "wifi-manage": { }, + "tcp-socket": { }, + "desktop-notification": { }, + "geolocation": { }, + }, + }; + + let manifestNoPerms = { + name: "Test App", + launch_path: "/index.html", + type: "privileged", + }; + + let appStatus = "privileged"; + + // Triggering error due to bad manifest + let origin = ""; + let manifestURL = ""; + let manifestString = "boum"; + + let res = mod.installPackagedWebapp(manifestString, origin, manifestURL); + equal(res, false); + + // Install a package with permissions + origin = "http://test.com^appId=1019&inBrowser=1"; + manifestURL = "http://test.com/manifest.json"; + manifestString = JSON.stringify(manifestWithPerms); + let manifestHelper = new ManifestHelper(manifestWithPerms, origin, manifestURL); + + cleanDB(manifestHelper, origin, manifestURL); + + res = mod.installPackagedWebapp(manifestString, origin, manifestURL); + equal(res, true); + checkPermissions(manifestHelper, origin, manifestURL, appStatus); + + // Install a package with permissions + origin = "http://test.com"; + manifestHelper = new ManifestHelper(manifestWithPerms, origin, manifestURL); + + cleanDB(manifestHelper, origin, manifestURL); + + res = mod.installPackagedWebapp(manifestString, origin, manifestURL); + equal(res, true); + checkPermissions(manifestHelper, origin, manifestURL, appStatus); + + + // Install a package with no permission + origin = "http://bar.com^appId=1337&inBrowser=1"; + manifestURL = "http://bar.com/manifest.json"; + manifestString = JSON.stringify(manifestNoPerms); + manifestHelper = new ManifestHelper(manifestNoPerms, origin, manifestURL); + + cleanDB(manifestHelper, origin, manifestURL); + + res = mod.installPackagedWebapp(manifestString, origin, manifestURL); + equal(res, true); + checkPermissions(manifestHelper, origin, manifestURL, appStatus); +} + +// Cleaning permissions database before running a test +function cleanDB(manifestHelper, origin, manifestURL) { + for (let permName in manifestHelper.permissions) { + PermissionSettingsModule.removePermission(permName, manifestURL, origin, "", true); + } +} + +// Check permissions are correctly set in the database +function checkPermissions(manifestHelper, origin, manifestURL, appStatus) { + let perm; + for (let permName in manifestHelper.permissions) { + let permValue = PermissionSettingsModule.getPermission( + permName, manifestURL, origin, "", true); + switch (PermissionsTable[permName][appStatus]) { + case Ci.nsIPermissionManager.UNKNOWN_ACTION: + perm = "unknown"; + break; + case Ci.nsIPermissionManager.ALLOW_ACTION: + perm = "allow"; + break; + case Ci.nsIPermissionManager.DENY_ACTION: + perm = "deny"; + break; + case Ci.nsIPermissionManager.PROMPT_ACTION: + perm = "prompt"; + break; + default: + break; + } + equal(permValue, perm); + } +} diff --git a/dom/newapps/test/xpcshell/xpcshell.ini b/dom/newapps/test/xpcshell/xpcshell.ini new file mode 100644 index 00000000000..5c1329e6e8b --- /dev/null +++ b/dom/newapps/test/xpcshell/xpcshell.ini @@ -0,0 +1,5 @@ +[DEFAULT] +head = +tail = + +[test_install.js] diff --git a/dom/permission/PermissionSettings.jsm b/dom/permission/PermissionSettings.jsm index 8cda5e09a04..ddd3a19c45d 100644 --- a/dom/permission/PermissionSettings.jsm +++ b/dom/permission/PermissionSettings.jsm @@ -67,13 +67,23 @@ this.PermissionSettingsModule = { _internalAddPermission: function _internalAddPermission(aData, aAllowAllChanges, aCallbacks) { - // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm - let uri = Services.io.newURI(aData.origin, null, null); - let app = appsService.getAppByManifestURL(aData.manifestURL); - let principal = - Services.scriptSecurityManager.createCodebasePrincipal(uri, - {appId: app.localId, - inBrowser: aData.browserFlag}); + // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm. + let app; + let principal; + // Test if app is cached (signed streamable package) or installed via DOMApplicationRegistry + if (aData.isCachedPackage) { + // If the app is from packaged web app, the origin includes origin attributes already. + principal = + Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(aData.origin); + app = {localId: principal.appId}; + } else { + app = appsService.getAppByManifestURL(aData.manifestURL); + let uri = Services.io.newURI(aData.origin, null, null); + principal = + Services.scriptSecurityManager.createCodebasePrincipal(uri, + {appId: app.localId, + inBrowser: aData.browserFlag}); + } let action; switch (aData.value) @@ -106,17 +116,24 @@ this.PermissionSettingsModule = { } }, - getPermission: function getPermission(aPermName, aManifestURL, aOrigin, aBrowserFlag) { + getPermission: function getPermission(aPermName, aManifestURL, aOrigin, aBrowserFlag, aIsCachedPackage) { // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm debug("getPermission: " + aPermName + ", " + aManifestURL + ", " + aOrigin); - let uri = Services.io.newURI(aOrigin, null, null); - let appID = appsService.getAppLocalIdByManifestURL(aManifestURL); - let principal = - Services.scriptSecurityManager.createCodebasePrincipal(uri, - {appId: appID, - inBrowser: aBrowserFlag}); + let principal; + // Test if app is cached (signed streamable package) or installed via DOMApplicationRegistry + if (aIsCachedPackage) { + // If the app is from packaged web app, the origin includes origin attributes already. + principal = + Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(aOrigin); + } else { + let uri = Services.io.newURI(aOrigin, null, null); + let appID = appsService.getAppLocalIdByManifestURL(aManifestURL); + principal = + Services.scriptSecurityManager.createCodebasePrincipal(uri, + {appId: appID, + inBrowser: aBrowserFlag}); + } let result = Services.perms.testExactPermissionFromPrincipal(principal, aPermName); - switch (result) { case Ci.nsIPermissionManager.UNKNOWN_ACTION: @@ -133,13 +150,14 @@ this.PermissionSettingsModule = { } }, - removePermission: function removePermission(aPermName, aManifestURL, aOrigin, aBrowserFlag) { + removePermission: function removePermission(aPermName, aManifestURL, aOrigin, aBrowserFlag, aIsCachedPackage) { let data = { type: aPermName, origin: aOrigin, manifestURL: aManifestURL, value: "unknown", - browserFlag: aBrowserFlag + browserFlag: aBrowserFlag, + isCachedPackage: aIsCachedPackage }; this._internalAddPermission(data, true); }, diff --git a/dom/push/PushRecord.jsm b/dom/push/PushRecord.jsm index a468ddd39cd..b2e3c0e2415 100644 --- a/dom/push/PushRecord.jsm +++ b/dom/push/PushRecord.jsm @@ -19,8 +19,6 @@ XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils", XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils", "resource://gre/modules/PrivateBrowsingUtils.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "BrowserUtils", - "resource://gre/modules/BrowserUtils.jsm"); this.EXPORTED_SYMBOLS = ["PushRecord"]; @@ -218,7 +216,7 @@ Object.defineProperties(PushRecord.prototype, { // Allow tests to omit origin attributes. url += this.originAttributes; } - principal = BrowserUtils.principalFromOrigin(url); + principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(url); principals.set(this, principal); } return principal; diff --git a/dom/requestsync/RequestSyncService.jsm b/dom/requestsync/RequestSyncService.jsm index c771f6438e1..4e6bdda0697 100644 --- a/dom/requestsync/RequestSyncService.jsm +++ b/dom/requestsync/RequestSyncService.jsm @@ -20,7 +20,6 @@ const RSYNC_STATE_ENABLED = "enabled"; const RSYNC_STATE_DISABLED = "disabled"; const RSYNC_STATE_WIFIONLY = "wifiOnly"; -Cu.import("resource://gre/modules/BrowserUtils.jsm"); Cu.import('resource://gre/modules/IndexedDBHelper.jsm'); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); @@ -175,7 +174,7 @@ this.RequestSyncService = { let dbKeys = []; for (let key in this._registrations) { - let prin = BrowserUtils.principalFromOrigin(key); + let prin = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(key); if (!ChromeUtils.originAttributesMatchPattern(prin.originAttributes, pattern)) { continue; } diff --git a/mobile/android/b2gdroid/installer/package-manifest.in b/mobile/android/b2gdroid/installer/package-manifest.in index 94ff64b2f38..10d10196239 100644 --- a/mobile/android/b2gdroid/installer/package-manifest.in +++ b/mobile/android/b2gdroid/installer/package-manifest.in @@ -138,6 +138,7 @@ @BINPATH@/components/dom.xpt @BINPATH@/components/dom_activities.xpt @BINPATH@/components/dom_apps.xpt +@BINPATH@/components/dom_newapps.xpt @BINPATH@/components/dom_base.xpt @BINPATH@/components/dom_canvas.xpt @BINPATH@/components/dom_core.xpt @@ -677,6 +678,9 @@ bin/libfreebl_32int64_3.so @BINPATH@/components/SystemMessageCache.js @BINPATH@/components/SystemMessageManager.manifest +@BINPATH@/components/InstallPackagedWebapp.manifest +@BINPATH@/components/InstallPackagedWebapp.js + @BINPATH@/components/B2GComponents.manifest @BINPATH@/components/AlertsService.js @BINPATH@/components/ContentPermissionPrompt.js diff --git a/mobile/android/installer/package-manifest.in b/mobile/android/installer/package-manifest.in index ce422257b08..8d720b5d314 100644 --- a/mobile/android/installer/package-manifest.in +++ b/mobile/android/installer/package-manifest.in @@ -138,6 +138,7 @@ @BINPATH@/components/dom.xpt @BINPATH@/components/dom_activities.xpt @BINPATH@/components/dom_apps.xpt +@BINPATH@/components/dom_newapps.xpt @BINPATH@/components/dom_base.xpt @BINPATH@/components/dom_canvas.xpt @BINPATH@/components/dom_core.xpt @@ -425,6 +426,9 @@ @BINPATH@/components/SystemMessageCache.js @BINPATH@/components/SystemMessageManager.manifest +@BINPATH@/components/InstallPackagedWebapp.manifest +@BINPATH@/components/InstallPackagedWebapp.js + #ifdef MOZ_WEBRTC @BINPATH@/components/PeerConnection.js @BINPATH@/components/PeerConnection.manifest diff --git a/netwerk/protocol/http/PackagedAppService.cpp b/netwerk/protocol/http/PackagedAppService.cpp index a9205370954..1d5998584f0 100644 --- a/netwerk/protocol/http/PackagedAppService.cpp +++ b/netwerk/protocol/http/PackagedAppService.cpp @@ -17,6 +17,7 @@ #include "mozilla/DebugOnly.h" #include "nsIHttpHeaderVisitor.h" #include "mozilla/LoadContext.h" +#include "nsIInstallPackagedWebapp.h" namespace mozilla { namespace net { @@ -430,6 +431,7 @@ PackagedAppService::PackagedAppDownloader::Init(nsILoadContextInfo* aInfo, mPackageKey = aKey; mPackageOrigin = aPackageOrigin; + mProcessingFirstRequest = true; return NS_OK; } @@ -623,6 +625,8 @@ PackagedAppService::PackagedAppDownloader::OnStopRequest(nsIRequest *aRequest, LOG(("[%p] PackagedAppDownloader::OnStopRequest > status:%X multiChannel:%p\n", this, aStatusCode, multiChannel.get())); + mProcessingFirstRequest = false; + // lastPart will be true if this is the last part in the package, // or if aRequest isn't a multipart channel bool lastPart = true; @@ -703,6 +707,11 @@ PackagedAppService::PackagedAppDownloader::ConsumeData(nsIInputStream *aStream, self->mWriter->ConsumeData(aFromRawSegment, aCount, aWriteCount); + if (self->mProcessingFirstRequest) { + // mProcessingFirstRequest will be set to false on the first OnStopRequest. + self->mManifestContent.Append(aFromRawSegment, aCount); + } + nsCOMPtr stream = CreateSharedStringStream(aFromRawSegment, aCount); return self->mVerifier->OnDataAvailable(nullptr, nullptr, stream, 0, aCount); } @@ -858,11 +867,39 @@ PackagedAppService::PackagedAppDownloader::NotifyOnStartSignedPackageRequest(con LOG(("Notifying the signed package is ready to load.")); } -void PackagedAppService::PackagedAppDownloader::InstallSignedPackagedApp() +void PackagedAppService::PackagedAppDownloader::InstallSignedPackagedApp(const ResourceCacheInfo* aInfo) { // TODO: Bug 1178533 to register permissions, system messages etc on navigation to // signed packages. LOG(("Install this packaged app.")); + bool isSuccess = false; + + nsCOMPtr installer = + do_GetService("@mozilla.org/newapps/installpackagedwebapp;1"); + + if (!installer) { + LOG(("InstallSignedPackagedApp: fail to get InstallPackagedWebapp service")); + return OnError(ERROR_GET_INSTALLER_FAILED); + } + + nsCString manifestURL; + aInfo->mURI->GetAsciiSpec(manifestURL); + + // Use the origin stored in the verifier since the signed packaged app would + // have a specifi package identifer defined in the manifest file. + nsCString packageOrigin; + mVerifier->GetPackageOrigin(packageOrigin); + + installer->InstallPackagedWebapp(mManifestContent.get(), + packageOrigin.get(), + manifestURL.get(), + &isSuccess); + if (!isSuccess) { + LOG(("InstallSignedPackagedApp: failed to install permissions")); + return OnError(ERROR_INSTALL_RESOURCE_FAILED); + } + + LOG(("InstallSignedPackagedApp: success.")); } //------------------------------------------------------------------ @@ -920,7 +957,7 @@ PackagedAppService::PackagedAppDownloader::OnManifestVerified(const ResourceCach nsCString packageOrigin; mVerifier->GetPackageOrigin(packageOrigin); NotifyOnStartSignedPackageRequest(packageOrigin); - InstallSignedPackagedApp(); + InstallSignedPackagedApp(aInfo); } void @@ -1087,7 +1124,7 @@ PackagedAppService::GetResource(nsIChannel *aChannel, downloader = new PackagedAppDownloader(); nsCString packageOrigin; - principal->GetOriginNoSuffix(packageOrigin); + principal->GetOrigin(packageOrigin); rv = downloader->Init(loadContextInfo, key, packageOrigin); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; diff --git a/netwerk/protocol/http/PackagedAppService.h b/netwerk/protocol/http/PackagedAppService.h index 4dd04dd0080..e1ae1b01d25 100644 --- a/netwerk/protocol/http/PackagedAppService.h +++ b/netwerk/protocol/http/PackagedAppService.h @@ -106,6 +106,8 @@ private: enum EErrorType { ERROR_MANIFEST_VERIFIED_FAILED, ERROR_RESOURCE_VERIFIED_FAILED, + ERROR_GET_INSTALLER_FAILED, + ERROR_INSTALL_RESOURCE_FAILED, }; public: @@ -172,7 +174,7 @@ private: // Handle all tasks about app installation like permission and system message // registration. - void InstallSignedPackagedApp(); + void InstallSignedPackagedApp(const ResourceCacheInfo* aInfo); // Calls all the callbacks registered for the given URI. // aURI is the full URI of a subresource, composed of packageURI + !// + subresourcePath @@ -211,6 +213,15 @@ private: // If you need the origin with the signity taken into account, use // PackagedAppVerifier::GetPackageOrigin(). nsCString mPackageOrigin; + + //The app id of the package loaded from the LoadContextInfo + uint32_t mAppId; + + // A flag to indicate if we are processing the first request. + bool mProcessingFirstRequest; + + // A in-memory copy of the manifest content. + nsCString mManifestContent; }; // Intercepts OnStartRequest, OnDataAvailable*, OnStopRequest method calls diff --git a/toolkit/modules/BrowserUtils.jsm b/toolkit/modules/BrowserUtils.jsm index bdeacf91840..920c18ade00 100644 --- a/toolkit/modules/BrowserUtils.jsm +++ b/toolkit/modules/BrowserUtils.jsm @@ -100,30 +100,6 @@ this.BrowserUtils = { return Services.io.newURI(aCPOWURI.spec, aCPOWURI.originCharset, null); }, - // Creates a codebase principal from a canonical origin string. This is - // the inverse operation of .origin on a codebase principal. - principalFromOrigin: function(aOriginString) { - if (aOriginString.startsWith('[')) { - throw new Error("principalFromOrigin does not support System and Expanded principals"); - } - - if (aOriginString.startsWith("moz-nullprincipal:")) { - throw new Error("principalFromOrigin does not support nsNullPrincipal"); - } - - var parts = aOriginString.split('^'); - if (parts.length > 2) { - throw new Error("bad origin string: " + aOriginString); - } - - var uri = Services.io.newURI(parts[0], null, null); - var attrs = {}; - // Parse the parameters string into a dictionary. - (parts[1] || "").split("&").map((x) => x.split('=')).forEach((x) => attrs[x[0]] = x[1]); - - return Services.scriptSecurityManager.createCodebasePrincipal(uri, attrs); - }, - /** * For a given DOM element, returns its position in "screen" * coordinates. In a content process, the coordinates returned will diff --git a/toolkit/modules/PermissionsUtils.jsm b/toolkit/modules/PermissionsUtils.jsm index 3f95437fb24..3f731a3fd94 100644 --- a/toolkit/modules/PermissionsUtils.jsm +++ b/toolkit/modules/PermissionsUtils.jsm @@ -29,7 +29,7 @@ function importPrefBranch(aPrefBranch, aPermission, aAction) { for (let origin of origins) { let principals = []; try { - principals = [ BrowserUtils.principalFromOrigin(origin) ]; + principals = [ Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin) ]; } catch (e) { // This preference used to contain a list of hosts. For back-compat // reasons, we convert these hosts into http:// and https:// permissions diff --git a/toolkit/modules/PermissionsUtils.jsm.rej b/toolkit/modules/PermissionsUtils.jsm.rej new file mode 100644 index 00000000000..2e1e2cbb904 --- /dev/null +++ b/toolkit/modules/PermissionsUtils.jsm.rej @@ -0,0 +1,10 @@ +diff a/toolkit/modules/PermissionsUtils.jsm b/toolkit/modules/PermissionsUtils.jsm (rejected hunks) +@@ -7,8 +7,6 @@ this.EXPORTED_SYMBOLS = ["PermissionsUtils"]; + const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + + Cu.import("resource://gre/modules/Services.jsm"); +-Cu.import("resource://gre/modules/BrowserUtils.jsm") +- + + let gImportedPrefBranches = new Set(); + From a354cb92b834fcb9fe113376c964f746e1e57d86 Mon Sep 17 00:00:00 2001 From: sajitk Date: Tue, 22 Sep 2015 18:16:00 +0200 Subject: [PATCH 06/78] Bug 1180940 - Changed return type of AudioDestinationNode::CreateAudioChannelAgent method to return errors, if any methods that it calls fail. Added code to handle the return value in AudioContext::Init(), and its callers. r=baku --- dom/audiochannel/AudioChannelAgent.cpp | 6 +++++- dom/media/webaudio/AudioContext.cpp | 24 ++++++++++++++++----- dom/media/webaudio/AudioContext.h | 2 +- dom/media/webaudio/AudioDestinationNode.cpp | 20 ++++++++++++----- dom/media/webaudio/AudioDestinationNode.h | 2 +- 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/dom/audiochannel/AudioChannelAgent.cpp b/dom/audiochannel/AudioChannelAgent.cpp index 8c4106b9100..b8f0009fbe8 100644 --- a/dom/audiochannel/AudioChannelAgent.cpp +++ b/dom/audiochannel/AudioChannelAgent.cpp @@ -104,7 +104,7 @@ AudioChannelAgent::InitInternal(nsIDOMWindow* aWindow, int32_t aChannelType, } if (NS_WARN_IF(!aWindow)) { - return NS_ERROR_FAILURE; + return NS_OK; } nsCOMPtr pInnerWindow = do_QueryInterface(aWindow); @@ -113,6 +113,10 @@ AudioChannelAgent::InitInternal(nsIDOMWindow* aWindow, int32_t aChannelType, nsCOMPtr topWindow; aWindow->GetScriptableTop(getter_AddRefs(topWindow)); + if (NS_WARN_IF(!topWindow)) { + return NS_OK; + } + mWindow = do_QueryInterface(topWindow); if (mWindow) { mWindow = mWindow->GetOuterWindow(); diff --git a/dom/media/webaudio/AudioContext.cpp b/dom/media/webaudio/AudioContext.cpp index 4e6a622f355..8a474905f35 100644 --- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -117,14 +117,22 @@ AudioContext::AudioContext(nsPIDOMWindow* aWindow, } } -void +nsresult AudioContext::Init() { // We skip calling SetIsOnlyNodeForContext and the creation of the // audioChannelAgent during mDestination's constructor, because we can only // call them after mDestination has been set up. - mDestination->CreateAudioChannelAgent(); - mDestination->SetIsOnlyNodeForContext(true); + if (mIsOffline) { + mDestination->SetIsOnlyNodeForContext(true); + } else { + nsresult rv = mDestination->CreateAudioChannelAgent(); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + } + + return NS_OK; } AudioContext::~AudioContext() @@ -160,7 +168,10 @@ AudioContext::Constructor(const GlobalObject& aGlobal, nsRefPtr object = new AudioContext(window, false, AudioChannelService::GetDefaultAudioChannel()); - object->Init(); + aRv = object->Init(); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } RegisterWeakMemoryReporter(object); @@ -179,7 +190,10 @@ AudioContext::Constructor(const GlobalObject& aGlobal, } nsRefPtr object = new AudioContext(window, false, aChannel); - object->Init(); + aRv = object->Init(); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } RegisterWeakMemoryReporter(object); diff --git a/dom/media/webaudio/AudioContext.h b/dom/media/webaudio/AudioContext.h index 256010e7d0d..9acc8b8ec77 100644 --- a/dom/media/webaudio/AudioContext.h +++ b/dom/media/webaudio/AudioContext.h @@ -124,7 +124,7 @@ class AudioContext final : public DOMEventTargetHelper, float aSampleRate = 0.0f); ~AudioContext(); - void Init(); + nsresult Init(); public: typedef uint64_t AudioContextId; diff --git a/dom/media/webaudio/AudioDestinationNode.cpp b/dom/media/webaudio/AudioDestinationNode.cpp index cb7bc010a6c..c620c17e773 100644 --- a/dom/media/webaudio/AudioDestinationNode.cpp +++ b/dom/media/webaudio/AudioDestinationNode.cpp @@ -630,23 +630,33 @@ AudioDestinationNode::CheckAudioChannelPermissions(AudioChannel aValue) return perm == nsIPermissionManager::ALLOW_ACTION; } -void +nsresult AudioDestinationNode::CreateAudioChannelAgent() { if (mIsOffline) { - return; + return NS_OK; } + nsresult rv = NS_OK; if (mAudioChannelAgent) { - mAudioChannelAgent->NotifyStoppedPlaying(nsIAudioChannelAgent::AUDIO_AGENT_NOTIFY); + rv = mAudioChannelAgent->NotifyStoppedPlaying(nsIAudioChannelAgent::AUDIO_AGENT_NOTIFY); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } } mAudioChannelAgent = new AudioChannelAgent(); - mAudioChannelAgent->InitWithWeakCallback(GetOwner(), + rv = mAudioChannelAgent->InitWithWeakCallback(GetOwner(), static_cast(mAudioChannel), this); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + rv = WindowAudioCaptureChanged(); + NS_WARN_IF(NS_FAILED(rv)); + return rv; - WindowAudioCaptureChanged(); } void diff --git a/dom/media/webaudio/AudioDestinationNode.h b/dom/media/webaudio/AudioDestinationNode.h index 632c2453e68..44fc06c98c4 100644 --- a/dom/media/webaudio/AudioDestinationNode.h +++ b/dom/media/webaudio/AudioDestinationNode.h @@ -73,7 +73,7 @@ public: // When aIsOnlyNode is true, this is the only node for the AudioContext. void SetIsOnlyNodeForContext(bool aIsOnlyNode); - void CreateAudioChannelAgent(); + nsresult CreateAudioChannelAgent(); void DestroyAudioChannelAgent(); virtual const char* NodeType() const override From 5de881cbe8e1304c6418124fadb8cb580f85c0cc Mon Sep 17 00:00:00 2001 From: William Lachance Date: Mon, 14 Sep 2015 17:52:18 -0400 Subject: [PATCH 07/78] Bug 1200716 - Simplify talos output for perfherder. r=jmaher --- testing/talos/talos/output.py | 4 ++-- testing/talos/talos/results.py | 20 ++++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/testing/talos/talos/output.py b/testing/talos/talos/output.py index c204946f142..cd3316a61ba 100755 --- a/testing/talos/talos/output.py +++ b/testing/talos/talos/output.py @@ -171,7 +171,7 @@ class GraphserverOutput(Output): for result in test.results: filtered_val = result.values(testname, test.test_config['filters']) - vals.extend([[i['filtered'], j] for i, j in filtered_val]) + vals.extend([[i['value'], j] for i, j in filtered_val]) result_strings.append(self.construct_results(vals, testname=testname, **info_dict)) @@ -515,7 +515,7 @@ class PerfherderOutput(Output): filtered_results = \ result.values(test_result['testrun']['suite'], test.test_config['filters']) - vals.extend([[i['filtered'], j] for i, j in filtered_results]) + vals.extend([[i['value'], j] for i, j in filtered_results]) for val, page in filtered_results: if page == 'NULL': summary['subtests'][test.name()] = val diff --git a/testing/talos/talos/results.py b/testing/talos/talos/results.py index a4b17a8aa8d..340b7c028e3 100755 --- a/testing/talos/talos/results.py +++ b/testing/talos/talos/results.py @@ -158,15 +158,6 @@ class Results(object): else: remaining_filters.append(f) - # calculate common numbers with the raw data - data_summary = { - 'min': min(data), - 'max': max(data), - 'mean': filter.mean(data), - 'median': filter.median(data), - 'std': filter.stddev(data) - } - # apply the summarization filters for f in remaining_filters: if f.func.__name__ == "v8_subtest": @@ -174,13 +165,14 @@ class Results(object): data = filter.v8_subtest(data, page) else: data = f.apply(data) - data_summary['filtered'] = data - # special case for dromaeo_dom and v8_7 - if testname == 'dromaeo_dom' or testname.startswith('v8_7'): - data_summary['value'] = data + summary = { + 'filtered': data, # for backwards compatibility with perfherder + 'value': data + } + + retval.append([summary, page]) - retval.append([data_summary, page]) return retval def raw_values(self): From 951ae8cd89af20d1fddce4ca8b289c2a910f87fb Mon Sep 17 00:00:00 2001 From: Tim Huang Date: Tue, 22 Sep 2015 19:29:00 +0200 Subject: [PATCH 08/78] Bug 1181489 - Adding sanity checks into the alarm service for preventing from the alarm works erroneously. r=fabrice --- dom/alarm/AlarmService.jsm | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/dom/alarm/AlarmService.jsm b/dom/alarm/AlarmService.jsm index aeaedfb724d..a87fc5982d1 100644 --- a/dom/alarm/AlarmService.jsm +++ b/dom/alarm/AlarmService.jsm @@ -252,6 +252,18 @@ this.AlarmService = { debug("_onAlarmFired()"); if (this._currentAlarm) { + let currentAlarmTime = this._getAlarmTime(this._currentAlarm); + + // If a alarm fired before the actual time that the current + // alarm should occur, we reset this current alarm. + if (currentAlarmTime > Date.now()) { + let currentAlarm = this._currentAlarm; + this._currentAlarm = currentAlarm; + + this._debugCurrentAlarm(); + return; + } + this._removeAlarmFromDb(this._currentAlarm.id, null); this._notifyAlarmObserver(this._currentAlarm); this._currentAlarm = null; @@ -414,6 +426,16 @@ this.AlarmService = { // the non-serializable callback to the in-memory object. aNewAlarm['alarmFiredCb'] = aAlarmFiredCb; + // If the new alarm already expired at this moment, we directly + // notify this alarm + let aNewAlarmTime = this._getAlarmTime(aNewAlarm); + if (aNewAlarmTime < Date.now()) { + aSuccessCb(aNewId); + this._removeAlarmFromDb(aNewAlarm.id, null); + this._notifyAlarmObserver(aNewAlarm); + return; + } + // If there is no alarm being set in system, set the new alarm. if (this._currentAlarm == null) { this._currentAlarm = aNewAlarm; @@ -425,7 +447,6 @@ this.AlarmService = { // If the new alarm is earlier than the current alarm, swap them and // push the previous alarm back to the queue. let alarmQueue = this._alarmQueue; - let aNewAlarmTime = this._getAlarmTime(aNewAlarm); let currentAlarmTime = this._getAlarmTime(this._currentAlarm); if (aNewAlarmTime < currentAlarmTime) { alarmQueue.unshift(this._currentAlarm); From 3e1ce22151b02d5e8cd2fd79d6f00c59fcadb529 Mon Sep 17 00:00:00 2001 From: Seth Fowler Date: Wed, 23 Sep 2015 16:53:40 -0700 Subject: [PATCH 09/78] Bug 1195878 - If we detect animation during a full decode, drop the results of the full decode on the floor. r=tn --- image/RasterImage.cpp | 25 +++++++++++++++++++------ image/RasterImage.h | 5 ++++- image/SurfaceFlags.h | 17 +++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/image/RasterImage.cpp b/image/RasterImage.cpp index e516b1e5099..69d3aea21a3 100644 --- a/image/RasterImage.cpp +++ b/image/RasterImage.cpp @@ -811,14 +811,14 @@ RasterImage::OnAddedFrame(uint32_t aNewFrameCount, } } -void +bool RasterImage::SetMetadata(const ImageMetadata& aMetadata, bool aFromMetadataDecode) { MOZ_ASSERT(NS_IsMainThread()); if (mError) { - return; + return true; } if (aMetadata.HasSize()) { @@ -826,7 +826,7 @@ RasterImage::SetMetadata(const ImageMetadata& aMetadata, if (size.width < 0 || size.height < 0) { NS_WARNING("Image has negative intrinsic size"); DoError(); - return; + return true; } MOZ_ASSERT(aMetadata.HasOrientation()); @@ -837,7 +837,7 @@ RasterImage::SetMetadata(const ImageMetadata& aMetadata, NS_WARNING("Image changed size or orientation on redecode! " "This should not happen!"); DoError(); - return; + return true; } // Set the size and flag that we have it. @@ -859,7 +859,7 @@ RasterImage::SetMetadata(const ImageMetadata& aMetadata, // discovered that it actually was during the full decode. This is a // rare failure that only occurs for corrupt images. To recover, we need // to discard all existing surfaces and redecode. - RecoverFromInvalidFrames(mSize, DECODE_FLAGS_DEFAULT); + return false; } } @@ -881,6 +881,8 @@ RasterImage::SetMetadata(const ImageMetadata& aMetadata, Set("hotspotX", intwrapx); Set("hotspotY", intwrapy); } + + return true; } NS_IMETHODIMP @@ -1732,7 +1734,18 @@ RasterImage::FinalizeDecoder(Decoder* aDecoder) } // Record all the metadata the decoder gathered about this image. - SetMetadata(aDecoder->GetImageMetadata(), wasMetadata); + bool metadataOK = SetMetadata(aDecoder->GetImageMetadata(), wasMetadata); + if (!metadataOK) { + // This indicates a serious error that requires us to discard all existing + // surfaces and redecode to recover. We'll drop the results from this + // decoder on the floor, since they aren't valid. + aDecoder->TakeProgress(); + aDecoder->TakeInvalidRect(); + RecoverFromInvalidFrames(mSize, + FromSurfaceFlags(aDecoder->GetSurfaceFlags())); + return; + } + MOZ_ASSERT(mError || mHasSize || !aDecoder->HasSize(), "SetMetadata should've gotten a size"); diff --git a/image/RasterImage.h b/image/RasterImage.h index c962cf69c87..4f3e9021d5b 100644 --- a/image/RasterImage.h +++ b/image/RasterImage.h @@ -324,8 +324,11 @@ private: * @param aMetadata The metadata to set on this image. * @param aFromMetadataDecode True if this metadata came from a metadata * decode; false if it came from a full decode. + * @return |true| unless a catastrophic failure was discovered. If |false| is + * returned, it indicates that the image is corrupt in a way that requires all + * surfaces to be discarded to recover. */ - void SetMetadata(const ImageMetadata& aMetadata, bool aFromMetadataDecode); + bool SetMetadata(const ImageMetadata& aMetadata, bool aFromMetadataDecode); /** * In catastrophic circumstances like a GPU driver crash, the contents of our diff --git a/image/SurfaceFlags.h b/image/SurfaceFlags.h index ac1c5e66463..1a0000542fd 100644 --- a/image/SurfaceFlags.h +++ b/image/SurfaceFlags.h @@ -50,6 +50,23 @@ ToSurfaceFlags(uint32_t aFlags) return flags; } +/** + * Given a set of SurfaceFlags, returns a set of imgIContainer FLAG_* flags with + * the corresponding flags set. + */ +inline uint32_t +FromSurfaceFlags(SurfaceFlags aFlags) +{ + uint32_t flags = imgIContainer::DECODE_FLAGS_DEFAULT; + if (aFlags & SurfaceFlags::NO_PREMULTIPLY_ALPHA) { + flags |= imgIContainer::FLAG_DECODE_NO_PREMULTIPLY_ALPHA; + } + if (aFlags & SurfaceFlags::NO_COLORSPACE_CONVERSION) { + flags |= imgIContainer::FLAG_DECODE_NO_COLORSPACE_CONVERSION; + } + return flags; +} + } // namespace image } // namespace mozilla From b15f8441aac8d0716d6568e38e267d6fbbf4643d Mon Sep 17 00:00:00 2001 From: Daniel Holbert Date: Fri, 18 Sep 2015 15:33:43 -0700 Subject: [PATCH 10/78] Bug 1205923: Make VectorImage::GetWidth/GetHeight set outparam to 0 (not -1) on failure, to accomodate callers that don't check error codes. r=seth --- image/VectorImage.cpp | 46 +++++++++++++++++++-------- image/test/crashtests/1205923-1.html | 36 +++++++++++++++++++++ image/test/crashtests/crashtests.list | 2 ++ image/test/crashtests/unsized-svg.svg | 1 + 4 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 image/test/crashtests/1205923-1.html create mode 100644 image/test/crashtests/unsized-svg.svg diff --git a/image/VectorImage.cpp b/image/VectorImage.cpp index 5ce1060bc6a..9e9fb00d3a2 100644 --- a/image/VectorImage.cpp +++ b/image/VectorImage.cpp @@ -497,14 +497,23 @@ NS_IMETHODIMP VectorImage::GetWidth(int32_t* aWidth) { if (mError || !mIsFullyLoaded) { - *aWidth = -1; - } else { - SVGSVGElement* rootElem = mSVGDocumentWrapper->GetRootSVGElem(); - MOZ_ASSERT(rootElem, "Should have a root SVG elem, since we finished " - "loading without errors"); - *aWidth = rootElem->GetIntrinsicWidth(); + // XXXdholbert Technically we should leave outparam untouched when we + // fail. But since many callers don't check for failure, we set it to 0 on + // failure, for sane/predictable results. + *aWidth = 0; + return NS_ERROR_FAILURE; } - return *aWidth >= 0 ? NS_OK : NS_ERROR_FAILURE; + + SVGSVGElement* rootElem = mSVGDocumentWrapper->GetRootSVGElem(); + MOZ_ASSERT(rootElem, "Should have a root SVG elem, since we finished " + "loading without errors"); + int32_t rootElemWidth = rootElem->GetIntrinsicWidth(); + if (rootElemWidth < 0) { + *aWidth = 0; + return NS_ERROR_FAILURE; + } + *aWidth = rootElemWidth; + return NS_OK; } //****************************************************************************** @@ -561,14 +570,23 @@ NS_IMETHODIMP VectorImage::GetHeight(int32_t* aHeight) { if (mError || !mIsFullyLoaded) { - *aHeight = -1; - } else { - SVGSVGElement* rootElem = mSVGDocumentWrapper->GetRootSVGElem(); - MOZ_ASSERT(rootElem, "Should have a root SVG elem, since we finished " - "loading without errors"); - *aHeight = rootElem->GetIntrinsicHeight(); + // XXXdholbert Technically we should leave outparam untouched when we + // fail. But since many callers don't check for failure, we set it to 0 on + // failure, for sane/predictable results. + *aHeight = 0; + return NS_ERROR_FAILURE; } - return *aHeight >= 0 ? NS_OK : NS_ERROR_FAILURE; + + SVGSVGElement* rootElem = mSVGDocumentWrapper->GetRootSVGElem(); + MOZ_ASSERT(rootElem, "Should have a root SVG elem, since we finished " + "loading without errors"); + int32_t rootElemHeight = rootElem->GetIntrinsicHeight(); + if (rootElemHeight < 0) { + *aHeight = 0; + return NS_ERROR_FAILURE; + } + *aHeight = rootElemHeight; + return NS_OK; } //****************************************************************************** diff --git a/image/test/crashtests/1205923-1.html b/image/test/crashtests/1205923-1.html new file mode 100644 index 00000000000..456fc51b6e3 --- /dev/null +++ b/image/test/crashtests/1205923-1.html @@ -0,0 +1,36 @@ + + + + + + diff --git a/image/test/crashtests/crashtests.list b/image/test/crashtests/crashtests.list index 8d9ff40cd70..d43861fa46c 100644 --- a/image/test/crashtests/crashtests.list +++ b/image/test/crashtests/crashtests.list @@ -53,6 +53,8 @@ load invalid-disposal-method-1.gif load invalid-disposal-method-2.gif load invalid-disposal-method-3.gif +load 1205923-1.html + # Ensure we handle ICO directory entries which specify the wrong size for the # contained resource. load invalid_ico_height.ico diff --git a/image/test/crashtests/unsized-svg.svg b/image/test/crashtests/unsized-svg.svg new file mode 100644 index 00000000000..714efc7ef0b --- /dev/null +++ b/image/test/crashtests/unsized-svg.svg @@ -0,0 +1 @@ + From 47e66f0c7d7f7e998adf87622f2bf1de02b80cab Mon Sep 17 00:00:00 2001 From: Jim Blandy Date: Wed, 23 Sep 2015 13:59:28 -0700 Subject: [PATCH 11/78] Bug 1206356: Add mfbt/Random.h, implementing the xorshift128+ random number generator. r=waldo --- mfbt/XorShift128PlusRNG.h | 97 ++++++++++++++++++++++ mfbt/moz.build | 1 + mfbt/tests/TestXorShift128PlusRNG.cpp | 115 ++++++++++++++++++++++++++ mfbt/tests/moz.build | 1 + 4 files changed, 214 insertions(+) create mode 100644 mfbt/XorShift128PlusRNG.h create mode 100644 mfbt/tests/TestXorShift128PlusRNG.cpp diff --git a/mfbt/XorShift128PlusRNG.h b/mfbt/XorShift128PlusRNG.h new file mode 100644 index 00000000000..fb4139fe0e1 --- /dev/null +++ b/mfbt/XorShift128PlusRNG.h @@ -0,0 +1,97 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* The xorshift128+ pseudo-random number generator. */ + +#ifndef mozilla_XorShift128Plus_h +#define mozilla_XorShift128Plus_h + +#include "mozilla/Assertions.h" +#include "mozilla/FloatingPoint.h" + +#include +#include +#include + +namespace mozilla { +namespace non_crypto { + +/* + * A stream of pseudo-random numbers generated using the xorshift+ technique + * described here: + * + * Vigna, Sebastiano (2014). "Further scramblings of Marsaglia's xorshift + * generators". arXiv:1404.0390 (http://arxiv.org/abs/1404.0390) + * + * That paper says: + * + * In particular, we propose a tightly coded xorshift128+ generator that + * does not fail systematically any test from the BigCrush suite of TestU01 + * (even reversed) and generates 64 pseudorandom bits in 1.10 ns on an + * Intel(R) Core(TM) i7-4770 CPU @3.40GHz (Haswell). It is the fastest + * generator we are aware of with such empirical statistical properties. + * + * This generator is not suitable as a cryptographically secure random number + * generator. + */ +class XorShift128PlusRNG { + uint64_t mState[2]; + + public: + /* + * Construct a xorshift128+ pseudo-random number stream using |aInitial0| and + * |aInitial1| as the initial state. These may not both be zero; ideally, they + * should have an almost even mix of zero and one bits. + */ + XorShift128PlusRNG(uint64_t aInitial0, uint64_t aInitial1) { + setState(aInitial0, aInitial1); + } + + /* Return a pseudo-random 64-bit number. */ + uint64_t next() { + uint64_t s1 = mState[0]; + const uint64_t s0 = mState[1]; + mState[0] = s0; + s1 ^= s1 << 23; + mState[1] = s1 ^ s0 ^ (s1 >> 17) ^ (s0 >> 26); + return mState[1] + s0; + } + + /* + * Return a pseudo-random floating-point value in the range [0, 1). + * More precisely, choose an integer in the range [0, 2**53) and + * divide it by 2**53. + */ + double nextDouble() { + /* + * Because the IEEE 64-bit floating point format stores the leading '1' bit + * of the mantissa implicitly, it effectively represents a mantissa in the + * range [0, 2**53) in only 52 bits. FloatingPoint::kExponentShift + * is the width of the bitfield in the in-memory format, so we must add one + * to get the mantissa's range. + */ + static const int kMantissaBits = + mozilla::FloatingPoint::kExponentShift + 1; + uint64_t mantissa = next() & ((1ULL << kMantissaBits) - 1); + return ldexp(static_cast(mantissa), -kMantissaBits); + } + + /* + * Set the stream's current state to |aState0| and |aState1|. These must not + * both be zero; ideally, they should have an almost even mix of zero and one + * bits. + */ + void setState(uint64_t aState0, uint64_t aState1) { + MOZ_ASSERT(aState0 || aState1); + mState[0] = aState0; + mState[1] = aState1; + } +}; + +} // namespace non_crypto +} // namespace mozilla + +#endif // mozilla_XorShift128Plus_h diff --git a/mfbt/moz.build b/mfbt/moz.build index 18ae47637e2..5de80c48de7 100644 --- a/mfbt/moz.build +++ b/mfbt/moz.build @@ -91,6 +91,7 @@ EXPORTS.mozilla = [ 'Variant.h', 'Vector.h', 'WeakPtr.h', + 'XorShift128PlusRNG.h', 'unused.h', ] diff --git a/mfbt/tests/TestXorShift128PlusRNG.cpp b/mfbt/tests/TestXorShift128PlusRNG.cpp new file mode 100644 index 00000000000..dbe83d049ee --- /dev/null +++ b/mfbt/tests/TestXorShift128PlusRNG.cpp @@ -0,0 +1,115 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include + +#include "mozilla/Assertions.h" +#include "mozilla/PodOperations.h" +#include "mozilla/XorShift128PlusRNG.h" + +using mozilla::non_crypto::XorShift128PlusRNG; + +static void +TestDumbSequence() +{ + XorShift128PlusRNG rng(1, 4); + + // Calculated by hand following the algorithm given in the paper. The upper + // bits are mostly zero because we started with a poor seed; once it has run + // for a while, we'll get an even mix of ones and zeros in all 64 bits. + MOZ_RELEASE_ASSERT(rng.next() == 0x800049); + MOZ_RELEASE_ASSERT(rng.next() == 0x3000186); + MOZ_RELEASE_ASSERT(rng.next() == 0x400003001145); + + // Using ldexp here lets us write out the mantissa in hex, so we can compare + // them with the results generated by hand. + MOZ_RELEASE_ASSERT(rng.nextDouble() + == ldexp(static_cast(0x1400003105049), -53)); + MOZ_RELEASE_ASSERT(rng.nextDouble() + == ldexp(static_cast(0x2000802e49146), -53)); + MOZ_RELEASE_ASSERT(rng.nextDouble() + == ldexp(static_cast(0x248300468544d), -53)); +} + +static size_t +Population(uint64_t n) +{ + size_t pop = 0; + + while (n > 0) { + n &= n-1; // Clear the rightmost 1-bit in n. + pop++; + } + + return pop; +} + +static void +TestPopulation() +{ + XorShift128PlusRNG rng(698079309544035222ULL, 6012389156611637584ULL); + + // Give it some time to warm up; it should tend towards more + // even distributions of zeros and ones. + for (size_t i = 0; i < 40; i++) + rng.next(); + + for (size_t i = 0; i < 40; i++) { + size_t pop = Population(rng.next()); + MOZ_RELEASE_ASSERT(24 <= pop && pop <= 40); + } +} + +static void +TestSetState() +{ + static const uint64_t seed[2] = { 1795644156779822404ULL, 14162896116325912595ULL }; + XorShift128PlusRNG rng(seed[0], seed[1]); + + const size_t n = 10; + uint64_t log[n]; + + for (size_t i = 0; i < n; i++) + log[i] = rng.next(); + + rng.setState(seed[0], seed[1]); + + for (size_t i = 0; i < n; i++) + MOZ_RELEASE_ASSERT(log[i] == rng.next()); +} + +static void +TestDoubleDistribution() +{ + XorShift128PlusRNG rng(0xa207aaede6859736, 0xaca6ca5060804791); + + const size_t n = 100; + size_t bins[n]; + mozilla::PodArrayZero(bins); + + // This entire file runs in 0.006s on my laptop. Generating + // more numbers lets us put tighter bounds on the bins. + for (size_t i = 0; i < 100000; i++) { + double d = rng.nextDouble(); + MOZ_RELEASE_ASSERT(0.0 <= d && d < 1.0); + bins[(int) (d * n)]++; + } + + for (size_t i = 0; i < n; i++) { + MOZ_RELEASE_ASSERT(900 <= bins[i] && bins[i] <= 1100); + } +} + +int +main() +{ + TestDumbSequence(); + TestPopulation(); + TestSetState(); + TestDoubleDistribution(); + + return 0; +} diff --git a/mfbt/tests/moz.build b/mfbt/tests/moz.build index ade09f5aa94..f9cd9e27022 100644 --- a/mfbt/tests/moz.build +++ b/mfbt/tests/moz.build @@ -40,6 +40,7 @@ CppUnitTests([ 'TestVariant', 'TestVector', 'TestWeakPtr', + 'TestXorShift128PlusRNG', ]) if not CONFIG['MOZ_ASAN']: From 08b1362c5805389505bb38dfccefa632346f2d21 Mon Sep 17 00:00:00 2001 From: Garvan Keeley Date: Thu, 24 Sep 2015 01:29:47 -0400 Subject: [PATCH 12/78] Bug 1207266 - turn off b2g stumbler if no RIL. r=jdm --- dom/system/gonk/GonkGPSGeolocationProvider.cpp | 4 +++- dom/system/gonk/moz.build | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dom/system/gonk/GonkGPSGeolocationProvider.cpp b/dom/system/gonk/GonkGPSGeolocationProvider.cpp index b56b4da2e8c..68c1a38a0a9 100644 --- a/dom/system/gonk/GonkGPSGeolocationProvider.cpp +++ b/dom/system/gonk/GonkGPSGeolocationProvider.cpp @@ -15,7 +15,6 @@ */ #include "GonkGPSGeolocationProvider.h" -#include "mozstumbler/MozStumbler.h" #include #include @@ -40,6 +39,7 @@ #include "mozilla/dom/SettingChangeNotificationBinding.h" #ifdef MOZ_B2G_RIL +#include "mozstumbler/MozStumbler.h" #include "nsIIccInfo.h" #include "nsIMobileConnectionInfo.h" #include "nsIMobileConnectionService.h" @@ -148,7 +148,9 @@ GonkGPSGeolocationProvider::LocationCallback(GpsLocation* location) nsRefPtr event = new UpdateLocationEvent(somewhere); NS_DispatchToMainThread(event); +#ifdef MOZ_B2G_RIL MozStumble(somewhere); +#endif } void diff --git a/dom/system/gonk/moz.build b/dom/system/gonk/moz.build index 61142641afd..3552b6fb1ee 100644 --- a/dom/system/gonk/moz.build +++ b/dom/system/gonk/moz.build @@ -35,7 +35,6 @@ XPIDL_MODULE = 'dom_system_gonk' EXPORTS += [ 'GeolocationUtil.h', 'GonkGPSGeolocationProvider.h', - 'mozstumbler/MozStumbler.h', 'nsVolume.h', 'nsVolumeService.h', ] @@ -49,10 +48,6 @@ UNIFIED_SOURCES += [ 'MozMtpDatabase.cpp', 'MozMtpServer.cpp', 'MozMtpStorage.cpp', - 'mozstumbler/MozStumbler.cpp', - 'mozstumbler/StumblerLogging.cpp', - 'mozstumbler/UploadStumbleRunnable.cpp', - 'mozstumbler/WriteStumbleOnThread.cpp', 'NetIdManager.cpp', 'NetworkUtils.cpp', 'NetworkWorker.cpp', @@ -93,6 +88,15 @@ EXTRA_JS_MODULES += [ ] if CONFIG['MOZ_B2G_RIL']: + EXPORTS += [ + 'mozstumbler/MozStumbler.h', + ] + UNIFIED_SOURCES += [ + 'mozstumbler/MozStumbler.cpp', + 'mozstumbler/StumblerLogging.cpp', + 'mozstumbler/UploadStumbleRunnable.cpp', + 'mozstumbler/WriteStumbleOnThread.cpp' + ] XPIDL_SOURCES += [ 'nsIDataCallInterfaceService.idl', 'nsIDataCallManager.idl', From fe24dad3e4b181c7b5b77f8963675f55d3170fda Mon Sep 17 00:00:00 2001 From: JW Wang Date: Thu, 24 Sep 2015 14:20:38 +0800 Subject: [PATCH 13/78] Bug 1188643. Buffer more audio in audio capture mode to avoid glitches. r=cpearce. --- dom/media/MediaDecoderStateMachine.cpp | 37 ++++++++++++++++++-------- dom/media/MediaDecoderStateMachine.h | 6 +++-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index ff4b99da344..f5bd9aec891 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -205,7 +205,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mQuickBufferingLowDataThresholdUsecs(detail::QUICK_BUFFERING_LOW_DATA_USECS), mIsAudioPrerolling(false), mIsVideoPrerolling(false), - mAudioCaptured(false), + mAudioCaptured(false, "MediaDecoderStateMachine::mAudioCaptured"), mAudioCompleted(false, "MediaDecoderStateMachine::mAudioCompleted"), mNotifyMetadataBeforeFirstFrame(false), mDispatchedEventToDecode(false), @@ -219,7 +219,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mCorruptFrames(60), mDecodingFirstFrame(true), mSentLoadedMetadataEvent(false), - mSentFirstFrameLoadedEvent(false), + mSentFirstFrameLoadedEvent(false, "MediaDecoderStateMachine::mSentFirstFrameLoadedEvent"), mSentPlaybackEndedEvent(false), mStreamSink(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)), mResource(aDecoder->GetResource()), @@ -350,6 +350,8 @@ MediaDecoderStateMachine::InitializationTask() mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged); mWatchManager.Watch(mLogicallySeeking, &MediaDecoderStateMachine::LogicallySeekingChanged); mWatchManager.Watch(mSameOriginMedia, &MediaDecoderStateMachine::SameOriginMediaChanged); + mWatchManager.Watch(mSentFirstFrameLoadedEvent, &MediaDecoderStateMachine::AdjustAudioThresholds); + mWatchManager.Watch(mAudioCaptured, &MediaDecoderStateMachine::AdjustAudioThresholds); // Propagate mSameOriginMedia to mDecodedStream. SameOriginMediaChanged(); @@ -2068,6 +2070,28 @@ MediaDecoderStateMachine::IsDecodingFirstFrame() return mState == DECODER_STATE_DECODING && mDecodingFirstFrame; } +void +MediaDecoderStateMachine::AdjustAudioThresholds() +{ + MOZ_ASSERT(OnTaskQueue()); + AssertCurrentThreadInMonitor(); + + // Experiments show that we need to buffer more if audio is captured to avoid + // audio glitch. See bug 1188643 comment 16 for the details. + auto divisor = mAudioCaptured ? NO_VIDEO_AMPLE_AUDIO_DIVISOR / 2 + : NO_VIDEO_AMPLE_AUDIO_DIVISOR; + + // We're playing audio only. We don't need to worry about slow video + // decodes causing audio underruns, so don't buffer so much audio in + // order to reduce memory usage. + if (HasAudio() && !HasVideo() && mSentFirstFrameLoadedEvent) { + mAmpleAudioThresholdUsecs = detail::AMPLE_AUDIO_USECS / divisor; + mLowAudioThresholdUsecs = detail::LOW_AUDIO_USECS / divisor; + mQuickBufferingLowDataThresholdUsecs = + detail::QUICK_BUFFERING_LOW_DATA_USECS / divisor; + } +} + void MediaDecoderStateMachine::FinishDecodeFirstFrame() { @@ -2088,15 +2112,6 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame() "transportSeekable=%d, mediaSeekable=%d", Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable.Ref()); - if (HasAudio() && !HasVideo() && !mSentFirstFrameLoadedEvent) { - // We're playing audio only. We don't need to worry about slow video - // decodes causing audio underruns, so don't buffer so much audio in - // order to reduce memory usage. - mAmpleAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; - mLowAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; - mQuickBufferingLowDataThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; - } - // Get potentially updated metadata { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index f10b28bb41e..0f2ee96c26e 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -654,6 +654,8 @@ private: // play time. bool NeedToSkipToNextKeyframe(); + void AdjustAudioThresholds(); + // The decoder object that created this state machine. The state machine // holds a strong reference to the decoder to ensure that the decoder stays // alive once media element has started the decoder shutdown process, and has @@ -1136,7 +1138,7 @@ private: // True if we shouldn't play our audio (but still write it to any capturing // streams). When this is true, the audio thread will never start again after // it has stopped. - bool mAudioCaptured; + Watchable mAudioCaptured; // True if the audio playback thread has finished. It is finished // when either all the audio frames have completed playing, or we've moved @@ -1228,7 +1230,7 @@ private: // SetStartTime because the mStartTime already set before. Also we don't need // to decode any audio/video since the MediaDecoder will trigger a seek // operation soon. - bool mSentFirstFrameLoadedEvent; + Watchable mSentFirstFrameLoadedEvent; bool mSentPlaybackEndedEvent; From 0ac19de34435d54dc41b6406b440252b79065fbe Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Thu, 24 Sep 2015 08:31:34 +0200 Subject: [PATCH 14/78] Backed out 1 changesets (bug 1152046) for bustage again on a CLOSED TREE Backed out changeset f579176f1c3e (bug 1152046) --- netwerk/base/ClosingService.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/netwerk/base/ClosingService.cpp b/netwerk/base/ClosingService.cpp index 81c83523926..b8117dd79b3 100644 --- a/netwerk/base/ClosingService.cpp +++ b/netwerk/base/ClosingService.cpp @@ -26,7 +26,6 @@ public: namespace mozilla { namespace net { -#ifndef MOZ_WIDGET_GONK static PRIOMethods sTcpUdpPRCloseLayerMethods; static PRIOMethods *sTcpUdpPRCloseLayerMethodsPtr = nullptr; static PRDescIdentity sTcpUdpPRCloseLayerId; @@ -69,7 +68,6 @@ TcpUdpPRCloseLayerClose(PRFileDesc *aFd) delete closingLayerSecret; return status; } -#endif //MOZ_WIDGET_GONK ClosingService* ClosingService::sInstance = nullptr; @@ -85,7 +83,6 @@ ClosingService::ClosingService() void ClosingService::Start() { -#ifndef MOZ_WIDGET_GONK if (!sTcpUdpPRCloseLayerMethodsPtr) { sTcpUdpPRCloseLayerId = PR_GetUniqueIdentity("TCP and UDP PRClose layer"); PR_ASSERT(PR_INVALID_IO_LAYER != sTcpUdpPRCloseLayerId); @@ -104,7 +101,6 @@ ClosingService::Start() delete service; } } -#endif } nsresult @@ -123,12 +119,6 @@ ClosingService::StartInternal() nsresult ClosingService::AttachIOLayer(PRFileDesc *aFd) { -#ifdef MOZ_WIDGET_GONK - - return NS_OK; - -#else - if (!sTcpUdpPRCloseLayerMethodsPtr) { return NS_OK; } @@ -153,8 +143,6 @@ ClosingService::AttachIOLayer(PRFileDesc *aFd) PR_DELETE(layer); } return NS_OK; - -#endif //MOZ_WIDGET_GONK } void @@ -183,14 +171,12 @@ ClosingService::PostRequest(PRFileDesc *aFd) void ClosingService::Shutdown() { -#ifndef MOZ_WIDGET_GONK MOZ_ASSERT(NS_IsMainThread()); if (sInstance) { sInstance->ShutdownInternal(); NS_RELEASE(sInstance); } -#endif //MOZ_WIDGET_GONK } void From e8e44d28d79cff510f3fa93322b17177bcd58ef1 Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Thu, 24 Sep 2015 08:41:04 +0200 Subject: [PATCH 15/78] Backed out 1 changesets (bug 1178533) for xpcshell-1 test failures Backed out changeset b12584fad334 (bug 1178533) --- b2g/installer/package-manifest.in | 5 - .../preferences/in-content/advanced.js | 3 +- browser/components/preferences/translation.js | 5 +- .../sessionstore/SessionStorage.jsm | 3 +- browser/installer/package-manifest.in | 5 - caps/BasePrincipal.cpp | 22 ---- caps/BasePrincipal.h | 1 - caps/nsIScriptSecurityManager.idl | 9 +- caps/nsScriptSecurityManager.cpp | 18 --- caps/tests/unit/test_origin.js | 5 +- dom/apps/PermissionsInstaller.jsm | 9 +- dom/moz.build | 1 - dom/newapps/InstallPackagedWebapp.js | 63 --------- dom/newapps/InstallPackagedWebapp.js.rej | 12 -- dom/newapps/InstallPackagedWebapp.manifest | 2 - dom/newapps/interfaces/moz.build | 11 -- .../interfaces/nsIInstallPackagedWebapp.idl | 13 -- dom/newapps/moz.build | 14 -- .../test/xpcshell/0001-1st-round-test.patch | 54 -------- dom/newapps/test/xpcshell/test_install.js | 121 ------------------ dom/newapps/test/xpcshell/xpcshell.ini | 5 - dom/permission/PermissionSettings.jsm | 52 +++----- dom/push/PushRecord.jsm | 4 +- dom/requestsync/RequestSyncService.jsm | 3 +- .../b2gdroid/installer/package-manifest.in | 4 - mobile/android/installer/package-manifest.in | 4 - netwerk/protocol/http/PackagedAppService.cpp | 43 +------ netwerk/protocol/http/PackagedAppService.h | 13 +- toolkit/modules/BrowserUtils.jsm | 24 ++++ toolkit/modules/PermissionsUtils.jsm | 2 +- toolkit/modules/PermissionsUtils.jsm.rej | 10 -- 31 files changed, 65 insertions(+), 475 deletions(-) delete mode 100644 dom/newapps/InstallPackagedWebapp.js delete mode 100644 dom/newapps/InstallPackagedWebapp.js.rej delete mode 100644 dom/newapps/InstallPackagedWebapp.manifest delete mode 100644 dom/newapps/interfaces/moz.build delete mode 100644 dom/newapps/interfaces/nsIInstallPackagedWebapp.idl delete mode 100644 dom/newapps/moz.build delete mode 100644 dom/newapps/test/xpcshell/0001-1st-round-test.patch delete mode 100644 dom/newapps/test/xpcshell/test_install.js delete mode 100644 dom/newapps/test/xpcshell/xpcshell.ini delete mode 100644 toolkit/modules/PermissionsUtils.jsm.rej diff --git a/b2g/installer/package-manifest.in b/b2g/installer/package-manifest.in index 26f980fd3e0..7951bb470ff 100644 --- a/b2g/installer/package-manifest.in +++ b/b2g/installer/package-manifest.in @@ -185,7 +185,6 @@ @RESPATH@/components/dom.xpt @RESPATH@/components/dom_activities.xpt @RESPATH@/components/dom_apps.xpt -@RESPATH@/components/dom_newapps.xpt @RESPATH@/components/dom_audiochannel.xpt @RESPATH@/components/dom_base.xpt @RESPATH@/components/dom_system.xpt @@ -728,10 +727,6 @@ @RESPATH@/components/@DLL_PREFIX@mozgnome@DLL_SUFFIX@ #endif -; Signed Packaged Content -@RESPATH@/components/InstallPackagedWebapp.manifest -@RESPATH@/components/InstallPackagedWebapp.js - ; ANGLE on Win32 #ifdef XP_WIN32 #ifndef HAVE_64BIT_BUILD diff --git a/browser/components/preferences/in-content/advanced.js b/browser/components/preferences/in-content/advanced.js index 90949210aee..7ec64ba4e6d 100644 --- a/browser/components/preferences/in-content/advanced.js +++ b/browser/components/preferences/in-content/advanced.js @@ -6,6 +6,7 @@ Components.utils.import("resource://gre/modules/DownloadUtils.jsm"); Components.utils.import("resource://gre/modules/LoadContextInfo.jsm"); Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); +Components.utils.import("resource://gre/modules/BrowserUtils.jsm"); var gAdvancedPane = { _inited: false, @@ -571,7 +572,7 @@ var gAdvancedPane = { var list = document.getElementById("offlineAppsList"); var item = list.selectedItem; var origin = item.getAttribute("origin"); - var principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); + var principal = BrowserUtils.principalFromOrigin(origin); var prompts = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] .getService(Components.interfaces.nsIPromptService); diff --git a/browser/components/preferences/translation.js b/browser/components/preferences/translation.js index 8341740362a..b10baf761e3 100644 --- a/browser/components/preferences/translation.js +++ b/browser/components/preferences/translation.js @@ -9,6 +9,7 @@ const {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/BrowserUtils.jsm"); XPCOMUtils.defineLazyGetter(this, "gLangBundle", () => Services.strings.createBundle("chrome://global/locale/languageNames.properties")); @@ -189,7 +190,7 @@ var gTranslationExceptions = { onSiteDeleted: function() { let removedSites = this._siteTree.getSelectedItems(); for (let origin of removedSites) { - let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); + let principal = BrowserUtils.principalFromOrigin(origin); Services.perms.removeFromPrincipal(principal, kPermissionType); } }, @@ -202,7 +203,7 @@ var gTranslationExceptions = { this._siteTree.boxObject.rowCountChanged(0, -removedSites.length); for (let origin of removedSites) { - let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); + let principal = BrowserUtils.principalFromOrigin(origin); Services.perms.removeFromPrincipal(principal, kPermissionType); } diff --git a/browser/components/sessionstore/SessionStorage.jsm b/browser/components/sessionstore/SessionStorage.jsm index a2322e0181f..5ed1dbdfa86 100644 --- a/browser/components/sessionstore/SessionStorage.jsm +++ b/browser/components/sessionstore/SessionStorage.jsm @@ -9,6 +9,7 @@ this.EXPORTED_SYMBOLS = ["SessionStorage"]; const Cu = Components.utils; const Ci = Components.interfaces; +Cu.import("resource://gre/modules/BrowserUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); @@ -104,7 +105,7 @@ var SessionStorageInternal = { restore: function (aDocShell, aStorageData) { for (let origin of Object.keys(aStorageData)) { let data = aStorageData[origin]; - let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin); + let principal = BrowserUtils.principalFromOrigin(origin); let storageManager = aDocShell.QueryInterface(Ci.nsIDOMStorageManager); let window = aDocShell.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIDOMWindow); diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index 3e19abd1d43..aaf28ca4a54 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -193,7 +193,6 @@ @RESPATH@/components/dom_messages.xpt #endif @RESPATH@/components/dom_apps.xpt -@RESPATH@/components/dom_newapps.xpt @RESPATH@/components/dom_base.xpt @RESPATH@/components/dom_system.xpt #ifdef MOZ_B2G_BT @@ -637,10 +636,6 @@ @RESPATH@/components/PrivateBrowsing.manifest @RESPATH@/components/PrivateBrowsingTrackingProtectionWhitelist.js -; Signed Packaged Content -@RESPATH@/components/InstallPackagedWebapp.manifest -@RESPATH@/components/InstallPackagedWebapp.js - ; ANGLE GLES-on-D3D rendering library #ifdef MOZ_ANGLE_RENDERER @BINPATH@/libEGL.dll diff --git a/caps/BasePrincipal.cpp b/caps/BasePrincipal.cpp index 4faa72190d8..a864c85a234 100644 --- a/caps/BasePrincipal.cpp +++ b/caps/BasePrincipal.cpp @@ -403,28 +403,6 @@ BasePrincipal::CreateCodebasePrincipal(nsIURI* aURI, OriginAttributes& aAttrs) return codebase.forget(); } -already_AddRefed -BasePrincipal::CreateCodebasePrincipal(const nsACString& aOrigin) -{ - MOZ_ASSERT(!StringBeginsWith(aOrigin, NS_LITERAL_CSTRING("[")), - "CreateCodebasePrincipal does not support System and Expanded principals"); - - MOZ_ASSERT(!StringBeginsWith(aOrigin, NS_LITERAL_CSTRING(NS_NULLPRINCIPAL_SCHEME ":")), - "CreateCodebasePrincipal does not support nsNullPrincipal"); - - nsAutoCString originNoSuffix; - mozilla::OriginAttributes attrs; - if (!attrs.PopulateFromOrigin(aOrigin, originNoSuffix)) { - return nullptr; - } - - nsCOMPtr uri; - nsresult rv = NS_NewURI(getter_AddRefs(uri), originNoSuffix); - NS_ENSURE_SUCCESS(rv, nullptr); - - return BasePrincipal::CreateCodebasePrincipal(uri, attrs); -} - bool BasePrincipal::AddonAllowsLoad(nsIURI* aURI) { diff --git a/caps/BasePrincipal.h b/caps/BasePrincipal.h index 67b4831de95..01ed3927fa8 100644 --- a/caps/BasePrincipal.h +++ b/caps/BasePrincipal.h @@ -137,7 +137,6 @@ public: static BasePrincipal* Cast(nsIPrincipal* aPrin) { return static_cast(aPrin); } static already_AddRefed CreateCodebasePrincipal(nsIURI* aURI, OriginAttributes& aAttrs); - static already_AddRefed CreateCodebasePrincipal(const nsACString& aOrigin); const OriginAttributes& OriginAttributesRef() { return mOriginAttributes; } uint32_t AppId() const { return mOriginAttributes.mAppId; } diff --git a/caps/nsIScriptSecurityManager.idl b/caps/nsIScriptSecurityManager.idl index cac4beec6a7..916563d8587 100644 --- a/caps/nsIScriptSecurityManager.idl +++ b/caps/nsIScriptSecurityManager.idl @@ -26,7 +26,7 @@ class DomainPolicyClone; [ptr] native JSObjectPtr(JSObject); [ptr] native DomainPolicyClonePtr(mozilla::dom::DomainPolicyClone); -[scriptable, uuid(b7ae2310-576e-11e5-a837-0800200c9a66)] +[scriptable, uuid(6e8a4d1e-d9c6-4d86-bf53-d73f58f36148)] interface nsIScriptSecurityManager : nsISupports { /** @@ -197,13 +197,6 @@ interface nsIScriptSecurityManager : nsISupports [implicit_jscontext] nsIPrincipal createCodebasePrincipal(in nsIURI uri, in jsval originAttributes); - /** - * Returns a principal whose origin is the one we pass in. - * See nsIPrincipal.idl for a description of origin attributes, and - * ChromeUtils.webidl for a list of origin attributes and their defaults. - */ - nsIPrincipal createCodebasePrincipalFromOrigin(in ACString origin); - /** * Returns a unique nonce principal with |originAttributes|. * See nsIPrincipal.idl for a description of origin attributes, and diff --git a/caps/nsScriptSecurityManager.cpp b/caps/nsScriptSecurityManager.cpp index 1b2b825a95d..b9d414b3b2f 100644 --- a/caps/nsScriptSecurityManager.cpp +++ b/caps/nsScriptSecurityManager.cpp @@ -67,7 +67,6 @@ #include "nsContentUtils.h" #include "nsJSUtils.h" #include "nsILoadInfo.h" -#include "nsXPCOMStrings.h" // This should be probably defined on some other place... but I couldn't find it #define WEBAPPS_PERM_NAME "webapps-manage" @@ -1044,23 +1043,6 @@ nsScriptSecurityManager::CreateCodebasePrincipal(nsIURI* aURI, JS::Handle prin = BasePrincipal::CreateCodebasePrincipal(aOrigin); - prin.forget(aPrincipal); - return *aPrincipal ? NS_OK : NS_ERROR_FAILURE; -} - NS_IMETHODIMP nsScriptSecurityManager::CreateNullPrincipal(JS::Handle aOriginAttributes, JSContext* aCx, nsIPrincipal** aPrincipal) diff --git a/caps/tests/unit/test_origin.js b/caps/tests/unit/test_origin.js index ff2d2449ab5..8a84102d2d8 100644 --- a/caps/tests/unit/test_origin.js +++ b/caps/tests/unit/test_origin.js @@ -3,6 +3,7 @@ const Ci = Components.interfaces; const Cu = Components.utils; Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/BrowserUtils.jsm"); var ssm = Services.scriptSecurityManager; function makeURI(uri) { return Services.io.newURI(uri, null, null); } @@ -27,9 +28,9 @@ function checkOriginAttributes(prin, attrs, suffix) { do_check_eq(prin.originAttributes.inBrowser, attrs.inBrowser || false); do_check_eq(prin.originSuffix, suffix || ''); if (!prin.isNullPrincipal && !prin.origin.startsWith('[')) { - do_check_true(ssm.createCodebasePrincipalFromOrigin(prin.origin).equals(prin)); + do_check_true(BrowserUtils.principalFromOrigin(prin.origin).equals(prin)); } else { - checkThrows(() => ssm.createCodebasePrincipalFromOrigin(prin.origin)); + checkThrows(() => BrowserUtils.principalFromOrigin(prin.origin)); } } diff --git a/dom/apps/PermissionsInstaller.jsm b/dom/apps/PermissionsInstaller.jsm index e3ed3aca83d..208b681dfbf 100644 --- a/dom/apps/PermissionsInstaller.jsm +++ b/dom/apps/PermissionsInstaller.jsm @@ -164,8 +164,7 @@ this.PermissionsInstaller = { PermissionSettingsModule.getPermission(expandedPermNames[idx], aApp.manifestURL, aApp.origin, - false, - aApp.isCachedPackage); + false); if (permValue === "unknown") { permValue = PERM_TO_STRING[permission]; } @@ -193,7 +192,7 @@ this.PermissionsInstaller = { * The permission value. * @param object aApp * The just-installed app configuration. - * The properties used are manifestURL, origin, appId, isCachedPackage. + * The properties used are manifestURL and origin. * @returns void **/ _setPermission: function setPermission(aPermName, aPermValue, aApp) { @@ -202,9 +201,7 @@ this.PermissionsInstaller = { origin: aApp.origin, manifestURL: aApp.manifestURL, value: aPermValue, - browserFlag: false, - localId: aApp.localId, - isCachedPackage: aApp.isCachedPackage, + browserFlag: false }); } }; diff --git a/dom/moz.build b/dom/moz.build index 59c67682ceb..4279c8ff757 100644 --- a/dom/moz.build +++ b/dom/moz.build @@ -113,7 +113,6 @@ DIRS += [ 'resourcestats', 'manifest', 'vr', - 'newapps', ] if CONFIG['OS_ARCH'] == 'WINNT': diff --git a/dom/newapps/InstallPackagedWebapp.js b/dom/newapps/InstallPackagedWebapp.js deleted file mode 100644 index 5577e23b0db..00000000000 --- a/dom/newapps/InstallPackagedWebapp.js +++ /dev/null @@ -1,63 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr, Constructor: CC } = Components; - -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -Cu.import("resource://gre/modules/AppsUtils.jsm"); - -XPCOMUtils.defineLazyModuleGetter(this, "PermissionsInstaller", - "resource://gre/modules/PermissionsInstaller.jsm"); - -function debug(aMsg) { - dump("-*-*- InstallPackagedWebapps.js : " + aMsg + "\n"); -} - -function InstallPackagedWebapp() { -} - -InstallPackagedWebapp.prototype = { - QueryInterface: XPCOMUtils.generateQI([Ci.nsIInstallPackagedWebapp]), - classID: Components.ID("{5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f}"), - - /** - * Install permissions for signed packaged web content - * @param string manifestContent - * The manifest content of the cached package. - * @param string aOrigin - * The package origin. - * @param string aManifestURL - * The manifest URL of the package. - * @returns boolean - **/ - - installPackagedWebapp: function(aManifestContent, aOrigin, aManifestURL) { - - try { - let isSuccess = true; - let manifest = JSON.parse(aManifestContent); - - PermissionsInstaller.installPermissions({ - manifest: manifest, - manifestURL: aManifestURL, - origin: aOrigin, - isPreinstalled: false, - isCachedPackage: true - }, false, function() { - Cu.reportError(ex); - }); - - // TODO Bug 1206058 - Register app handlers (system msg) on navigation - // to signed packages. - - return isSuccess; - } - catch(ex) { - Cu.reportError(ex); - return false; - } - }, -}; - -this.NSGetFactory = XPCOMUtils.generateNSGetFactory([InstallPackagedWebapp]); diff --git a/dom/newapps/InstallPackagedWebapp.js.rej b/dom/newapps/InstallPackagedWebapp.js.rej deleted file mode 100644 index 4f0861e6c6b..00000000000 --- a/dom/newapps/InstallPackagedWebapp.js.rej +++ /dev/null @@ -1,12 +0,0 @@ -diff a/dom/newapps/InstallPackagedWebapp.js b/dom/newapps/InstallPackagedWebapp.js (rejected hunks) -@@ -50,7 +50,9 @@ InstallPackagedWebapp.prototype = { - origin: aOrigin, - isPreinstalled: false, - isCachedPackage: true -- }, false); -+ }, false, function() { -+ Cu.reportError(ex); -+ }); - - // TODO: register app handlers (system msg) - diff --git a/dom/newapps/InstallPackagedWebapp.manifest b/dom/newapps/InstallPackagedWebapp.manifest deleted file mode 100644 index 62699dc434f..00000000000 --- a/dom/newapps/InstallPackagedWebapp.manifest +++ /dev/null @@ -1,2 +0,0 @@ -component {5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f} InstallPackagedWebapp.js -contract @mozilla.org/newapps/installpackagedwebapp;1 {5cc6554a-5421-4a5e-b8c2-c62e8b7f4f3f} diff --git a/dom/newapps/interfaces/moz.build b/dom/newapps/interfaces/moz.build deleted file mode 100644 index f9eb7a739b0..00000000000 --- a/dom/newapps/interfaces/moz.build +++ /dev/null @@ -1,11 +0,0 @@ -# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -XPIDL_SOURCES += [ - 'nsIInstallPackagedWebapp.idl' -] - -XPIDL_MODULE = 'dom_newapps' diff --git a/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl b/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl deleted file mode 100644 index b61ae1ed9ef..00000000000 --- a/dom/newapps/interfaces/nsIInstallPackagedWebapp.idl +++ /dev/null @@ -1,13 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsISupports.idl" - -[scriptable, uuid(3b4b69a0-56dc-11e5-a837-0800200c9a66)] -interface nsIInstallPackagedWebapp : nsISupports -{ - boolean installPackagedWebapp(in string aManifestContent, - in string aOrigin, - in string aManifestURL); -}; diff --git a/dom/newapps/moz.build b/dom/newapps/moz.build deleted file mode 100644 index f213894e296..00000000000 --- a/dom/newapps/moz.build +++ /dev/null @@ -1,14 +0,0 @@ -# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -DIRS += ['interfaces'] - -EXTRA_COMPONENTS += [ - 'InstallPackagedWebapp.js', - 'InstallPackagedWebapp.manifest', -] - -XPCSHELL_TESTS_MANIFESTS += ['test/xpcshell/xpcshell.ini'] diff --git a/dom/newapps/test/xpcshell/0001-1st-round-test.patch b/dom/newapps/test/xpcshell/0001-1st-round-test.patch deleted file mode 100644 index 421951e9761..00000000000 --- a/dom/newapps/test/xpcshell/0001-1st-round-test.patch +++ /dev/null @@ -1,54 +0,0 @@ -From d196359810da4b3fb11517b1b3d28dd0a9fcdf39 Mon Sep 17 00:00:00 2001 -From: Stephanie Ouillon -Date: Fri, 18 Sep 2015 15:22:47 +0200 -Subject: [PATCH] 1st round test - ---- - dom/newapps/test/xpcshell/test_install.js | 22 ++++++++++++++++------ - 1 file changed, 16 insertions(+), 6 deletions(-) - -diff --git a/dom/newapps/test/xpcshell/test_install.js b/dom/newapps/test/xpcshell/test_install.js -index b9073a0..53b4a26 100644 ---- a/dom/newapps/test/xpcshell/test_install.js -+++ b/dom/newapps/test/xpcshell/test_install.js -@@ -15,21 +15,31 @@ const mod = Cc['@mozilla.org/newapps/installpackagedwebapp;1'] - function run_test() { - - let manifest = { -- start_url: "start.html", -- launch_path: "other.html" -- }; -+ name: "Test App", -+ launch_path: "/index.html", -+ permissions: { -+ alarms: { }, -+ wifi-manage: { }, -+ tcp-socket: { }, -+ browser: { }, -+ desktop-notification: { }, -+ geolocation: { }, -+ }, -+ } - - //trigger error at install - let aOrigin = ""; - let aManifestURL = ""; - let manifestString = "boum" -- let appId = 123456789; -- let res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL, appId); -+ let res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL); - equal(res, false); - - aOrigin = "http://test.com"; - aManifestURL = "http://test.com/manifest.json"; - manifestString = JSON.stringify(manifest); -- res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL, appId); -+ res = mod.installPackagedWebapp(manifestString, aOrigin, aManifestURL); - equal(res, true); -+ -+ //test permissions are set -+ //try modifying permissions r/w - } --- -1.9.1 - diff --git a/dom/newapps/test/xpcshell/test_install.js b/dom/newapps/test/xpcshell/test_install.js deleted file mode 100644 index 344120d99af..00000000000 --- a/dom/newapps/test/xpcshell/test_install.js +++ /dev/null @@ -1,121 +0,0 @@ -/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -'use strict'; - -const { classes: Cc, interfaces: Ci, utils: Cu } = Components; - -Cu.import('resource://gre/modules/XPCOMUtils.jsm'); -Cu.import('resource://gre/modules/Services.jsm'); -Cu.import("resource://gre/modules/PermissionSettings.jsm"); -Cu.import("resource://gre/modules/PermissionsTable.jsm"); -Cu.import("resource://gre/modules/AppsUtils.jsm"); - -const mod = Cc['@mozilla.org/newapps/installpackagedwebapp;1'] - .getService(Ci.nsIInstallPackagedWebapp); - -XPCOMUtils.defineLazyServiceGetter(this, - "appsService", - "@mozilla.org/AppsService;1", - "nsIAppsService"); - -function run_test() { - - let manifestWithPerms = { - name: "Test App", - launch_path: "/index.html", - type: "privileged", - permissions: { - "alarms": { }, - "wifi-manage": { }, - "tcp-socket": { }, - "desktop-notification": { }, - "geolocation": { }, - }, - }; - - let manifestNoPerms = { - name: "Test App", - launch_path: "/index.html", - type: "privileged", - }; - - let appStatus = "privileged"; - - // Triggering error due to bad manifest - let origin = ""; - let manifestURL = ""; - let manifestString = "boum"; - - let res = mod.installPackagedWebapp(manifestString, origin, manifestURL); - equal(res, false); - - // Install a package with permissions - origin = "http://test.com^appId=1019&inBrowser=1"; - manifestURL = "http://test.com/manifest.json"; - manifestString = JSON.stringify(manifestWithPerms); - let manifestHelper = new ManifestHelper(manifestWithPerms, origin, manifestURL); - - cleanDB(manifestHelper, origin, manifestURL); - - res = mod.installPackagedWebapp(manifestString, origin, manifestURL); - equal(res, true); - checkPermissions(manifestHelper, origin, manifestURL, appStatus); - - // Install a package with permissions - origin = "http://test.com"; - manifestHelper = new ManifestHelper(manifestWithPerms, origin, manifestURL); - - cleanDB(manifestHelper, origin, manifestURL); - - res = mod.installPackagedWebapp(manifestString, origin, manifestURL); - equal(res, true); - checkPermissions(manifestHelper, origin, manifestURL, appStatus); - - - // Install a package with no permission - origin = "http://bar.com^appId=1337&inBrowser=1"; - manifestURL = "http://bar.com/manifest.json"; - manifestString = JSON.stringify(manifestNoPerms); - manifestHelper = new ManifestHelper(manifestNoPerms, origin, manifestURL); - - cleanDB(manifestHelper, origin, manifestURL); - - res = mod.installPackagedWebapp(manifestString, origin, manifestURL); - equal(res, true); - checkPermissions(manifestHelper, origin, manifestURL, appStatus); -} - -// Cleaning permissions database before running a test -function cleanDB(manifestHelper, origin, manifestURL) { - for (let permName in manifestHelper.permissions) { - PermissionSettingsModule.removePermission(permName, manifestURL, origin, "", true); - } -} - -// Check permissions are correctly set in the database -function checkPermissions(manifestHelper, origin, manifestURL, appStatus) { - let perm; - for (let permName in manifestHelper.permissions) { - let permValue = PermissionSettingsModule.getPermission( - permName, manifestURL, origin, "", true); - switch (PermissionsTable[permName][appStatus]) { - case Ci.nsIPermissionManager.UNKNOWN_ACTION: - perm = "unknown"; - break; - case Ci.nsIPermissionManager.ALLOW_ACTION: - perm = "allow"; - break; - case Ci.nsIPermissionManager.DENY_ACTION: - perm = "deny"; - break; - case Ci.nsIPermissionManager.PROMPT_ACTION: - perm = "prompt"; - break; - default: - break; - } - equal(permValue, perm); - } -} diff --git a/dom/newapps/test/xpcshell/xpcshell.ini b/dom/newapps/test/xpcshell/xpcshell.ini deleted file mode 100644 index 5c1329e6e8b..00000000000 --- a/dom/newapps/test/xpcshell/xpcshell.ini +++ /dev/null @@ -1,5 +0,0 @@ -[DEFAULT] -head = -tail = - -[test_install.js] diff --git a/dom/permission/PermissionSettings.jsm b/dom/permission/PermissionSettings.jsm index ddd3a19c45d..8cda5e09a04 100644 --- a/dom/permission/PermissionSettings.jsm +++ b/dom/permission/PermissionSettings.jsm @@ -67,23 +67,13 @@ this.PermissionSettingsModule = { _internalAddPermission: function _internalAddPermission(aData, aAllowAllChanges, aCallbacks) { - // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm. - let app; - let principal; - // Test if app is cached (signed streamable package) or installed via DOMApplicationRegistry - if (aData.isCachedPackage) { - // If the app is from packaged web app, the origin includes origin attributes already. - principal = - Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(aData.origin); - app = {localId: principal.appId}; - } else { - app = appsService.getAppByManifestURL(aData.manifestURL); - let uri = Services.io.newURI(aData.origin, null, null); - principal = - Services.scriptSecurityManager.createCodebasePrincipal(uri, - {appId: app.localId, - inBrowser: aData.browserFlag}); - } + // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm + let uri = Services.io.newURI(aData.origin, null, null); + let app = appsService.getAppByManifestURL(aData.manifestURL); + let principal = + Services.scriptSecurityManager.createCodebasePrincipal(uri, + {appId: app.localId, + inBrowser: aData.browserFlag}); let action; switch (aData.value) @@ -116,24 +106,17 @@ this.PermissionSettingsModule = { } }, - getPermission: function getPermission(aPermName, aManifestURL, aOrigin, aBrowserFlag, aIsCachedPackage) { + getPermission: function getPermission(aPermName, aManifestURL, aOrigin, aBrowserFlag) { // TODO: Bug 1196644 - Add signPKg parameter into PermissionSettings.jsm debug("getPermission: " + aPermName + ", " + aManifestURL + ", " + aOrigin); - let principal; - // Test if app is cached (signed streamable package) or installed via DOMApplicationRegistry - if (aIsCachedPackage) { - // If the app is from packaged web app, the origin includes origin attributes already. - principal = - Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(aOrigin); - } else { - let uri = Services.io.newURI(aOrigin, null, null); - let appID = appsService.getAppLocalIdByManifestURL(aManifestURL); - principal = - Services.scriptSecurityManager.createCodebasePrincipal(uri, - {appId: appID, - inBrowser: aBrowserFlag}); - } + let uri = Services.io.newURI(aOrigin, null, null); + let appID = appsService.getAppLocalIdByManifestURL(aManifestURL); + let principal = + Services.scriptSecurityManager.createCodebasePrincipal(uri, + {appId: appID, + inBrowser: aBrowserFlag}); let result = Services.perms.testExactPermissionFromPrincipal(principal, aPermName); + switch (result) { case Ci.nsIPermissionManager.UNKNOWN_ACTION: @@ -150,14 +133,13 @@ this.PermissionSettingsModule = { } }, - removePermission: function removePermission(aPermName, aManifestURL, aOrigin, aBrowserFlag, aIsCachedPackage) { + removePermission: function removePermission(aPermName, aManifestURL, aOrigin, aBrowserFlag) { let data = { type: aPermName, origin: aOrigin, manifestURL: aManifestURL, value: "unknown", - browserFlag: aBrowserFlag, - isCachedPackage: aIsCachedPackage + browserFlag: aBrowserFlag }; this._internalAddPermission(data, true); }, diff --git a/dom/push/PushRecord.jsm b/dom/push/PushRecord.jsm index b2e3c0e2415..a468ddd39cd 100644 --- a/dom/push/PushRecord.jsm +++ b/dom/push/PushRecord.jsm @@ -19,6 +19,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils", XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils", "resource://gre/modules/PrivateBrowsingUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "BrowserUtils", + "resource://gre/modules/BrowserUtils.jsm"); this.EXPORTED_SYMBOLS = ["PushRecord"]; @@ -216,7 +218,7 @@ Object.defineProperties(PushRecord.prototype, { // Allow tests to omit origin attributes. url += this.originAttributes; } - principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(url); + principal = BrowserUtils.principalFromOrigin(url); principals.set(this, principal); } return principal; diff --git a/dom/requestsync/RequestSyncService.jsm b/dom/requestsync/RequestSyncService.jsm index 4e6bdda0697..c771f6438e1 100644 --- a/dom/requestsync/RequestSyncService.jsm +++ b/dom/requestsync/RequestSyncService.jsm @@ -20,6 +20,7 @@ const RSYNC_STATE_ENABLED = "enabled"; const RSYNC_STATE_DISABLED = "disabled"; const RSYNC_STATE_WIFIONLY = "wifiOnly"; +Cu.import("resource://gre/modules/BrowserUtils.jsm"); Cu.import('resource://gre/modules/IndexedDBHelper.jsm'); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); @@ -174,7 +175,7 @@ this.RequestSyncService = { let dbKeys = []; for (let key in this._registrations) { - let prin = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(key); + let prin = BrowserUtils.principalFromOrigin(key); if (!ChromeUtils.originAttributesMatchPattern(prin.originAttributes, pattern)) { continue; } diff --git a/mobile/android/b2gdroid/installer/package-manifest.in b/mobile/android/b2gdroid/installer/package-manifest.in index 10d10196239..94ff64b2f38 100644 --- a/mobile/android/b2gdroid/installer/package-manifest.in +++ b/mobile/android/b2gdroid/installer/package-manifest.in @@ -138,7 +138,6 @@ @BINPATH@/components/dom.xpt @BINPATH@/components/dom_activities.xpt @BINPATH@/components/dom_apps.xpt -@BINPATH@/components/dom_newapps.xpt @BINPATH@/components/dom_base.xpt @BINPATH@/components/dom_canvas.xpt @BINPATH@/components/dom_core.xpt @@ -678,9 +677,6 @@ bin/libfreebl_32int64_3.so @BINPATH@/components/SystemMessageCache.js @BINPATH@/components/SystemMessageManager.manifest -@BINPATH@/components/InstallPackagedWebapp.manifest -@BINPATH@/components/InstallPackagedWebapp.js - @BINPATH@/components/B2GComponents.manifest @BINPATH@/components/AlertsService.js @BINPATH@/components/ContentPermissionPrompt.js diff --git a/mobile/android/installer/package-manifest.in b/mobile/android/installer/package-manifest.in index 8d720b5d314..ce422257b08 100644 --- a/mobile/android/installer/package-manifest.in +++ b/mobile/android/installer/package-manifest.in @@ -138,7 +138,6 @@ @BINPATH@/components/dom.xpt @BINPATH@/components/dom_activities.xpt @BINPATH@/components/dom_apps.xpt -@BINPATH@/components/dom_newapps.xpt @BINPATH@/components/dom_base.xpt @BINPATH@/components/dom_canvas.xpt @BINPATH@/components/dom_core.xpt @@ -426,9 +425,6 @@ @BINPATH@/components/SystemMessageCache.js @BINPATH@/components/SystemMessageManager.manifest -@BINPATH@/components/InstallPackagedWebapp.manifest -@BINPATH@/components/InstallPackagedWebapp.js - #ifdef MOZ_WEBRTC @BINPATH@/components/PeerConnection.js @BINPATH@/components/PeerConnection.manifest diff --git a/netwerk/protocol/http/PackagedAppService.cpp b/netwerk/protocol/http/PackagedAppService.cpp index 1d5998584f0..a9205370954 100644 --- a/netwerk/protocol/http/PackagedAppService.cpp +++ b/netwerk/protocol/http/PackagedAppService.cpp @@ -17,7 +17,6 @@ #include "mozilla/DebugOnly.h" #include "nsIHttpHeaderVisitor.h" #include "mozilla/LoadContext.h" -#include "nsIInstallPackagedWebapp.h" namespace mozilla { namespace net { @@ -431,7 +430,6 @@ PackagedAppService::PackagedAppDownloader::Init(nsILoadContextInfo* aInfo, mPackageKey = aKey; mPackageOrigin = aPackageOrigin; - mProcessingFirstRequest = true; return NS_OK; } @@ -625,8 +623,6 @@ PackagedAppService::PackagedAppDownloader::OnStopRequest(nsIRequest *aRequest, LOG(("[%p] PackagedAppDownloader::OnStopRequest > status:%X multiChannel:%p\n", this, aStatusCode, multiChannel.get())); - mProcessingFirstRequest = false; - // lastPart will be true if this is the last part in the package, // or if aRequest isn't a multipart channel bool lastPart = true; @@ -707,11 +703,6 @@ PackagedAppService::PackagedAppDownloader::ConsumeData(nsIInputStream *aStream, self->mWriter->ConsumeData(aFromRawSegment, aCount, aWriteCount); - if (self->mProcessingFirstRequest) { - // mProcessingFirstRequest will be set to false on the first OnStopRequest. - self->mManifestContent.Append(aFromRawSegment, aCount); - } - nsCOMPtr stream = CreateSharedStringStream(aFromRawSegment, aCount); return self->mVerifier->OnDataAvailable(nullptr, nullptr, stream, 0, aCount); } @@ -867,39 +858,11 @@ PackagedAppService::PackagedAppDownloader::NotifyOnStartSignedPackageRequest(con LOG(("Notifying the signed package is ready to load.")); } -void PackagedAppService::PackagedAppDownloader::InstallSignedPackagedApp(const ResourceCacheInfo* aInfo) +void PackagedAppService::PackagedAppDownloader::InstallSignedPackagedApp() { // TODO: Bug 1178533 to register permissions, system messages etc on navigation to // signed packages. LOG(("Install this packaged app.")); - bool isSuccess = false; - - nsCOMPtr installer = - do_GetService("@mozilla.org/newapps/installpackagedwebapp;1"); - - if (!installer) { - LOG(("InstallSignedPackagedApp: fail to get InstallPackagedWebapp service")); - return OnError(ERROR_GET_INSTALLER_FAILED); - } - - nsCString manifestURL; - aInfo->mURI->GetAsciiSpec(manifestURL); - - // Use the origin stored in the verifier since the signed packaged app would - // have a specifi package identifer defined in the manifest file. - nsCString packageOrigin; - mVerifier->GetPackageOrigin(packageOrigin); - - installer->InstallPackagedWebapp(mManifestContent.get(), - packageOrigin.get(), - manifestURL.get(), - &isSuccess); - if (!isSuccess) { - LOG(("InstallSignedPackagedApp: failed to install permissions")); - return OnError(ERROR_INSTALL_RESOURCE_FAILED); - } - - LOG(("InstallSignedPackagedApp: success.")); } //------------------------------------------------------------------ @@ -957,7 +920,7 @@ PackagedAppService::PackagedAppDownloader::OnManifestVerified(const ResourceCach nsCString packageOrigin; mVerifier->GetPackageOrigin(packageOrigin); NotifyOnStartSignedPackageRequest(packageOrigin); - InstallSignedPackagedApp(aInfo); + InstallSignedPackagedApp(); } void @@ -1124,7 +1087,7 @@ PackagedAppService::GetResource(nsIChannel *aChannel, downloader = new PackagedAppDownloader(); nsCString packageOrigin; - principal->GetOrigin(packageOrigin); + principal->GetOriginNoSuffix(packageOrigin); rv = downloader->Init(loadContextInfo, key, packageOrigin); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; diff --git a/netwerk/protocol/http/PackagedAppService.h b/netwerk/protocol/http/PackagedAppService.h index e1ae1b01d25..4dd04dd0080 100644 --- a/netwerk/protocol/http/PackagedAppService.h +++ b/netwerk/protocol/http/PackagedAppService.h @@ -106,8 +106,6 @@ private: enum EErrorType { ERROR_MANIFEST_VERIFIED_FAILED, ERROR_RESOURCE_VERIFIED_FAILED, - ERROR_GET_INSTALLER_FAILED, - ERROR_INSTALL_RESOURCE_FAILED, }; public: @@ -174,7 +172,7 @@ private: // Handle all tasks about app installation like permission and system message // registration. - void InstallSignedPackagedApp(const ResourceCacheInfo* aInfo); + void InstallSignedPackagedApp(); // Calls all the callbacks registered for the given URI. // aURI is the full URI of a subresource, composed of packageURI + !// + subresourcePath @@ -213,15 +211,6 @@ private: // If you need the origin with the signity taken into account, use // PackagedAppVerifier::GetPackageOrigin(). nsCString mPackageOrigin; - - //The app id of the package loaded from the LoadContextInfo - uint32_t mAppId; - - // A flag to indicate if we are processing the first request. - bool mProcessingFirstRequest; - - // A in-memory copy of the manifest content. - nsCString mManifestContent; }; // Intercepts OnStartRequest, OnDataAvailable*, OnStopRequest method calls diff --git a/toolkit/modules/BrowserUtils.jsm b/toolkit/modules/BrowserUtils.jsm index 920c18ade00..bdeacf91840 100644 --- a/toolkit/modules/BrowserUtils.jsm +++ b/toolkit/modules/BrowserUtils.jsm @@ -100,6 +100,30 @@ this.BrowserUtils = { return Services.io.newURI(aCPOWURI.spec, aCPOWURI.originCharset, null); }, + // Creates a codebase principal from a canonical origin string. This is + // the inverse operation of .origin on a codebase principal. + principalFromOrigin: function(aOriginString) { + if (aOriginString.startsWith('[')) { + throw new Error("principalFromOrigin does not support System and Expanded principals"); + } + + if (aOriginString.startsWith("moz-nullprincipal:")) { + throw new Error("principalFromOrigin does not support nsNullPrincipal"); + } + + var parts = aOriginString.split('^'); + if (parts.length > 2) { + throw new Error("bad origin string: " + aOriginString); + } + + var uri = Services.io.newURI(parts[0], null, null); + var attrs = {}; + // Parse the parameters string into a dictionary. + (parts[1] || "").split("&").map((x) => x.split('=')).forEach((x) => attrs[x[0]] = x[1]); + + return Services.scriptSecurityManager.createCodebasePrincipal(uri, attrs); + }, + /** * For a given DOM element, returns its position in "screen" * coordinates. In a content process, the coordinates returned will diff --git a/toolkit/modules/PermissionsUtils.jsm b/toolkit/modules/PermissionsUtils.jsm index 3f731a3fd94..3f95437fb24 100644 --- a/toolkit/modules/PermissionsUtils.jsm +++ b/toolkit/modules/PermissionsUtils.jsm @@ -29,7 +29,7 @@ function importPrefBranch(aPrefBranch, aPermission, aAction) { for (let origin of origins) { let principals = []; try { - principals = [ Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin) ]; + principals = [ BrowserUtils.principalFromOrigin(origin) ]; } catch (e) { // This preference used to contain a list of hosts. For back-compat // reasons, we convert these hosts into http:// and https:// permissions diff --git a/toolkit/modules/PermissionsUtils.jsm.rej b/toolkit/modules/PermissionsUtils.jsm.rej deleted file mode 100644 index 2e1e2cbb904..00000000000 --- a/toolkit/modules/PermissionsUtils.jsm.rej +++ /dev/null @@ -1,10 +0,0 @@ -diff a/toolkit/modules/PermissionsUtils.jsm b/toolkit/modules/PermissionsUtils.jsm (rejected hunks) -@@ -7,8 +7,6 @@ this.EXPORTED_SYMBOLS = ["PermissionsUtils"]; - const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; - - Cu.import("resource://gre/modules/Services.jsm"); --Cu.import("resource://gre/modules/BrowserUtils.jsm") -- - - let gImportedPrefBranches = new Set(); - From b7749e5a96369478e36a338c427de47a1e973adb Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 09:29:54 +1000 Subject: [PATCH 16/78] Bug 1207478: P1. Prevent microseconds calculation overflow. r=gerald This leaves us safe until year 294441 (assuming unix timestamp). --- media/libstagefright/binding/include/mp4_demuxer/MoofParser.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h index 4b1740b1e23..3d827013302 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h +++ b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h @@ -35,7 +35,9 @@ public: Microseconds ToMicroseconds(int64_t aTimescaleUnits) { - return aTimescaleUnits * 1000000ll / mTimescale; + int64_t major = aTimescaleUnits / mTimescale; + int64_t remainder = aTimescaleUnits % mTimescale; + return major * 1000000ll + remainder * 1000000ll / mTimescale; } uint64_t mCreationTime; From e2c131f5a9805ec4e1faa730e12a53e651877ac0 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Wed, 23 Sep 2015 23:17:11 +1000 Subject: [PATCH 17/78] Bug 1207478: P2. Prevent potential division by zero. r=gerald --- media/libstagefright/binding/MoofParser.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/media/libstagefright/binding/MoofParser.cpp b/media/libstagefright/binding/MoofParser.cpp index 12c7944e4cd..40caeb5bca6 100644 --- a/media/libstagefright/binding/MoofParser.cpp +++ b/media/libstagefright/binding/MoofParser.cpp @@ -649,7 +649,9 @@ Mvhd::Mvhd(Box& aBox) } // More stuff that we don't care about reader->DiscardRemaining(); - mValid = true; + if (mTimescale) { + mValid = true; + } } Mdhd::Mdhd(Box& aBox) From 2b4422f3589da3633c2017994ca4681e836586d2 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 10:42:54 +1000 Subject: [PATCH 18/78] Bug 1207478: P3. Reduce chance of overflow. r=jwwang --- dom/media/VideoUtils.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dom/media/VideoUtils.cpp b/dom/media/VideoUtils.cpp index fd42792e926..ca8ea60f242 100644 --- a/dom/media/VideoUtils.cpp +++ b/dom/media/VideoUtils.cpp @@ -25,20 +25,29 @@ namespace mozilla { using layers::PlanarYCbCrImage; +static inline CheckedInt64 SaferMultDiv(int64_t aValue, uint32_t aMul, uint32_t aDiv) { + int64_t major = aValue / aDiv; + int64_t remainder = aValue % aDiv; + return CheckedInt64(remainder) * aMul / aDiv + major * aMul; +} + // Converts from number of audio frames to microseconds, given the specified // audio rate. CheckedInt64 FramesToUsecs(int64_t aFrames, uint32_t aRate) { - return (CheckedInt64(aFrames) * USECS_PER_S) / aRate; + return SaferMultDiv(aFrames, USECS_PER_S, aRate); } media::TimeUnit FramesToTimeUnit(int64_t aFrames, uint32_t aRate) { - return (media::TimeUnit::FromMicroseconds(aFrames) * USECS_PER_S) / aRate; + int64_t major = aFrames / aRate; + int64_t remainder = aFrames % aRate; + return media::TimeUnit::FromMicroseconds(major) * USECS_PER_S + + (media::TimeUnit::FromMicroseconds(remainder) * USECS_PER_S) / aRate; } // Converts from microseconds to number of audio frames, given the specified // audio rate. CheckedInt64 UsecsToFrames(int64_t aUsecs, uint32_t aRate) { - return (CheckedInt64(aUsecs) * aRate) / USECS_PER_S; + return SaferMultDiv(aUsecs, aRate, USECS_PER_S); } // Format TimeUnit as number of frames at given rate. From 6fec0200f7eedb52e9626c0e1c8b046f559413bf Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 15:36:51 +1000 Subject: [PATCH 19/78] Bug 1207478: P4: Make it impossible for the MDSM frame calculation to overflow. r=jwwang This should make P3 unnecessary as far as the MDSM is concerned. --- dom/media/MediaDecoderStateMachine.cpp | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index f5bd9aec891..3a85948d3cd 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -2743,19 +2743,19 @@ MediaDecoderStateMachine::DropAudioUpToSeekTarget(MediaData* aSample) mCurrentSeek.Exists() && mCurrentSeek.mTarget.mType == SeekTarget::Accurate); - CheckedInt64 startFrame = UsecsToFrames(audio->mTime, - mInfo.mAudio.mRate); - CheckedInt64 targetFrame = UsecsToFrames(mCurrentSeek.mTarget.mTime, - mInfo.mAudio.mRate); - if (!startFrame.isValid() || !targetFrame.isValid()) { + CheckedInt64 sampleDuration = + FramesToUsecs(audio->mFrames, mInfo.mAudio.mRate); + if (!sampleDuration.isValid()) { return NS_ERROR_FAILURE; } - if (startFrame.value() + audio->mFrames <= targetFrame.value()) { + + if (audio->mTime + sampleDuration.value() <= mCurrentSeek.mTarget.mTime) { // Our seek target lies after the frames in this AudioData. Don't // push it onto the audio queue, and keep decoding forwards. return NS_OK; } - if (startFrame.value() > targetFrame.value()) { + + if (audio->mTime > mCurrentSeek.mTarget.mTime) { // The seek target doesn't lie in the audio block just after the last // audio frames we've seen which were before the seek target. This // could have been the first audio data we've seen after seek, i.e. the @@ -2771,23 +2771,27 @@ MediaDecoderStateMachine::DropAudioUpToSeekTarget(MediaData* aSample) // The seek target lies somewhere in this AudioData's frames, strip off // any frames which lie before the seek target, so we'll begin playback // exactly at the seek target. - NS_ASSERTION(targetFrame.value() >= startFrame.value(), + NS_ASSERTION(mCurrentSeek.mTarget.mTime >= audio->mTime, "Target must at or be after data start."); - NS_ASSERTION(targetFrame.value() < startFrame.value() + audio->mFrames, + NS_ASSERTION(mCurrentSeek.mTarget.mTime < audio->mTime + sampleDuration.value(), "Data must end after target."); - int64_t framesToPrune = targetFrame.value() - startFrame.value(); - if (framesToPrune > audio->mFrames) { + CheckedInt64 framesToPrune = + UsecsToFrames(mCurrentSeek.mTarget.mTime - audio->mTime, mInfo.mAudio.mRate); + if (!framesToPrune.isValid()) { + return NS_ERROR_FAILURE; + } + if (framesToPrune.value() > audio->mFrames) { // We've messed up somehow. Don't try to trim frames, the |frames| // variable below will overflow. DECODER_WARN("Can't prune more frames that we have!"); return NS_ERROR_FAILURE; } - uint32_t frames = audio->mFrames - static_cast(framesToPrune); + uint32_t frames = audio->mFrames - static_cast(framesToPrune.value()); uint32_t channels = audio->mChannels; nsAutoArrayPtr audioData(new AudioDataValue[frames * channels]); memcpy(audioData.get(), - audio->mAudioData.get() + (framesToPrune * channels), + audio->mAudioData.get() + (framesToPrune.value() * channels), frames * channels * sizeof(AudioDataValue)); CheckedInt64 duration = FramesToUsecs(frames, mInfo.mAudio.mRate); if (!duration.isValid()) { From e4cef95c1f8dd637789e16ce124d7545665f85ae Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 13:05:01 +1000 Subject: [PATCH 20/78] Bug 1206211: P1. Ensure operation can't overflow. r=kentuckyfriedtakahe --- .../frameworks/av/media/libstagefright/MPEG4Extractor.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp index db52e61f997..544e50410fe 100644 --- a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp @@ -507,10 +507,13 @@ status_t MPEG4Extractor::readMetaData() { CHECK_NE(err, (status_t)NO_INIT); // copy pssh data into file metadata - int psshsize = 0; + uint64_t psshsize = 0; for (size_t i = 0; i < mPssh.Length(); i++) { psshsize += 20 + mPssh[i].datalen; } + if (psshsize > kMAX_ALLOCATION) { + return ERROR_MALFORMED; + } if (psshsize) { char *buf = (char*)malloc(psshsize); char *ptr = buf; From b215de8c9af0acfda3d5db2bbf5cfe09acfc4315 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 13:08:28 +1000 Subject: [PATCH 21/78] Bug 1206211: P2. Abort on OOM. r=kentuckyfriedtakahe --- .../frameworks/av/media/libstagefright/MPEG4Extractor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp index 544e50410fe..f08c94b7912 100644 --- a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp @@ -516,6 +516,9 @@ status_t MPEG4Extractor::readMetaData() { } if (psshsize) { char *buf = (char*)malloc(psshsize); + if (!buf) { + return ERROR_MALFORMED; + } char *ptr = buf; for (size_t i = 0; i < mPssh.Length(); i++) { memcpy(ptr, mPssh[i].uuid, 20); // uuid + length From f3f19c5088164b7dee6f32409abab0bef71b0592 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 24 Sep 2015 16:46:14 +1000 Subject: [PATCH 22/78] Bug 1159343: Interrupt seek early when possible. r=jwwang --- dom/media/MediaDecoderStateMachine.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 3a85948d3cd..af328a88646 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -932,8 +932,10 @@ MediaDecoderStateMachine::OnVideoDecoded(MediaData* aVideoSample) // in this case, we'll just decode forward. Bug 1026330. mCurrentSeek.mTarget.mType = SeekTarget::Accurate; } - if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint) { - // Non-precise seek; we can stop the seek at the first sample. + if (mCurrentSeek.mTarget.mType == SeekTarget::PrevSyncPoint || + mPendingSeek.Exists()) { + // Non-precise seek; or a pending seek exists ; we can stop the seek + // at the first sample. Push(video, MediaData::VIDEO_DATA); } else { // We're doing an accurate seek. We still need to discard From 4d1aa7fcf71b9b39189d385564a86fe9c0dbc625 Mon Sep 17 00:00:00 2001 From: JW Wang Date: Thu, 24 Sep 2015 14:29:01 +0800 Subject: [PATCH 23/78] Bug 1207915 - Apply the fix of bug 1052206 to DecodedStream. r=roc. --- dom/media/mediasink/DecodedStream.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp index 11ad564f844..0b0405d79da 100644 --- a/dom/media/mediasink/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -605,6 +605,9 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, MediaData* aData, AudioSegment* aOutput, uint32_t aRate, double aVolume) { + // The amount of audio frames that is used to fuzz rounding errors. + static const int64_t AUDIO_FUZZ_FRAMES = 1; + MOZ_ASSERT(aData); AudioData* audio = aData->As(); // This logic has to mimic AudioSink closely to make sure we write @@ -616,11 +619,11 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, if (!audioWrittenOffset.isValid() || !frameOffset.isValid() || // ignore packet that we've already processed - frameOffset.value() + audio->mFrames <= audioWrittenOffset.value()) { + frameOffset.value() < audioWrittenOffset.value()) { return; } - if (audioWrittenOffset.value() < frameOffset.value()) { + if (audioWrittenOffset.value() + AUDIO_FUZZ_FRAMES < frameOffset.value()) { int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value(); // Write silence to catch up AudioSegment silence; @@ -630,20 +633,17 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, aOutput->AppendFrom(&silence); } - MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value()); - - int64_t offset = audioWrittenOffset.value() - frameOffset.value(); - size_t framesToWrite = audio->mFrames - offset; - + // Always write the whole sample without truncation to be consistent with + // DecodedAudioDataSink::PlayFromAudioQueue() audio->EnsureAudioBuffer(); nsRefPtr buffer = audio->mAudioBuffer; AudioDataValue* bufferData = static_cast(buffer->Data()); nsAutoTArray channels; for (uint32_t i = 0; i < audio->mChannels; ++i) { - channels.AppendElement(bufferData + i * audio->mFrames + offset); + channels.AppendElement(bufferData + i * audio->mFrames); } - aOutput->AppendFrames(buffer.forget(), channels, framesToWrite); - aStream->mAudioFramesWritten += framesToWrite; + aOutput->AppendFrames(buffer.forget(), channels, audio->mFrames); + aStream->mAudioFramesWritten += audio->mFrames; aOutput->ApplyVolume(aVolume); aStream->mNextAudioTime = audio->GetEndTime(); From 7b89ede8a65530092fa7ab41c2e3eda25f5f9018 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 24 Sep 2015 12:31:20 +0530 Subject: [PATCH 24/78] Backed out changeset dcbabf6ce153 (bug 1188643) for linux crashtest bustage --- dom/media/MediaDecoderStateMachine.cpp | 37 ++++++++------------------ dom/media/MediaDecoderStateMachine.h | 6 ++--- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index af328a88646..db40b9397a7 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -205,7 +205,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mQuickBufferingLowDataThresholdUsecs(detail::QUICK_BUFFERING_LOW_DATA_USECS), mIsAudioPrerolling(false), mIsVideoPrerolling(false), - mAudioCaptured(false, "MediaDecoderStateMachine::mAudioCaptured"), + mAudioCaptured(false), mAudioCompleted(false, "MediaDecoderStateMachine::mAudioCompleted"), mNotifyMetadataBeforeFirstFrame(false), mDispatchedEventToDecode(false), @@ -219,7 +219,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mCorruptFrames(60), mDecodingFirstFrame(true), mSentLoadedMetadataEvent(false), - mSentFirstFrameLoadedEvent(false, "MediaDecoderStateMachine::mSentFirstFrameLoadedEvent"), + mSentFirstFrameLoadedEvent(false), mSentPlaybackEndedEvent(false), mStreamSink(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)), mResource(aDecoder->GetResource()), @@ -350,8 +350,6 @@ MediaDecoderStateMachine::InitializationTask() mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged); mWatchManager.Watch(mLogicallySeeking, &MediaDecoderStateMachine::LogicallySeekingChanged); mWatchManager.Watch(mSameOriginMedia, &MediaDecoderStateMachine::SameOriginMediaChanged); - mWatchManager.Watch(mSentFirstFrameLoadedEvent, &MediaDecoderStateMachine::AdjustAudioThresholds); - mWatchManager.Watch(mAudioCaptured, &MediaDecoderStateMachine::AdjustAudioThresholds); // Propagate mSameOriginMedia to mDecodedStream. SameOriginMediaChanged(); @@ -2072,28 +2070,6 @@ MediaDecoderStateMachine::IsDecodingFirstFrame() return mState == DECODER_STATE_DECODING && mDecodingFirstFrame; } -void -MediaDecoderStateMachine::AdjustAudioThresholds() -{ - MOZ_ASSERT(OnTaskQueue()); - AssertCurrentThreadInMonitor(); - - // Experiments show that we need to buffer more if audio is captured to avoid - // audio glitch. See bug 1188643 comment 16 for the details. - auto divisor = mAudioCaptured ? NO_VIDEO_AMPLE_AUDIO_DIVISOR / 2 - : NO_VIDEO_AMPLE_AUDIO_DIVISOR; - - // We're playing audio only. We don't need to worry about slow video - // decodes causing audio underruns, so don't buffer so much audio in - // order to reduce memory usage. - if (HasAudio() && !HasVideo() && mSentFirstFrameLoadedEvent) { - mAmpleAudioThresholdUsecs = detail::AMPLE_AUDIO_USECS / divisor; - mLowAudioThresholdUsecs = detail::LOW_AUDIO_USECS / divisor; - mQuickBufferingLowDataThresholdUsecs = - detail::QUICK_BUFFERING_LOW_DATA_USECS / divisor; - } -} - void MediaDecoderStateMachine::FinishDecodeFirstFrame() { @@ -2114,6 +2090,15 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame() "transportSeekable=%d, mediaSeekable=%d", Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable.Ref()); + if (HasAudio() && !HasVideo() && !mSentFirstFrameLoadedEvent) { + // We're playing audio only. We don't need to worry about slow video + // decodes causing audio underruns, so don't buffer so much audio in + // order to reduce memory usage. + mAmpleAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; + mLowAudioThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; + mQuickBufferingLowDataThresholdUsecs /= NO_VIDEO_AMPLE_AUDIO_DIVISOR; + } + // Get potentially updated metadata { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index 0f2ee96c26e..f10b28bb41e 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -654,8 +654,6 @@ private: // play time. bool NeedToSkipToNextKeyframe(); - void AdjustAudioThresholds(); - // The decoder object that created this state machine. The state machine // holds a strong reference to the decoder to ensure that the decoder stays // alive once media element has started the decoder shutdown process, and has @@ -1138,7 +1136,7 @@ private: // True if we shouldn't play our audio (but still write it to any capturing // streams). When this is true, the audio thread will never start again after // it has stopped. - Watchable mAudioCaptured; + bool mAudioCaptured; // True if the audio playback thread has finished. It is finished // when either all the audio frames have completed playing, or we've moved @@ -1230,7 +1228,7 @@ private: // SetStartTime because the mStartTime already set before. Also we don't need // to decode any audio/video since the MediaDecoder will trigger a seek // operation soon. - Watchable mSentFirstFrameLoadedEvent; + bool mSentFirstFrameLoadedEvent; bool mSentPlaybackEndedEvent; From 159d6449040fb48c2fb6a252a481b89e4b6354cd Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 24 Sep 2015 12:37:52 +0530 Subject: [PATCH 25/78] Backed out changeset 49847eb6c1ce (bug 1180940) for Mulet Gij(4) bustage on CLOSED TREE --- dom/audiochannel/AudioChannelAgent.cpp | 6 +----- dom/media/webaudio/AudioContext.cpp | 24 +++++---------------- dom/media/webaudio/AudioContext.h | 2 +- dom/media/webaudio/AudioDestinationNode.cpp | 20 +++++------------ dom/media/webaudio/AudioDestinationNode.h | 2 +- 5 files changed, 13 insertions(+), 41 deletions(-) diff --git a/dom/audiochannel/AudioChannelAgent.cpp b/dom/audiochannel/AudioChannelAgent.cpp index b8f0009fbe8..8c4106b9100 100644 --- a/dom/audiochannel/AudioChannelAgent.cpp +++ b/dom/audiochannel/AudioChannelAgent.cpp @@ -104,7 +104,7 @@ AudioChannelAgent::InitInternal(nsIDOMWindow* aWindow, int32_t aChannelType, } if (NS_WARN_IF(!aWindow)) { - return NS_OK; + return NS_ERROR_FAILURE; } nsCOMPtr pInnerWindow = do_QueryInterface(aWindow); @@ -113,10 +113,6 @@ AudioChannelAgent::InitInternal(nsIDOMWindow* aWindow, int32_t aChannelType, nsCOMPtr topWindow; aWindow->GetScriptableTop(getter_AddRefs(topWindow)); - if (NS_WARN_IF(!topWindow)) { - return NS_OK; - } - mWindow = do_QueryInterface(topWindow); if (mWindow) { mWindow = mWindow->GetOuterWindow(); diff --git a/dom/media/webaudio/AudioContext.cpp b/dom/media/webaudio/AudioContext.cpp index 8a474905f35..4e6a622f355 100644 --- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -117,22 +117,14 @@ AudioContext::AudioContext(nsPIDOMWindow* aWindow, } } -nsresult +void AudioContext::Init() { // We skip calling SetIsOnlyNodeForContext and the creation of the // audioChannelAgent during mDestination's constructor, because we can only // call them after mDestination has been set up. - if (mIsOffline) { - mDestination->SetIsOnlyNodeForContext(true); - } else { - nsresult rv = mDestination->CreateAudioChannelAgent(); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - } - - return NS_OK; + mDestination->CreateAudioChannelAgent(); + mDestination->SetIsOnlyNodeForContext(true); } AudioContext::~AudioContext() @@ -168,10 +160,7 @@ AudioContext::Constructor(const GlobalObject& aGlobal, nsRefPtr object = new AudioContext(window, false, AudioChannelService::GetDefaultAudioChannel()); - aRv = object->Init(); - if (NS_WARN_IF(aRv.Failed())) { - return nullptr; - } + object->Init(); RegisterWeakMemoryReporter(object); @@ -190,10 +179,7 @@ AudioContext::Constructor(const GlobalObject& aGlobal, } nsRefPtr object = new AudioContext(window, false, aChannel); - aRv = object->Init(); - if (NS_WARN_IF(aRv.Failed())) { - return nullptr; - } + object->Init(); RegisterWeakMemoryReporter(object); diff --git a/dom/media/webaudio/AudioContext.h b/dom/media/webaudio/AudioContext.h index 9acc8b8ec77..256010e7d0d 100644 --- a/dom/media/webaudio/AudioContext.h +++ b/dom/media/webaudio/AudioContext.h @@ -124,7 +124,7 @@ class AudioContext final : public DOMEventTargetHelper, float aSampleRate = 0.0f); ~AudioContext(); - nsresult Init(); + void Init(); public: typedef uint64_t AudioContextId; diff --git a/dom/media/webaudio/AudioDestinationNode.cpp b/dom/media/webaudio/AudioDestinationNode.cpp index c620c17e773..cb7bc010a6c 100644 --- a/dom/media/webaudio/AudioDestinationNode.cpp +++ b/dom/media/webaudio/AudioDestinationNode.cpp @@ -630,33 +630,23 @@ AudioDestinationNode::CheckAudioChannelPermissions(AudioChannel aValue) return perm == nsIPermissionManager::ALLOW_ACTION; } -nsresult +void AudioDestinationNode::CreateAudioChannelAgent() { if (mIsOffline) { - return NS_OK; + return; } - nsresult rv = NS_OK; if (mAudioChannelAgent) { - rv = mAudioChannelAgent->NotifyStoppedPlaying(nsIAudioChannelAgent::AUDIO_AGENT_NOTIFY); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } + mAudioChannelAgent->NotifyStoppedPlaying(nsIAudioChannelAgent::AUDIO_AGENT_NOTIFY); } mAudioChannelAgent = new AudioChannelAgent(); - rv = mAudioChannelAgent->InitWithWeakCallback(GetOwner(), + mAudioChannelAgent->InitWithWeakCallback(GetOwner(), static_cast(mAudioChannel), this); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - rv = WindowAudioCaptureChanged(); - NS_WARN_IF(NS_FAILED(rv)); - return rv; + WindowAudioCaptureChanged(); } void diff --git a/dom/media/webaudio/AudioDestinationNode.h b/dom/media/webaudio/AudioDestinationNode.h index 44fc06c98c4..632c2453e68 100644 --- a/dom/media/webaudio/AudioDestinationNode.h +++ b/dom/media/webaudio/AudioDestinationNode.h @@ -73,7 +73,7 @@ public: // When aIsOnlyNode is true, this is the only node for the AudioContext. void SetIsOnlyNodeForContext(bool aIsOnlyNode); - nsresult CreateAudioChannelAgent(); + void CreateAudioChannelAgent(); void DestroyAudioChannelAgent(); virtual const char* NodeType() const override From 81770c36ba314774eaf9553d8f75314190edb34c Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 26/78] Bug 1193519 pt 1 - Update coordinate conversions in WritingModes.h to account for sideways-lr writing mode. r=dholbert * * * Bug 1193519 pt 1.1 - Fix error in LogicalRect::LineLeft/Right accessors. r=dholbert --- layout/generic/WritingModes.h | 102 ++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/layout/generic/WritingModes.h b/layout/generic/WritingModes.h index 36286cc5b92..fa78609a485 100644 --- a/layout/generic/WritingModes.h +++ b/layout/generic/WritingModes.h @@ -204,7 +204,15 @@ public: BidiDir GetBidiDir() const { return BidiDir(mWritingMode & eBidiMask); } /** - * Return true if LTR. (Convenience method) + * Return true if the inline flow direction is against physical direction + * (i.e. right-to-left or bottom-to-top). + * This occurs when writing-mode is sideways-lr OR direction is rtl (but not + * if both of those are true). + */ + bool IsInlineReversed() const { return !!(mWritingMode & eInlineFlowMask); } + + /** + * Return true if bidi direction is LTR. (Convenience method) */ bool IsBidiLTR() const { return eBidiLTR == GetBidiDir(); } @@ -426,7 +434,7 @@ public: { auto side = static_cast(aDir); if (IsInline(side)) { - return IsBidiLTR() ? side : GetOppositeSide(side); + return !IsInlineReversed() ? side : GetOppositeSide(side); } return !IsLineInverted() ? side : GetOppositeSide(side); } @@ -686,13 +694,13 @@ public: #endif { if (aWritingMode.IsVertical()) { - I() = aWritingMode.IsBidiLTR() ? aPoint.y - : aContainerSize.height - aPoint.y; + I() = aWritingMode.IsInlineReversed() ? aContainerSize.height - aPoint.y + : aPoint.y; B() = aWritingMode.IsVerticalLR() ? aPoint.x : aContainerSize.width - aPoint.x; } else { - I() = aWritingMode.IsBidiLTR() ? aPoint.x - : aContainerSize.width - aPoint.x; + I() = aWritingMode.IsInlineReversed() ? aContainerSize.width - aPoint.x + : aPoint.x; B() = aPoint.y; } } @@ -737,11 +745,11 @@ public: if (aWritingMode.IsVertical()) { return nsPoint(aWritingMode.IsVerticalLR() ? B() : aContainerSize.width - B(), - aWritingMode.IsBidiLTR() - ? I() : aContainerSize.height - I()); + aWritingMode.IsInlineReversed() + ? aContainerSize.height - I() : I()); } else { - return nsPoint(aWritingMode.IsBidiLTR() - ? I() : aContainerSize.width - I(), + return nsPoint(aWritingMode.IsInlineReversed() + ? aContainerSize.width - I() : I(), B()); } } @@ -1097,22 +1105,22 @@ public: mMargin.top = aPhysicalMargin.right; mMargin.bottom = aPhysicalMargin.left; } - if (aWritingMode.IsBidiLTR()) { - mMargin.left = aPhysicalMargin.top; - mMargin.right = aPhysicalMargin.bottom; - } else { + if (aWritingMode.IsInlineReversed()) { mMargin.left = aPhysicalMargin.bottom; mMargin.right = aPhysicalMargin.top; + } else { + mMargin.left = aPhysicalMargin.top; + mMargin.right = aPhysicalMargin.bottom; } } else { mMargin.top = aPhysicalMargin.top; mMargin.bottom = aPhysicalMargin.bottom; - if (aWritingMode.IsBidiLTR()) { - mMargin.left = aPhysicalMargin.left; - mMargin.right = aPhysicalMargin.right; - } else { + if (aWritingMode.IsInlineReversed()) { mMargin.left = aPhysicalMargin.right; mMargin.right = aPhysicalMargin.left; + } else { + mMargin.left = aPhysicalMargin.left; + mMargin.right = aPhysicalMargin.right; } } } @@ -1211,14 +1219,14 @@ public: { CHECK_WRITING_MODE(aWritingMode); return aWritingMode.IsVertical() ? - (aWritingMode.IsBidiLTR() ? IStart() : IEnd()) : BStart(); + (aWritingMode.IsInlineReversed() ? IEnd() : IStart()) : BStart(); } nscoord Bottom(WritingMode aWritingMode) const { CHECK_WRITING_MODE(aWritingMode); return aWritingMode.IsVertical() ? - (aWritingMode.IsBidiLTR() ? IEnd() : IStart()) : BEnd(); + (aWritingMode.IsInlineReversed() ? IStart() : IEnd()) : BEnd(); } nscoord Left(WritingMode aWritingMode) const @@ -1226,7 +1234,7 @@ public: CHECK_WRITING_MODE(aWritingMode); return aWritingMode.IsVertical() ? (aWritingMode.IsVerticalLR() ? BStart() : BEnd()) : - (aWritingMode.IsBidiLTR() ? IStart() : IEnd()); + (aWritingMode.IsInlineReversed() ? IEnd() : IStart()); } nscoord Right(WritingMode aWritingMode) const @@ -1234,7 +1242,7 @@ public: CHECK_WRITING_MODE(aWritingMode); return aWritingMode.IsVertical() ? (aWritingMode.IsVerticalLR() ? BEnd() : BStart()) : - (aWritingMode.IsBidiLTR() ? IEnd() : IStart()); + (aWritingMode.IsInlineReversed() ? IStart() : IEnd()); } nscoord LeftRight(WritingMode aWritingMode) const @@ -1264,15 +1272,15 @@ public: CHECK_WRITING_MODE(aWritingMode); return aWritingMode.IsVertical() ? (aWritingMode.IsVerticalLR() - ? (aWritingMode.IsBidiLTR() - ? nsMargin(IStart(), BEnd(), IEnd(), BStart()) - : nsMargin(IEnd(), BEnd(), IStart(), BStart())) - : (aWritingMode.IsBidiLTR() - ? nsMargin(IStart(), BStart(), IEnd(), BEnd()) - : nsMargin(IEnd(), BStart(), IStart(), BEnd()))) - : (aWritingMode.IsBidiLTR() - ? nsMargin(BStart(), IEnd(), BEnd(), IStart()) - : nsMargin(BStart(), IStart(), BEnd(), IEnd())); + ? (aWritingMode.IsInlineReversed() + ? nsMargin(IEnd(), BEnd(), IStart(), BStart()) + : nsMargin(IStart(), BEnd(), IEnd(), BStart())) + : (aWritingMode.IsInlineReversed() + ? nsMargin(IEnd(), BStart(), IStart(), BEnd()) + : nsMargin(IStart(), BStart(), IEnd(), BEnd()))) + : (aWritingMode.IsInlineReversed() + ? nsMargin(BStart(), IStart(), BEnd(), IEnd()) + : nsMargin(BStart(), IEnd(), BEnd(), IStart())); } /** @@ -1439,13 +1447,13 @@ public: if (aWritingMode.IsVertical()) { mRect.y = aWritingMode.IsVerticalLR() ? aRect.x : aContainerSize.width - aRect.XMost(); - mRect.x = aWritingMode.IsBidiLTR() - ? aRect.y : aContainerSize.height - aRect.YMost(); + mRect.x = aWritingMode.IsInlineReversed() + ? aContainerSize.height - aRect.YMost() : aRect.y; mRect.height = aRect.width; mRect.width = aRect.height; } else { - mRect.x = aWritingMode.IsBidiLTR() - ? aRect.x : aContainerSize.width - aRect.XMost(); + mRect.x = aWritingMode.IsInlineReversed() + ? aContainerSize.width - aRect.XMost() : aRect.x; mRect.y = aRect.y; mRect.width = aRect.width; mRect.height = aRect.height; @@ -1548,8 +1556,8 @@ public: return aWritingMode.IsVerticalLR() ? mRect.Y() : aContainerWidth - mRect.YMost(); } else { - return aWritingMode.IsBidiLTR() ? - mRect.X() : aContainerWidth - mRect.XMost(); + return aWritingMode.IsInlineReversed() ? + aContainerWidth - mRect.XMost() : mRect.X(); } } @@ -1557,8 +1565,8 @@ public: { CHECK_WRITING_MODE(aWritingMode); if (aWritingMode.IsVertical()) { - return aWritingMode.IsBidiLTR() ? mRect.X() - : aContainerHeight - mRect.XMost(); + return aWritingMode.IsInlineReversed() ? aContainerHeight - mRect.XMost() + : mRect.X(); } else { return mRect.Y(); } @@ -1583,8 +1591,8 @@ public: return aWritingMode.IsVerticalLR() ? mRect.YMost() : aContainerWidth - mRect.Y(); } else { - return aWritingMode.IsBidiLTR() ? - mRect.XMost() : aContainerWidth - mRect.X(); + return aWritingMode.IsInlineReversed() ? + aContainerWidth - mRect.X() : mRect.XMost(); } } @@ -1592,8 +1600,8 @@ public: { CHECK_WRITING_MODE(aWritingMode); if (aWritingMode.IsVertical()) { - return aWritingMode.IsBidiLTR() ? mRect.XMost() - : aContainerHeight - mRect.x; + return aWritingMode.IsInlineReversed() ? aContainerHeight - mRect.x + : mRect.XMost(); } else { return mRect.YMost(); } @@ -1707,12 +1715,12 @@ public: if (aWritingMode.IsVertical()) { return nsRect(aWritingMode.IsVerticalLR() ? BStart() : aContainerSize.width - BEnd(), - aWritingMode.IsBidiLTR() - ? IStart() : aContainerSize.height - IEnd(), + aWritingMode.IsInlineReversed() + ? aContainerSize.height - IEnd() : IStart(), BSize(), ISize()); } else { - return nsRect(aWritingMode.IsBidiLTR() - ? IStart() : aContainerSize.width - IEnd(), + return nsRect(aWritingMode.IsInlineReversed() + ? aContainerSize.width - IEnd() : IStart(), BStart(), ISize(), BSize()); } } From 93aeab026e5495762ddd8c9caf0096c3c2ce1f55 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 27/78] Bug 1193519 pt 2 - Handle sideways-left orientation in gfx text-drawing code. r=dholbert --- gfx/thebes/gfxFont.cpp | 29 +++++++++++++++++++++-------- gfx/thebes/gfxFont.h | 13 ++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/gfx/thebes/gfxFont.cpp b/gfx/thebes/gfxFont.cpp index 1aee28739ed..ae21f69f08e 100644 --- a/gfx/thebes/gfxFont.cpp +++ b/gfx/thebes/gfxFont.cpp @@ -1742,7 +1742,8 @@ gfxFont::DrawGlyphs(gfxShapedText *aShapedText, gfxFloat& inlineCoord = aFontParams.isVerticalFont ? aPt->y : aPt->x; if (aRunParams.spacing) { - inlineCoord += aRunParams.direction * aRunParams.spacing[0].mBefore; + inlineCoord += aRunParams.isRTL ? -aRunParams.spacing[0].mBefore + : aRunParams.spacing[0].mBefore; } const gfxShapedText::CompressedGlyph *glyphData = @@ -1821,7 +1822,7 @@ gfxFont::DrawGlyphs(gfxShapedText *aShapedText, buffer, &emittedGlyphs); } - inlineCoord += aRunParams.direction * advance; + inlineCoord += aRunParams.isRTL ? -advance : advance; } } } @@ -1831,7 +1832,7 @@ gfxFont::DrawGlyphs(gfxShapedText *aShapedText, if (i + 1 < aCount) { space += aRunParams.spacing[i + 1].mBefore; } - inlineCoord += aRunParams.direction * space; + inlineCoord += aRunParams.isRTL ? -space : space; } } @@ -1874,10 +1875,15 @@ gfxFont::Draw(gfxTextRun *aTextRun, uint32_t aStart, uint32_t aEnd, const Metrics& metrics = GetMetrics(eHorizontal); // Get a matrix we can use to draw the (horizontally-shaped) textrun // with 90-degree CW rotation. - gfxMatrix mat = aRunParams.context->CurrentMatrix(). - Translate(p). // translate origin for rotation - Rotate(M_PI / 2.0). // turn 90deg clockwise - Translate(-p); // undo the translation + const gfxFloat + rotation = (aOrientation == + gfxTextRunFactory::TEXT_ORIENT_VERTICAL_SIDEWAYS_LEFT) + ? -M_PI / 2.0 : M_PI / 2.0; + gfxMatrix mat = + aRunParams.context->CurrentMatrix(). + Translate(p). // translate origin for rotation + Rotate(rotation). // turn 90deg CCW (sideways-left) or CW (*-right) + Translate(-p); // undo the translation // If we're drawing rotated horizontal text for an element styled // text-orientation:mixed, the dominant baseline will be vertical- @@ -1987,7 +1993,14 @@ gfxFont::Draw(gfxTextRun *aTextRun, uint32_t aStart, uint32_t aEnd, if (sideways) { aRunParams.context->Restore(); - *aPt = gfxPoint(origPt.x, origPt.y + (aPt->x - origPt.x)); + // adjust updated aPt to account for the transform we were using + gfxFloat advance = aPt->x - origPt.x; + if (aOrientation == + gfxTextRunFactory::TEXT_ORIENT_VERTICAL_SIDEWAYS_LEFT) { + *aPt = gfxPoint(origPt.x, origPt.y - advance); + } else { + *aPt = gfxPoint(origPt.x, origPt.y + advance); + } } } diff --git a/gfx/thebes/gfxFont.h b/gfx/thebes/gfxFont.h index c3f34a760b2..cb8f97dd2ae 100644 --- a/gfx/thebes/gfxFont.h +++ b/gfx/thebes/gfxFont.h @@ -965,8 +965,19 @@ public: return (GetFlags() & gfxTextRunFactory::TEXT_IS_RTL) != 0; } + bool IsSidewaysLeft() const { + return (GetFlags() & gfxTextRunFactory::TEXT_ORIENT_MASK) == + gfxTextRunFactory::TEXT_ORIENT_VERTICAL_SIDEWAYS_LEFT; + } + + // Return true if the logical inline direction is reversed compared to + // normal physical coordinates (i.e. if it is leftwards or upwards) + bool IsInlineReversed() const { + return IsSidewaysLeft() != IsRightToLeft(); + } + gfxFloat GetDirection() const { - return IsRightToLeft() ? -1.0f : 1.0f; + return IsInlineReversed() ? -1.0f : 1.0f; } bool DisableLigatures() const { From 427bb40573abe44807a53d3d6d75655d7f529515 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 28/78] Bug 1193519 pt 3 - Handle writing-mode:sideways-lr in nsTextFrame selection and rendering. r=dholbert --- layout/generic/nsTextFrame.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/layout/generic/nsTextFrame.cpp b/layout/generic/nsTextFrame.cpp index 79ae536f72d..7e20ffa86b3 100644 --- a/layout/generic/nsTextFrame.cpp +++ b/layout/generic/nsTextFrame.cpp @@ -5792,7 +5792,7 @@ nsTextFrame::PaintTextWithSelectionColors(gfxContext* aCtx, mTextRun->GetAdvanceWidth(offset, length, &aProvider); if (NS_GET_A(background) > 0) { gfxRect bgRect; - gfxFloat offs = iOffset - (mTextRun->IsRightToLeft() ? advance : 0); + gfxFloat offs = iOffset - (mTextRun->IsInlineReversed() ? advance : 0); if (vertical) { bgRect = gfxRect(aFramePt.x, aFramePt.y + offs, GetSize().width, advance); @@ -5829,7 +5829,7 @@ nsTextFrame::PaintTextWithSelectionColors(gfxContext* aCtx, GetSelectionTextShadow(this, type, aTextPaintStyle, &shadow); if (shadow) { nscoord startEdge = iOffset; - if (mTextRun->IsRightToLeft()) { + if (mTextRun->IsInlineReversed()) { startEdge -= mTextRun->GetAdvanceWidth(offset, length, &aProvider) + hyphenWidth; } @@ -6246,7 +6246,7 @@ nsTextFrame::PaintText(nsRenderingContext* aRenderingContext, nsPoint aPt, provider.InitializeForDisplay(!IsSelected()); gfxContext* ctx = aRenderingContext->ThebesContext(); - const bool rtl = mTextRun->IsRightToLeft(); + const bool reversed = mTextRun->IsInlineReversed(); const bool verticalRun = mTextRun->IsVertical(); WritingMode wm = GetWritingMode(); const nscoord frameWidth = GetSize().width; @@ -6261,10 +6261,11 @@ nsTextFrame::PaintText(nsRenderingContext* aRenderingContext, nsPoint aPt, nsLayoutUtils::GetSnappedBaselineX(this, ctx, aPt.x + frameWidth, -mAscent); } - textBaselinePt.y = rtl ? aPt.y + GetSize().height : aPt.y; + textBaselinePt.y = reversed ? aPt.y + GetSize().height : aPt.y; } else { - textBaselinePt = gfxPoint(rtl ? gfxFloat(aPt.x + frameWidth) : framePt.x, - nsLayoutUtils::GetSnappedBaselineY(this, ctx, aPt.y, mAscent)); + textBaselinePt = + gfxPoint(reversed ? gfxFloat(aPt.x + frameWidth) : framePt.x, + nsLayoutUtils::GetSnappedBaselineY(this, ctx, aPt.y, mAscent)); } uint32_t startOffset = provider.GetStart().GetSkippedOffset(); uint32_t maxLength = ComputeTransformedLength(provider); @@ -6274,9 +6275,9 @@ nsTextFrame::PaintText(nsRenderingContext* aRenderingContext, nsPoint aPt, return; } if (verticalRun) { - textBaselinePt.y += rtl ? -snappedEndEdge : snappedStartEdge; + textBaselinePt.y += reversed ? -snappedEndEdge : snappedStartEdge; } else { - textBaselinePt.x += rtl ? -snappedEndEdge : snappedStartEdge; + textBaselinePt.x += reversed ? -snappedEndEdge : snappedStartEdge; } nsCharClipDisplayItem::ClipEdges clipEdges(aItem, snappedStartEdge, snappedEndEdge); @@ -6631,9 +6632,9 @@ nsTextFrame::GetCharacterOffsetAtFramePointInternal(nsPoint aPoint, PropertyProvider provider(this, iter, nsTextFrame::eInflated); // Trim leading but not trailing whitespace if possible provider.InitializeForDisplay(false); - gfxFloat width = mTextRun->IsVertical() ? - (mTextRun->IsRightToLeft() ? mRect.height - aPoint.y : aPoint.y) : - (mTextRun->IsRightToLeft() ? mRect.width - aPoint.x : aPoint.x); + gfxFloat width = mTextRun->IsVertical() + ? (mTextRun->IsInlineReversed() ? mRect.height - aPoint.y : aPoint.y) + : (mTextRun->IsInlineReversed() ? mRect.width - aPoint.x : aPoint.x); gfxFloat fitWidth; uint32_t skippedLength = ComputeTransformedLength(provider); @@ -6860,13 +6861,13 @@ nsTextFrame::GetPointFromOffset(int32_t inOffset, nscoord iSize = NSToCoordCeilClamped(advance); if (mTextRun->IsVertical()) { - if (mTextRun->IsRightToLeft()) { + if (mTextRun->IsInlineReversed()) { outPoint->y = mRect.height - iSize; } else { outPoint->y = iSize; } } else { - if (mTextRun->IsRightToLeft()) { + if (mTextRun->IsInlineReversed()) { outPoint->x = mRect.width - iSize; } else { outPoint->x = iSize; From a244425aaf2d0e64ead82ddda1efd1d16ea5a1d9 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 29/78] Bug 1193519 pt 4 - Reverse the direction of text-decoration offsets in sideways-lr mode. r=dholbert --- layout/base/nsCSSRendering.cpp | 3 +-- layout/generic/nsTextFrame.cpp | 26 ++++++++++++++++++-------- layout/generic/nsTextFrame.h | 1 + 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/layout/base/nsCSSRendering.cpp b/layout/base/nsCSSRendering.cpp index 4bbb186cb7a..8e027dd9713 100644 --- a/layout/base/nsCSSRendering.cpp +++ b/layout/base/nsCSSRendering.cpp @@ -4606,8 +4606,7 @@ nsCSSRendering::GetTextDecorationRectInternal(const gfxPoint& aPt, } if (aVertical) { - r.y = baseline + floor(offset + 0.5); // this will need updating when we - // support sideways-left orientation + r.y = baseline + floor(offset + 0.5); Swap(r.x, r.y); Swap(r.width, r.height); } else { diff --git a/layout/generic/nsTextFrame.cpp b/layout/generic/nsTextFrame.cpp index 7e20ffa86b3..4b6aacee387 100644 --- a/layout/generic/nsTextFrame.cpp +++ b/layout/generic/nsTextFrame.cpp @@ -5311,6 +5311,7 @@ nsTextFrame::DrawSelectionDecorations(gfxContext* aContext, const gfxFont::Metrics& aFontMetrics, DrawPathCallbacks* aCallbacks, bool aVertical, + gfxFloat aDecorationOffsetDir, uint8_t aDecoration) { gfxPoint pt(aPt); @@ -5417,8 +5418,8 @@ nsTextFrame::DrawSelectionDecorations(gfxContext* aContext, size.height *= relativeSize; PaintDecorationLine(aContext, aDirtyRect, color, nullptr, pt, (aVertical ? (pt.y - aPt.y) : (pt.x - aPt.x)) + aICoordInFrame, - size, aAscent, offset, aDecoration, style, eSelectionDecoration, - aCallbacks, aVertical, descentLimit); + size, aAscent, offset * aDecorationOffsetDir, aDecoration, style, + eSelectionDecoration, aCallbacks, aVertical, descentLimit); } /* static */ @@ -5919,6 +5920,7 @@ nsTextFrame::PaintTextSelectionDecorations(gfxContext* aCtx, } gfxRect dirtyRect(aDirtyRect.x / app, aDirtyRect.y / app, aDirtyRect.width / app, aDirtyRect.height / app); + gfxFloat decorationOffsetDir = mTextRun->IsSidewaysLeft() ? -1.0 : 1.0; SelectionType type; TextRangeStyle selectedStyle; while (iterator.GetNextSegment(&iOffset, &offset, &length, &hyphenWidth, @@ -5928,17 +5930,18 @@ nsTextFrame::PaintTextSelectionDecorations(gfxContext* aCtx, if (type == aSelectionType) { if (verticalRun) { pt.y = (aFramePt.y + iOffset - - (mTextRun->IsRightToLeft() ? advance : 0)) / app; + (mTextRun->IsInlineReversed() ? advance : 0)) / app; } else { pt.x = (aFramePt.x + iOffset - - (mTextRun->IsRightToLeft() ? advance : 0)) / app; + (mTextRun->IsInlineReversed() ? advance : 0)) / app; } gfxFloat width = Abs(advance) / app; gfxFloat xInFrame = pt.x - (aFramePt.x / app); DrawSelectionDecorations(aCtx, dirtyRect, aSelectionType, aTextPaintStyle, selectedStyle, pt, xInFrame, width, mAscent / app, decorationMetrics, - aCallbacks, verticalRun, kDecoration); + aCallbacks, verticalRun, decorationOffsetDir, + kDecoration); } iterator.UpdateWithAdvance(advance); } @@ -6433,6 +6436,10 @@ nsTextFrame::DrawTextRunAndDecorations( nscoord inflationMinFontSize = nsLayoutUtils::InflationMinFontSizeFor(this); + // The decoration-line offsets need to be reversed for sideways-lr mode, + // so we will multiply the values from metrics by this factor. + gfxFloat decorationOffsetDir = mTextRun->IsSidewaysLeft() ? -1.0 : 1.0; + // Underlines for (uint32_t i = aDecorations.mUnderlines.Length(); i-- > 0; ) { const LineDecoration& dec = aDecorations.mUnderlines[i]; @@ -6451,7 +6458,8 @@ nsTextFrame::DrawTextRunAndDecorations( PaintDecorationLine(aCtx, dirtyRect, dec.mColor, aDecorationOverrideColor, decPt, 0.0, decSize, ascent, - metrics.underlineOffset, NS_STYLE_TEXT_DECORATION_LINE_UNDERLINE, + decorationOffsetDir * metrics.underlineOffset, + NS_STYLE_TEXT_DECORATION_LINE_UNDERLINE, dec.mStyle, eNormalDecoration, aCallbacks, verticalRun); } // Overlines @@ -6472,7 +6480,8 @@ nsTextFrame::DrawTextRunAndDecorations( PaintDecorationLine(aCtx, dirtyRect, dec.mColor, aDecorationOverrideColor, decPt, 0.0, decSize, ascent, - metrics.maxAscent, NS_STYLE_TEXT_DECORATION_LINE_OVERLINE, dec.mStyle, + decorationOffsetDir * metrics.maxAscent, + NS_STYLE_TEXT_DECORATION_LINE_OVERLINE, dec.mStyle, eNormalDecoration, aCallbacks, verticalRun); } @@ -6499,7 +6508,8 @@ nsTextFrame::DrawTextRunAndDecorations( PaintDecorationLine(aCtx, dirtyRect, dec.mColor, aDecorationOverrideColor, decPt, 0.0, decSize, ascent, - metrics.strikeoutOffset, NS_STYLE_TEXT_DECORATION_LINE_LINE_THROUGH, + decorationOffsetDir * metrics.strikeoutOffset, + NS_STYLE_TEXT_DECORATION_LINE_LINE_THROUGH, dec.mStyle, eNormalDecoration, aCallbacks, verticalRun); } } diff --git a/layout/generic/nsTextFrame.h b/layout/generic/nsTextFrame.h index a13fc4a5312..202e1cdd129 100644 --- a/layout/generic/nsTextFrame.h +++ b/layout/generic/nsTextFrame.h @@ -746,6 +746,7 @@ protected: const gfxFont::Metrics& aFontMetrics, DrawPathCallbacks* aCallbacks, bool aVertical, + gfxFloat aDecorationOffsetDir, uint8_t aDecoration); enum DecorationType { From ce8b369744a987bc34912f57f9ecbb1f15e84c8c Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 30/78] Bug 1193519 pt 5 - Expose the sideways-lr value for writing-mode to CSS. r=dholbert --- layout/base/nsLayoutUtils.cpp | 3 +-- layout/style/nsCSSKeywordList.h | 2 +- layout/style/nsCSSProps.cpp | 2 +- layout/style/test/property_database.js | 4 ++-- layout/style/test/test_logical_properties.html | 18 ++---------------- 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/layout/base/nsLayoutUtils.cpp b/layout/base/nsLayoutUtils.cpp index c9a320f2e16..8b517c8a369 100644 --- a/layout/base/nsLayoutUtils.cpp +++ b/layout/base/nsLayoutUtils.cpp @@ -6663,10 +6663,9 @@ nsLayoutUtils::GetTextRunOrientFlagsForStyle(nsStyleContext* aStyleContext) return 0; } - /* not yet implemented: case NS_STYLE_WRITING_MODE_SIDEWAYS_LR: return gfxTextRunFactory::TEXT_ORIENT_VERTICAL_SIDEWAYS_LEFT; - */ + case NS_STYLE_WRITING_MODE_SIDEWAYS_RL: return gfxTextRunFactory::TEXT_ORIENT_VERTICAL_SIDEWAYS_RIGHT; diff --git a/layout/style/nsCSSKeywordList.h b/layout/style/nsCSSKeywordList.h index 72ec941cffc..a29ffeae918 100644 --- a/layout/style/nsCSSKeywordList.h +++ b/layout/style/nsCSSKeywordList.h @@ -485,7 +485,7 @@ CSS_KEY(sepia, sepia) CSS_KEY(serif, serif) CSS_KEY(show, show) CSS_KEY(sideways, sideways) -/*CSS_KEY(sideways-lr, sideways_lr)*/ +CSS_KEY(sideways-lr, sideways_lr) CSS_KEY(sideways-right, sideways_right) /* alias for 'sideways' */ CSS_KEY(sideways-rl, sideways_rl) CSS_KEY(simp-chinese-formal, simp_chinese_formal) diff --git a/layout/style/nsCSSProps.cpp b/layout/style/nsCSSProps.cpp index 065c1b63488..fd3a0d99db7 100644 --- a/layout/style/nsCSSProps.cpp +++ b/layout/style/nsCSSProps.cpp @@ -1944,7 +1944,7 @@ const KTableValue nsCSSProps::kWritingModeKTable[] = { eCSSKeyword_horizontal_tb, NS_STYLE_WRITING_MODE_HORIZONTAL_TB, eCSSKeyword_vertical_lr, NS_STYLE_WRITING_MODE_VERTICAL_LR, eCSSKeyword_vertical_rl, NS_STYLE_WRITING_MODE_VERTICAL_RL, -/* eCSSKeyword_sideways_lr, NS_STYLE_WRITING_MODE_SIDEWAYS_LR, */ + eCSSKeyword_sideways_lr, NS_STYLE_WRITING_MODE_SIDEWAYS_LR, eCSSKeyword_sideways_rl, NS_STYLE_WRITING_MODE_SIDEWAYS_RL, eCSSKeyword_lr, NS_STYLE_WRITING_MODE_HORIZONTAL_TB, eCSSKeyword_lr_tb, NS_STYLE_WRITING_MODE_HORIZONTAL_TB, diff --git a/layout/style/test/property_database.js b/layout/style/test/property_database.js index 2cb240503a9..bf49b183295 100644 --- a/layout/style/test/property_database.js +++ b/layout/style/test/property_database.js @@ -4731,8 +4731,8 @@ if (SpecialPowers.getBoolPref("layout.css.vertical-text.enabled")) { inherited: true, type: CSS_TYPE_LONGHAND, initial_values: [ "horizontal-tb", "lr", "lr-tb", "rl", "rl-tb" ], - other_values: [ "vertical-lr", "vertical-rl", "sideways-rl", "tb", "tb-rl" ], - invalid_values: [ "10px", "30%", "justify", "auto", "1em", "sideways-lr" ] /* sideways-lr not yet supported */ + other_values: [ "vertical-lr", "vertical-rl", "sideways-rl", "sideways-lr", "tb", "tb-rl" ], + invalid_values: [ "10px", "30%", "justify", "auto", "1em" ] }, "text-orientation": { domProp: "textOrientation", diff --git a/layout/style/test/test_logical_properties.html b/layout/style/test/test_logical_properties.html index 8d19e7dfa7b..66e596887e2 100644 --- a/layout/style/test/test_logical_properties.html +++ b/layout/style/test/test_logical_properties.html @@ -72,13 +72,13 @@ var gWritingModes = [ block: "horizontal", inline: "vertical" }, { style: [ "writing-mode: vertical-lr; direction: rtl; ", - // "writing-mode: sideways-lr; direction: ltr; ", + "writing-mode: sideways-lr; direction: ltr; ", ], blockStart: "left", blockEnd: "right", inlineStart: "bottom", inlineEnd: "top", block: "horizontal", inline: "vertical" }, { style: [ "writing-mode: vertical-lr; direction: ltr; ", - // "writing-mode: sideways-lr; direction: rtl; ", + "writing-mode: sideways-lr; direction: rtl; ", ], blockStart: "left", blockEnd: "right", inlineStart: "top", inlineEnd: "bottom", block: "horizontal", inline: "vertical" }, @@ -143,20 +143,6 @@ function init() { make_declaration(gCSSProperties[`${aPrefix}height`].prerequisites) }); }); - - // Assume that the sideways-lr keyword is not parsed yet for writing-mode. - // When we start supporting this keyword, the entries in the .style - // properties of the gWritingModes objects above should be uncommented. - var s = document.createElement("style"); - document.body.appendChild(s); - - s.textContent = "div { }"; - s.sheet.cssRules[0].style.writingMode = "sideways-lr"; - todo(s.sheet.cssRules[0].style.writingMode, "sideways-lr", - "uncomment sideways-lr cases from gWritingModes and " + - "remove this todo()!"); - - s.remove(); } function test_computed_values(aTestName, aRules, aExpectedValues) { From 8df83a20972d61dcc8fdfe09ac28a346d08e752a Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 31/78] Bug 1193519 pt 6 - Adjust the position of the caret bidi indicator appropriately for sideways-lr mode. r=dholbert --- layout/base/nsCaret.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/layout/base/nsCaret.cpp b/layout/base/nsCaret.cpp index 2ed4cf9afa1..0adbedcfde3 100644 --- a/layout/base/nsCaret.cpp +++ b/layout/base/nsCaret.cpp @@ -925,7 +925,8 @@ nsCaret::ComputeCaretRects(nsIFrame* aFrame, int32_t aFrameOffset, { NS_ASSERTION(aFrame, "Should have a frame here"); - bool isVertical = aFrame->GetWritingMode().IsVertical(); + WritingMode wm = aFrame->GetWritingMode(); + bool isVertical = wm.IsVertical(); nscoord bidiIndicatorSize; *aCaretRect = GetGeometryForFrame(aFrame, aFrameOffset, &bidiIndicatorSize); @@ -957,11 +958,20 @@ nsCaret::ComputeCaretRects(nsIFrame* aFrame, int32_t aFrameOffset, } bool isCaretRTL = caretBidiLevel % 2; if (isVertical) { - aHookRect->SetRect(aCaretRect->XMost() - bidiIndicatorSize, - aCaretRect->y + (isCaretRTL ? bidiIndicatorSize * -1 : - aCaretRect->height), - aCaretRect->height, - bidiIndicatorSize); + bool isSidewaysLR = wm.IsVerticalLR() && !wm.IsLineInverted(); + if (isSidewaysLR) { + aHookRect->SetRect(aCaretRect->x + bidiIndicatorSize, + aCaretRect->y + (!isCaretRTL ? bidiIndicatorSize * -1 : + aCaretRect->height), + aCaretRect->height, + bidiIndicatorSize); + } else { + aHookRect->SetRect(aCaretRect->XMost() - bidiIndicatorSize, + aCaretRect->y + (isCaretRTL ? bidiIndicatorSize * -1 : + aCaretRect->height), + aCaretRect->height, + bidiIndicatorSize); + } } else { aHookRect->SetRect(aCaretRect->x + (isCaretRTL ? bidiIndicatorSize * -1 : aCaretRect->width), From c3f2e439aa83420e633aca47d6326aeee019a74d Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 32/78] Bug 1193519 pt 7 - Basic reftests for sideways-lr writing mode. r=dholbert --- .../1193519-sideways-lr-1-ref.html | 24 ++++++++++++++++ .../writing-mode/1193519-sideways-lr-1.html | 28 +++++++++++++++++++ .../1193519-sideways-lr-2-ref.html | 24 ++++++++++++++++ .../writing-mode/1193519-sideways-lr-2.html | 28 +++++++++++++++++++ .../1193519-sideways-lr-3-ref.html | 21 ++++++++++++++ .../writing-mode/1193519-sideways-lr-3.html | 21 ++++++++++++++ .../1193519-sideways-lr-4-ref.html | 27 ++++++++++++++++++ .../writing-mode/1193519-sideways-lr-4.html | 27 ++++++++++++++++++ layout/reftests/writing-mode/reftest.list | 7 +++++ 9 files changed, 207 insertions(+) create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-1-ref.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-1.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-2-ref.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-2.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-3-ref.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-3.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-4-ref.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-4.html diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-1-ref.html b/layout/reftests/writing-mode/1193519-sideways-lr-1-ref.html new file mode 100644 index 00000000000..e3c96714c7b --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-1-ref.html @@ -0,0 +1,24 @@ + + + + + + + + + +

All four columns should look the same: +

+
+
+
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-1.html b/layout/reftests/writing-mode/1193519-sideways-lr-1.html new file mode 100644 index 00000000000..66df9d85f71 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-1.html @@ -0,0 +1,28 @@ + + + + + + + + + +

All four columns should look the same: +

+
+
+
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-2-ref.html b/layout/reftests/writing-mode/1193519-sideways-lr-2-ref.html new file mode 100644 index 00000000000..681cb2890a7 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-2-ref.html @@ -0,0 +1,24 @@ + + + + + + + + + +

All four columns should look the same: +

+
+
+
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-2.html b/layout/reftests/writing-mode/1193519-sideways-lr-2.html new file mode 100644 index 00000000000..5a6e7ea0eab --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-2.html @@ -0,0 +1,28 @@ + + + + + + + + + +

All four columns should look the same: +

+
+
+
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-3-ref.html b/layout/reftests/writing-mode/1193519-sideways-lr-3-ref.html new file mode 100644 index 00000000000..e9ac42e259c --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-3-ref.html @@ -0,0 +1,21 @@ + + + + + + + + + +
one two three four + five six seven
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-3.html b/layout/reftests/writing-mode/1193519-sideways-lr-3.html new file mode 100644 index 00000000000..09287844eb6 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-3.html @@ -0,0 +1,21 @@ + + + + + + + + + +
one two three four + five six seven
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-4-ref.html b/layout/reftests/writing-mode/1193519-sideways-lr-4-ref.html new file mode 100644 index 00000000000..368d38834f3 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-4-ref.html @@ -0,0 +1,27 @@ + + + + + + + + + +
"One two + three four five + six seven."
+ +
"One two + three four five + six seven."
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-4.html b/layout/reftests/writing-mode/1193519-sideways-lr-4.html new file mode 100644 index 00000000000..f01956e3f69 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-4.html @@ -0,0 +1,27 @@ + + + + + + + + + +
"One two + three four five + six seven."
+ +
"One two + three four five + six seven."
diff --git a/layout/reftests/writing-mode/reftest.list b/layout/reftests/writing-mode/reftest.list index 0a89e1ea117..a138f8b752d 100644 --- a/layout/reftests/writing-mode/reftest.list +++ b/layout/reftests/writing-mode/reftest.list @@ -152,6 +152,13 @@ test-pref(dom.meta-viewport.enabled,true) test-pref(font.size.inflation.emPerLin == 1175789-underline-overline-1.html 1175789-underline-overline-1-ref.html == 1188061-1-nsChangeHint_ClearAncestorIntrinsics.html 1188061-1-nsChangeHint_ClearAncestorIntrinsics-ref.html == 1188061-2-nsChangeHint_UpdateComputedBSize.html 1188061-2-nsChangeHint_UpdateComputedBSize-ref.html + +# tests involving sideways-lr mode +== 1193519-sideways-lr-1.html 1193519-sideways-lr-1-ref.html +== 1193519-sideways-lr-2.html 1193519-sideways-lr-2-ref.html +== 1193519-sideways-lr-3.html 1193519-sideways-lr-3-ref.html +== 1193519-sideways-lr-4.html 1193519-sideways-lr-4-ref.html + == 1196887-1-computed-display-inline-block.html 1196887-1-computed-display-inline-block-ref.html == 1205787-legacy-svg-values-1.html 1205787-legacy-svg-values-1-ref.html From 74ccaf01f3733a296351c986c308763367e25ac6 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:34 +0100 Subject: [PATCH 33/78] Bug 1193519 pt 8 - Reftests for table row and cell ordering in sideways-* writing modes. r=dholbert --- .../reftests/writing-mode/tables/reftest.list | 4 ++ .../sideways-lr-row-progression-1-ref.html | 51 ++++++++++++++ .../sideways-lr-row-progression-1a.html | 68 ++++++++++++++++++ .../sideways-lr-row-progression-1b.html | 69 +++++++++++++++++++ .../sideways-rl-row-progression-1-ref.html | 54 +++++++++++++++ .../sideways-rl-row-progression-1a.html | 68 ++++++++++++++++++ .../sideways-rl-row-progression-1b.html | 68 ++++++++++++++++++ 7 files changed, 382 insertions(+) create mode 100644 layout/reftests/writing-mode/tables/sideways-lr-row-progression-1-ref.html create mode 100644 layout/reftests/writing-mode/tables/sideways-lr-row-progression-1a.html create mode 100644 layout/reftests/writing-mode/tables/sideways-lr-row-progression-1b.html create mode 100644 layout/reftests/writing-mode/tables/sideways-rl-row-progression-1-ref.html create mode 100644 layout/reftests/writing-mode/tables/sideways-rl-row-progression-1a.html create mode 100644 layout/reftests/writing-mode/tables/sideways-rl-row-progression-1b.html diff --git a/layout/reftests/writing-mode/tables/reftest.list b/layout/reftests/writing-mode/tables/reftest.list index ba576e7bdf0..e1a1b2e3cc0 100644 --- a/layout/reftests/writing-mode/tables/reftest.list +++ b/layout/reftests/writing-mode/tables/reftest.list @@ -87,3 +87,7 @@ fuzzy-if(cocoaWidget,23,162) == border-collapse-bevels-1e.html border-collapse-b == vertical-rl-row-progression-1a.html vertical-rl-row-progression-1-ref.html == vertical-rl-row-progression-1b.html vertical-rl-row-progression-1-ref.html +== sideways-lr-row-progression-1a.html sideways-lr-row-progression-1-ref.html +== sideways-lr-row-progression-1b.html sideways-lr-row-progression-1-ref.html +== sideways-rl-row-progression-1a.html sideways-rl-row-progression-1-ref.html +== sideways-rl-row-progression-1b.html sideways-rl-row-progression-1-ref.html diff --git a/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1-ref.html b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1-ref.html new file mode 100644 index 00000000000..930ce16dd9c --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1-ref.html @@ -0,0 +1,51 @@ + +CSS Reference: sideways-lr Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + + +
+ + + + +
+ +
+ + + +
+ + +
diff --git a/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1a.html b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1a.html new file mode 100644 index 00000000000..68d15a91c96 --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1a.html @@ -0,0 +1,68 @@ + +CSS Test: sideways-lr Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + +
+ + +
+ +
+
+ +
diff --git a/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1b.html b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1b.html new file mode 100644 index 00000000000..70172efb945 --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-lr-row-progression-1b.html @@ -0,0 +1,69 @@ + +CSS Test: sideways-lr Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + +
+ + +
+ +
+
+ +
diff --git a/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1-ref.html b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1-ref.html new file mode 100644 index 00000000000..64f29765e78 --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1-ref.html @@ -0,0 +1,54 @@ + +CSS Reference: sideways-rl Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + +
+ + + +
+ + + + +
+ + +
+ +
diff --git a/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1a.html b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1a.html new file mode 100644 index 00000000000..c2096757f7e --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1a.html @@ -0,0 +1,68 @@ + +CSS Test: sideways-rl Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + +
+ + +
+ +
+
+ +
diff --git a/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1b.html b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1b.html new file mode 100644 index 00000000000..5ef68755111 --- /dev/null +++ b/layout/reftests/writing-mode/tables/sideways-rl-row-progression-1b.html @@ -0,0 +1,68 @@ + +CSS Test: sideways-rl Table Row/Rowgroup/Cell Ordering + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + +
+ + +
+ +
+
+ +
From cecdafc9094b660efce4deb7769cea4b2ad74151 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:35 +0100 Subject: [PATCH 34/78] Bug 1193519 pt 9 - Reftests with floats in writing-mode: sideways-lr. r=dholbert --- .../floats/float-in-rtl-slr-1-ref.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-1a.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-1b.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-1c.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-1d.html | 23 ++++++++++++++++ .../floats/float-in-rtl-slr-2-ref.html | 23 ++++++++++++++++ .../reftests/floats/float-in-rtl-slr-2a.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-2b.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-2c.html | 21 +++++++++++++++ .../reftests/floats/float-in-rtl-slr-2d.html | 23 ++++++++++++++++ .../floats/float-in-rtl-slr-3-ref.html | 23 ++++++++++++++++ .../reftests/floats/float-in-rtl-slr-3a.html | 23 ++++++++++++++++ .../reftests/floats/float-in-rtl-slr-3b.html | 23 ++++++++++++++++ .../reftests/floats/float-in-rtl-slr-3c.html | 23 ++++++++++++++++ .../reftests/floats/float-in-rtl-slr-3d.html | 25 ++++++++++++++++++ .../floats/float-in-rtl-slr-4-ref.html | 24 +++++++++++++++++ .../reftests/floats/float-in-rtl-slr-4a.html | 24 +++++++++++++++++ .../reftests/floats/float-in-rtl-slr-4b.html | 24 +++++++++++++++++ .../reftests/floats/float-in-rtl-slr-4c.html | 24 +++++++++++++++++ .../reftests/floats/float-in-rtl-slr-4d.html | 26 +++++++++++++++++++ layout/reftests/floats/reftest.list | 17 ++++++++++++ 21 files changed, 472 insertions(+) create mode 100644 layout/reftests/floats/float-in-rtl-slr-1-ref.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-1a.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-1b.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-1c.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-1d.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-2-ref.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-2a.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-2b.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-2c.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-2d.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-3-ref.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-3a.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-3b.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-3c.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-3d.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-4-ref.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-4a.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-4b.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-4c.html create mode 100644 layout/reftests/floats/float-in-rtl-slr-4d.html diff --git a/layout/reftests/floats/float-in-rtl-slr-1-ref.html b/layout/reftests/floats/float-in-rtl-slr-1-ref.html new file mode 100644 index 00000000000..f5af2bc1968 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-1-ref.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+
This text should appear BELOW the red and green blocks.
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-1a.html b/layout/reftests/floats/float-in-rtl-slr-1a.html new file mode 100644 index 00000000000..04e05bf2c4b --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-1a.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-1b.html b/layout/reftests/floats/float-in-rtl-slr-1b.html new file mode 100644 index 00000000000..0d674491980 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-1b.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-1c.html b/layout/reftests/floats/float-in-rtl-slr-1c.html new file mode 100644 index 00000000000..41e3be793d9 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-1c.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-1d.html b/layout/reftests/floats/float-in-rtl-slr-1d.html new file mode 100644 index 00000000000..8813c2a63ec --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-1d.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-2-ref.html b/layout/reftests/floats/float-in-rtl-slr-2-ref.html new file mode 100644 index 00000000000..b3cd61f6c71 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-2-ref.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+
This text should appear ABOVE the green and red blocks.
+
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-2a.html b/layout/reftests/floats/float-in-rtl-slr-2a.html new file mode 100644 index 00000000000..31c4fe178fa --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-2a.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-2b.html b/layout/reftests/floats/float-in-rtl-slr-2b.html new file mode 100644 index 00000000000..dd25587665e --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-2b.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-2c.html b/layout/reftests/floats/float-in-rtl-slr-2c.html new file mode 100644 index 00000000000..4ee217ce1df --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-2c.html @@ -0,0 +1,21 @@ + + + + + + +
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-2d.html b/layout/reftests/floats/float-in-rtl-slr-2d.html new file mode 100644 index 00000000000..b3b9623eec7 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-2d.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-3-ref.html b/layout/reftests/floats/float-in-rtl-slr-3-ref.html new file mode 100644 index 00000000000..4c99aa543cd --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-3-ref.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+
This text should appear ABOVE the green and red blocks.
+
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-3a.html b/layout/reftests/floats/float-in-rtl-slr-3a.html new file mode 100644 index 00000000000..641a6aa79ad --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-3a.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-3b.html b/layout/reftests/floats/float-in-rtl-slr-3b.html new file mode 100644 index 00000000000..613ffac32e2 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-3b.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-3c.html b/layout/reftests/floats/float-in-rtl-slr-3c.html new file mode 100644 index 00000000000..5c4669bb413 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-3c.html @@ -0,0 +1,23 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear ABOVE the green and red blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-3d.html b/layout/reftests/floats/float-in-rtl-slr-3d.html new file mode 100644 index 00000000000..ef7b1886fd2 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-3d.html @@ -0,0 +1,25 @@ + + + + + + +
+
+
+
+
+
+
+
+
+
+ This text should appear ABOVE the green and red blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-4-ref.html b/layout/reftests/floats/float-in-rtl-slr-4-ref.html new file mode 100644 index 00000000000..1696cdd4d89 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-4-ref.html @@ -0,0 +1,24 @@ + + + + + + +
+
+
+
+
+
+
+
+
This text should appear BELOW the red and green blocks.
+
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-4a.html b/layout/reftests/floats/float-in-rtl-slr-4a.html new file mode 100644 index 00000000000..f94ed509809 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-4a.html @@ -0,0 +1,24 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-4b.html b/layout/reftests/floats/float-in-rtl-slr-4b.html new file mode 100644 index 00000000000..7520ea6ddbf --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-4b.html @@ -0,0 +1,24 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-4c.html b/layout/reftests/floats/float-in-rtl-slr-4c.html new file mode 100644 index 00000000000..000938f91c6 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-4c.html @@ -0,0 +1,24 @@ + + + + + + +
+
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+
+ + diff --git a/layout/reftests/floats/float-in-rtl-slr-4d.html b/layout/reftests/floats/float-in-rtl-slr-4d.html new file mode 100644 index 00000000000..6fb5e8509b1 --- /dev/null +++ b/layout/reftests/floats/float-in-rtl-slr-4d.html @@ -0,0 +1,26 @@ + + + + + + +
+
+
+
+
+
+
+
+
+
+This text should appear BELOW the red and green blocks. +
+
+ + diff --git a/layout/reftests/floats/reftest.list b/layout/reftests/floats/reftest.list index 4058fa23878..2a8817fc00b 100644 --- a/layout/reftests/floats/reftest.list +++ b/layout/reftests/floats/reftest.list @@ -88,3 +88,20 @@ fuzzy-if(OSX==1010,26,7) fuzzy-if(Android,16,2) == orthogonal-floats-1a.html ort fuzzy-if(OSX==1010,26,7) == orthogonal-floats-1b.html orthogonal-floats-1-ref.html fuzzy-if(OSX==1010,103,802) fuzzy-if(winWidget,116,700) HTTP(..) == orthogonal-floats-1c.html orthogonal-floats-1-ref.html fuzzy-if(OSX==1010,103,802) fuzzy-if(winWidget,116,700) HTTP(..) == orthogonal-floats-1d.html orthogonal-floats-1-ref.html + +== float-in-rtl-slr-1a.html float-in-rtl-slr-1-ref.html +== float-in-rtl-slr-1b.html float-in-rtl-slr-1-ref.html +== float-in-rtl-slr-1c.html float-in-rtl-slr-1-ref.html +== float-in-rtl-slr-1d.html float-in-rtl-slr-1-ref.html +== float-in-rtl-slr-2a.html float-in-rtl-slr-2-ref.html +== float-in-rtl-slr-2b.html float-in-rtl-slr-2-ref.html +== float-in-rtl-slr-2c.html float-in-rtl-slr-2-ref.html +== float-in-rtl-slr-2d.html float-in-rtl-slr-2-ref.html +== float-in-rtl-slr-3a.html float-in-rtl-slr-3-ref.html +== float-in-rtl-slr-3b.html float-in-rtl-slr-3-ref.html +== float-in-rtl-slr-3c.html float-in-rtl-slr-3-ref.html +== float-in-rtl-slr-3d.html float-in-rtl-slr-3-ref.html +== float-in-rtl-slr-4a.html float-in-rtl-slr-4-ref.html +== float-in-rtl-slr-4b.html float-in-rtl-slr-4-ref.html +== float-in-rtl-slr-4c.html float-in-rtl-slr-4-ref.html +== float-in-rtl-slr-4d.html float-in-rtl-slr-4-ref.html From 71b9c62e4e8a38e492feab951f5929d5061d14cb Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:35 +0100 Subject: [PATCH 35/78] Bug 1193519 pt 10 - Clean up remaining mentions of 'sideways-left' in code comments. r=dholbert --- layout/generic/WritingModes.h | 40 +++++++++++++++++---------------- layout/generic/nsImageFrame.cpp | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/layout/generic/WritingModes.h b/layout/generic/WritingModes.h index fa78609a485..62e83214975 100644 --- a/layout/generic/WritingModes.h +++ b/layout/generic/WritingModes.h @@ -234,9 +234,7 @@ public: /** * True if line-over/line-under are inverted from block-start/block-end. - * This is true when - * - writing-mode is vertical-rl && text-orientation is sideways-left - * - writing-mode is vertical-lr && text-orientation is not sideways-left + * This is true only when writing-mode is vertical-lr. */ bool IsLineInverted() const { return !!(mWritingMode & eLineOrientMask); } @@ -323,23 +321,27 @@ public: // bit 1 = the eInlineFlowMask value // bit 2 = the eBlockFlowMask value // bit 3 = the eLineOrientMask value + // Not all of these combinations can actually be specified via CSS: there + // is no horizontal-bt writing-mode, and no text-orientation value that + // produces "inverted" text. (The former 'sideways-left' value, no longer + // in the spec, would have produced this in vertical-rl mode.) static const mozilla::css::Side kLogicalInlineSides[][2] = { - { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // horizontal-tb ltr - { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-rl ltr - { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // horizontal-tb rtl - { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-rl rtl - { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // (horizontal-bt) (inverted) ltr - { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-lr sideways-left rtl - { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // (horizontal-bt) (inverted) rtl - { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-lr sideways-left ltr - { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // horizontal-tb (inverted) rtl - { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-rl sideways-left rtl - { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // horizontal-tb (inverted) ltr - { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-rl sideways-left ltr - { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // (horizontal-bt) ltr - { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-lr ltr - { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // (horizontal-bt) rtl - { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-lr rtl + { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // horizontal-tb ltr + { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-rl ltr + { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // horizontal-tb rtl + { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-rl rtl + { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // (horizontal-bt) (inverted) ltr + { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // sideways-lr rtl + { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // (horizontal-bt) (inverted) rtl + { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // sideways-lr ltr + { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // horizontal-tb (inverted) rtl + { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-rl (inverted) rtl + { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // horizontal-tb (inverted) ltr + { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-rl (inverted) ltr + { NS_SIDE_LEFT, NS_SIDE_RIGHT }, // (horizontal-bt) ltr + { NS_SIDE_TOP, NS_SIDE_BOTTOM }, // vertical-lr ltr + { NS_SIDE_RIGHT, NS_SIDE_LEFT }, // (horizontal-bt) rtl + { NS_SIDE_BOTTOM, NS_SIDE_TOP }, // vertical-lr rtl }; // Inline axis sides depend on all three of writing-mode, text-orientation diff --git a/layout/generic/nsImageFrame.cpp b/layout/generic/nsImageFrame.cpp index 9f848db2187..216029059c4 100644 --- a/layout/generic/nsImageFrame.cpp +++ b/layout/generic/nsImageFrame.cpp @@ -1187,7 +1187,7 @@ nsImageFrame::DisplayAltText(nsPresContext* aPresContext, nscoord x, y; if (isVertical) { - x = pt.x + maxDescent; // XXX will need update for sideways-left + x = pt.x + maxDescent; if (wm.IsBidiLTR()) { y = aRect.y; dir = NSBIDI_LTR; From 4890f364b1a6ae8e30a22d7e8d340c5956f7d1dd Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 10:23:35 +0100 Subject: [PATCH 36/78] Bug 1193519 pt 11 - Reftest for sideways-lr writing mode with text-decoration. r=dholbert --- .../1193519-sideways-lr-decoration-1-ref.html | 10 ++++++++++ .../writing-mode/1193519-sideways-lr-decoration-1.html | 10 ++++++++++ layout/reftests/writing-mode/reftest.list | 1 + 3 files changed, 21 insertions(+) create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-decoration-1-ref.html create mode 100644 layout/reftests/writing-mode/1193519-sideways-lr-decoration-1.html diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1-ref.html b/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1-ref.html new file mode 100644 index 00000000000..24cac4884be --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1-ref.html @@ -0,0 +1,10 @@ + +
+ under + over + through +
diff --git a/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1.html b/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1.html new file mode 100644 index 00000000000..a9cc3dd73b5 --- /dev/null +++ b/layout/reftests/writing-mode/1193519-sideways-lr-decoration-1.html @@ -0,0 +1,10 @@ + +
+ under + over + through +
diff --git a/layout/reftests/writing-mode/reftest.list b/layout/reftests/writing-mode/reftest.list index a138f8b752d..fdd0a45d081 100644 --- a/layout/reftests/writing-mode/reftest.list +++ b/layout/reftests/writing-mode/reftest.list @@ -158,6 +158,7 @@ test-pref(dom.meta-viewport.enabled,true) test-pref(font.size.inflation.emPerLin == 1193519-sideways-lr-2.html 1193519-sideways-lr-2-ref.html == 1193519-sideways-lr-3.html 1193519-sideways-lr-3-ref.html == 1193519-sideways-lr-4.html 1193519-sideways-lr-4-ref.html +fuzzy-if(gtkWidget||B2G,255,6) == 1193519-sideways-lr-decoration-1.html 1193519-sideways-lr-decoration-1-ref.html == 1196887-1-computed-display-inline-block.html 1196887-1-computed-display-inline-block-ref.html == 1205787-legacy-svg-values-1.html 1205787-legacy-svg-values-1-ref.html From 478b2883eac8d868fb51e927e6e6a38ab829f10f Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Thu, 24 Sep 2015 11:44:25 +0200 Subject: [PATCH 37/78] Backed out 1 changesets (bug 1207915) for windows m3 test failures Backed out changeset 84fd96dc7ddc (bug 1207915) --- dom/media/mediasink/DecodedStream.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp index 0b0405d79da..11ad564f844 100644 --- a/dom/media/mediasink/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -605,9 +605,6 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, MediaData* aData, AudioSegment* aOutput, uint32_t aRate, double aVolume) { - // The amount of audio frames that is used to fuzz rounding errors. - static const int64_t AUDIO_FUZZ_FRAMES = 1; - MOZ_ASSERT(aData); AudioData* audio = aData->As(); // This logic has to mimic AudioSink closely to make sure we write @@ -619,11 +616,11 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, if (!audioWrittenOffset.isValid() || !frameOffset.isValid() || // ignore packet that we've already processed - frameOffset.value() < audioWrittenOffset.value()) { + frameOffset.value() + audio->mFrames <= audioWrittenOffset.value()) { return; } - if (audioWrittenOffset.value() + AUDIO_FUZZ_FRAMES < frameOffset.value()) { + if (audioWrittenOffset.value() < frameOffset.value()) { int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value(); // Write silence to catch up AudioSegment silence; @@ -633,17 +630,20 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, aOutput->AppendFrom(&silence); } - // Always write the whole sample without truncation to be consistent with - // DecodedAudioDataSink::PlayFromAudioQueue() + MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value()); + + int64_t offset = audioWrittenOffset.value() - frameOffset.value(); + size_t framesToWrite = audio->mFrames - offset; + audio->EnsureAudioBuffer(); nsRefPtr buffer = audio->mAudioBuffer; AudioDataValue* bufferData = static_cast(buffer->Data()); nsAutoTArray channels; for (uint32_t i = 0; i < audio->mChannels; ++i) { - channels.AppendElement(bufferData + i * audio->mFrames); + channels.AppendElement(bufferData + i * audio->mFrames + offset); } - aOutput->AppendFrames(buffer.forget(), channels, audio->mFrames); - aStream->mAudioFramesWritten += audio->mFrames; + aOutput->AppendFrames(buffer.forget(), channels, framesToWrite); + aStream->mAudioFramesWritten += framesToWrite; aOutput->ApplyVolume(aVolume); aStream->mNextAudioTime = audio->GetEndTime(); From 2522db05ac4c4cdf1b6fedccf710eb7428f62c00 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Mon, 21 Sep 2015 11:56:52 +0200 Subject: [PATCH 38/78] Bug 1144534 - If we have tiling, don't reduce layer resolution for large transforms. r=mattwoodrow --- layout/base/FrameLayerBuilder.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/layout/base/FrameLayerBuilder.cpp b/layout/base/FrameLayerBuilder.cpp index af833bf8474..2f5eff339cf 100644 --- a/layout/base/FrameLayerBuilder.cpp +++ b/layout/base/FrameLayerBuilder.cpp @@ -5118,9 +5118,11 @@ ChooseScaleAndSetTransform(FrameLayerBuilder* aLayerBuilder, scale = gfxSize(1.0, 1.0); } // If this is a transform container layer, then pre-rendering might - // mean we try render a layer bigger than the max texture size. Apply - // clmaping to prevent this. - if (aTransform) { + // mean we try render a layer bigger than the max texture size. If we have + // tiling, that's not a problem, since we'll automatically choose a tiled + // layer for layers of that size. If not, we need to apply clamping to + // prevent this. + if (aTransform && !gfxPrefs::LayersTilesEnabled()) { RestrictScaleToMaxLayerSize(scale, aVisibleRect, aContainerFrame, aLayer); } } else { From 5587a1d1fdaf6b9b2ec7b96f9bf3c207acf4a4bd Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 23 Sep 2015 16:18:29 +0200 Subject: [PATCH 39/78] Bug 1021845 - Before compositing, clip the visible region of a layer to the layer's clip rect. r=mattwoodrow --- .../composite/LayerManagerComposite.cpp | 59 +++++++++++++++---- gfx/layers/composite/LayerManagerComposite.h | 4 +- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/gfx/layers/composite/LayerManagerComposite.cpp b/gfx/layers/composite/LayerManagerComposite.cpp index 34032824b0b..5d0be4cb7ea 100644 --- a/gfx/layers/composite/LayerManagerComposite.cpp +++ b/gfx/layers/composite/LayerManagerComposite.cpp @@ -204,16 +204,31 @@ LayerManagerComposite::BeginTransactionWithDrawTarget(DrawTarget* aTarget, const mTargetBounds = aRect; } +template +Maybe +IntersectMaybeRects(const Maybe& aRect1, const Maybe& aRect2) +{ + if (aRect1) { + if (aRect2) { + return Some(aRect1->Intersect(*aRect2)); + } + return aRect1; + } + return aRect2; +} + void -LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion) +LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion, + const Maybe& aClipFromAncestors) { nsIntRegion localOpaque; + Matrix4x4 transform = aLayer->GetLocalTransform(); Matrix transform2d; bool isTranslation = false; // If aLayer has a simple transform (only an integer translation) then we // can easily convert aOpaqueRegion into pre-transform coordinates and include // that region. - if (aLayer->GetLocalTransform().Is2D(&transform2d)) { + if (transform.Is2D(&transform2d)) { if (transform2d.IsIntegerTranslation()) { isTranslation = true; localOpaque = aOpaqueRegion; @@ -221,19 +236,38 @@ LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaque } } + LayerComposite* composite = aLayer->AsLayerComposite(); + nsIntRegion visible = composite->GetShadowVisibleRegion(); + + // Combine our clip with the one from our ancestors. Do this even if the + // transform is not just a translation. + const Maybe& layerClip = aLayer->GetEffectiveClipRect(); + Maybe outsideClip = IntersectMaybeRects(layerClip, aClipFromAncestors); + LayerRect insideClip(LayerIntRect::FromUnknownRect(visible.GetBounds())); + if (outsideClip) { + Matrix4x4 inverse = transform; + if (inverse.Invert()) { + insideClip = UntransformTo(inverse, + ParentLayerRect(*outsideClip), + insideClip).valueOr(insideClip); + } + } + LayerIntRect combinedClip = RoundedOut(insideClip); + // Subtract any areas that we know to be opaque from our // visible region. - LayerComposite *composite = aLayer->AsLayerComposite(); if (!localOpaque.IsEmpty()) { - nsIntRegion visible = composite->GetShadowVisibleRegion(); - visible.Sub(visible, localOpaque); - composite->SetShadowVisibleRegion(visible); + visible.SubOut(localOpaque); } + visible.AndWith(LayerIntRect::ToUntyped(combinedClip)); + composite->SetShadowVisibleRegion(visible); // Compute occlusions for our descendants (in front-to-back order) and allow them to // contribute to localOpaque. for (Layer* child = aLayer->GetLastChild(); child; child = child->GetPrevSibling()) { - ApplyOcclusionCulling(child, localOpaque); + Maybe ancestorClipForChild = + Some(ViewAs(combinedClip, PixelCastJustification::MovingDownToChildren)); + ApplyOcclusionCulling(child, localOpaque, ancestorClipForChild); } // If we have a simple transform, then we can add our opaque area into @@ -242,12 +276,11 @@ LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaque !aLayer->HasMaskLayers() && aLayer->GetLocalOpacity() == 1.0f) { if (aLayer->GetContentFlags() & Layer::CONTENT_OPAQUE) { - localOpaque.Or(localOpaque, composite->GetFullyRenderedRegion()); + localOpaque.OrWith(composite->GetFullyRenderedRegion()); } localOpaque.MoveBy(transform2d._31, transform2d._32); - const Maybe& clip = aLayer->GetEffectiveClipRect(); - if (clip) { - localOpaque.And(localOpaque, ParentLayerIntRect::ToUntyped(*clip)); + if (layerClip) { + localOpaque.And(localOpaque, ParentLayerIntRect::ToUntyped(*layerClip)); } aOpaqueRegion.Or(aOpaqueRegion, localOpaque); } @@ -305,7 +338,7 @@ LayerManagerComposite::EndTransaction(const TimeStamp& aTimeStamp, mRoot->ComputeEffectiveTransforms(gfx::Matrix4x4()); nsIntRegion opaque; - ApplyOcclusionCulling(mRoot, opaque); + ApplyOcclusionCulling(mRoot, opaque, Nothing()); Render(); #if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK) @@ -972,7 +1005,7 @@ LayerManagerComposite::RenderToPresentationSurface() mRoot->ComputeEffectiveTransforms(matrix); nsIntRegion opaque; - ApplyOcclusionCulling(mRoot, opaque); + ApplyOcclusionCulling(mRoot, opaque, Nothing()); nsIntRegion invalid; Rect bounds(0.0f, 0.0f, scale * pageWidth, (float)actualHeight); diff --git a/gfx/layers/composite/LayerManagerComposite.h b/gfx/layers/composite/LayerManagerComposite.h index ca83ff2e0f3..24ed16fb3db 100644 --- a/gfx/layers/composite/LayerManagerComposite.h +++ b/gfx/layers/composite/LayerManagerComposite.h @@ -172,8 +172,10 @@ public: * Restricts the shadow visible region of layers that are covered with * opaque content. aOpaqueRegion is the region already known to be covered * with opaque content, in the post-transform coordinate space of aLayer. + * aClipFromAncestors is an optional clip rect. */ - void ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion); + void ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion, + const Maybe& aClipFromAncestors); /** * RAII helper class to add a mask effect with the compositable from aMaskLayer From 29e69c4d0d8b61d719f75412de55819d06905325 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 22 Sep 2015 17:55:29 +0100 Subject: [PATCH 40/78] Bug 1193215 - Support for passing test directories through mach try. This adds support for web-platform-tests to mach try. It changes the implementation so that instead of passing paths to manifests, the user passes arbitary paths in the source tree, and tests under that path are run, with test discovery mainly left to the harness. --- testing/mach_commands.py | 138 ++++++++------ .../configs/b2g/emulator_automation_config.py | 4 +- .../configs/unittests/linux_unittest.py | 2 +- .../mozharness/mozilla/testing/testbase.py | 33 +++- .../mozharness/mozilla/testing/try_tools.py | 173 ++++++++---------- .../scripts/android_emulator_unittest.py | 16 +- testing/mozharness/scripts/android_panda.py | 24 ++- .../scripts/b2g_desktop_unittest.py | 19 +- .../scripts/b2g_emulator_unittest.py | 23 ++- .../mozharness/scripts/desktop_unittest.py | 75 ++++---- testing/mozharness/scripts/marionette.py | 5 +- .../mozharness/scripts/web_platform_tests.py | 33 ++-- testing/tools/autotry/autotry.py | 151 ++++++++++----- 13 files changed, 396 insertions(+), 300 deletions(-) diff --git a/testing/mach_commands.py b/testing/mach_commands.py index 1d525f37d12..9b46ad5c9f1 100644 --- a/testing/mach_commands.py +++ b/testing/mach_commands.py @@ -436,6 +436,9 @@ class JsapiTestsCommand(MachCommandBase): return jsapi_tests_result +def autotry_parser(): + from autotry import parser + return parser() @CommandProvider class PushToTry(MachCommandBase): @@ -448,6 +451,18 @@ class PushToTry(MachCommandBase): if platforms is None: platforms = os.environ['AUTOTRY_PLATFORM_HINT'] + rv_platforms = [] + for item in platforms: + for platform in item.split(","): + if platform: + rv_platforms.append(platform) + + rv_tests = [] + for item in tests: + for test in item.split(","): + if test: + rv_tests.append(test) + for p in paths: p = os.path.normpath(os.path.abspath(p)) if not p.startswith(self.topsrcdir): @@ -459,69 +474,67 @@ class PushToTry(MachCommandBase): ' select all tests.' % p) sys.exit(1) - return builds, platforms + return builds, rv_platforms, rv_tests - @Command('try', category='testing', description='Push selected tests to the try server') - @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.') - @CommandArgument('-n', dest='verbose', action='store_true', default=False, - help='Print detailed information about the resulting test selection ' - 'and commands performed.') - @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ, - help='Platforms to run. (required if not found in the environment)') - @CommandArgument('-u', dest='tests', - help='Test jobs to run. These will be used in place of suites ' - 'determined by test paths, if any.') - @CommandArgument('--extra', dest='extra_tests', - help='Additional tests to run. These will be added to suites ' - 'determined by test paths, if any.') - @CommandArgument('-b', dest='builds', default='do', - help='Build types to run (d for debug, o for optimized)') - @CommandArgument('--tag', dest='tags', action='append', - help='Restrict tests to the given tag (may be specified multiple times)') - @CommandArgument('--no-push', dest='push', action='store_false', - help='Do not push to try as a result of running this command (if ' - 'specified this command will only print calculated try ' - 'syntax and selection info).') + @Command('try', + category='testing', + description='Push selected tests to the try server', + parser=autotry_parser) def autotry(self, builds=None, platforms=None, paths=None, verbose=None, - extra_tests=None, push=None, tags=None, tests=None): - """mach try is under development, please file bugs blocking 1149670. + push=None, tags=None, tests=None, extra_args=None, intersection=False): + """Autotry is in beta, please file bugs blocking 1149670. - Pushes the specified tests to try. The simplest way to specify tests is - by using the -u argument, which will behave as usual for try syntax. - This command also provides a mechanism to select test jobs and tests - within a job by path based on tests present in the tree under that - path. Mochitests, xpcshell tests, and reftests are eligible for - selection by this mechanism. Selected tests will be run in a single - chunk of the relevant suite, at this time in chunk 1. + Push the current tree to try, with the specified syntax. - Specifying platforms is still required with the -p argument (a default - is taken from the AUTOTRY_PLATFORM_HINT environment variable if set). + Build options, platforms and regression tests may be selected + using the usual try options (-b, -p and -u respectively). In + addition, tests in a given directory may be automatically + selected by passing that directory as a positional argument to the + command. For example: - Tests may be further filtered by passing one or more --tag to the - command. If one or more --tag is specified with out paths or -u, - tests with the given tags will be run in a single chunk of - applicable suites. + mach try -b d -p linux64 dom testing/web-platform/tests/dom - To run suites in addition to those determined from the tree, they - can be passed to the --extra arguent. + would schedule a try run for linux64 debug consisting of all + tests under dom/ and testing/web-platform/tests/dom. + + Test selection using positional arguments is available for + mochitests, reftests, xpcshell tests and web-platform-tests. + + Tests may be also filtered by passing --tag to the command, + which will run only tests marked as having the specified + tags e.g. + + mach try -b d -p win64 --tag media + + would run all tests tagged 'media' on Windows 64. + + If both positional arguments or tags and -u are supplied, the + suites in -u will be run in full. Where tests are selected by + positional argument they will be run in a single chunk. + + If no build option is selected, both debug and opt will be + scheduled. If no platform is selected a default is taken from + the AUTOTRY_PLATFORM_HINT environment variable, if set. The command requires either its own mercurial extension ("push-to-try", installable from mach mercurial-setup) or a git repo using git-cinnabar (available at https://github.com/glandium/git-cinnabar). + """ from mozbuild.testing import TestResolver from mozbuild.controller.building import BuildDriver from autotry import AutoTry - import pprint - print("mach try is under development, please file bugs blocking 1149670.") - builds, platforms = self.validate_args(paths, tests, tags, builds, platforms) + if tests is None: + tests = [] + + builds, platforms, tests = self.validate_args(paths, tests, tags, builds, platforms) resolver = self._spawn(TestResolver) at = AutoTry(self.topsrcdir, resolver, self._mach_context) - if at.find_uncommited_changes(): + if push and at.find_uncommited_changes(): print('ERROR please commit changes before continuing') sys.exit(1) @@ -529,27 +542,34 @@ class PushToTry(MachCommandBase): driver = self._spawn(BuildDriver) driver.install_tests(remove=False) - manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags) + paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) + for item in paths] + paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags) - if not manifests_by_flavor and not tests: - print("No tests were found when attempting to resolve paths:\n\n\t%s" % - paths) + if not paths_by_flavor and not tests: + print("No tests were found when attempting to resolve paths:\n\n\t%s" % + paths) + sys.exit(1) + + if not intersection: + paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests) + else: + paths_by_flavor = {} + + try: + msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags, + extra_args, intersection) + except ValueError as e: + print(e.message) sys.exit(1) - all_manifests = set() - for m in manifests_by_flavor.values(): - all_manifests |= m - all_manifests = list(all_manifests) - - msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests, - extra_tests, builds, all_manifests, tags) - - if verbose and manifests_by_flavor: - print('Tests from the following manifests will be selected: ') - pprint.pprint(manifests_by_flavor) + if verbose and paths_by_flavor: + print('The following tests will be selected: ') + for flavor, paths in paths_by_flavor.iteritems(): + print("%s: %s" % (flavor, ",".join(paths))) if verbose or not push: - print('The following try syntax was calculated:\n\n\t%s\n' % msg) + print('The following try syntax was calculated:\n%s' % msg) if push: at.push_to_try(msg, verbose) diff --git a/testing/mozharness/configs/b2g/emulator_automation_config.py b/testing/mozharness/configs/b2g/emulator_automation_config.py index 03a39952c02..a80053769c7 100644 --- a/testing/mozharness/configs/b2g/emulator_automation_config.py +++ b/testing/mozharness/configs/b2g/emulator_automation_config.py @@ -118,8 +118,8 @@ config = { "--log-errorsummary=%(error_summary_file)s", "--certificate-path=%(certificate_path)s", "--screenshot-on-fail", - "%(test_path)s" ], + "tests": ["%(test_path)s"], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" }, @@ -142,8 +142,8 @@ config = { "--log-errorsummary=%(error_summary_file)s", "--certificate-path=%(certificate_path)s", "--screenshot-on-fail", - "%(test_path)s" ], + "tests": ["%(test_path)s"], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" }, diff --git a/testing/mozharness/configs/unittests/linux_unittest.py b/testing/mozharness/configs/unittests/linux_unittest.py index 5d0f0969dda..2b1c5f357fc 100644 --- a/testing/mozharness/configs/unittests/linux_unittest.py +++ b/testing/mozharness/configs/unittests/linux_unittest.py @@ -256,7 +256,7 @@ config = { }, }, "all_cppunittest_suites": { - "cppunittest": ["tests/cppunittest"] + "cppunittest": {"tests": ["tests/cppunittest"]} }, "all_gtest_suites": { "gtest": [] diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py index aa69546be2e..fc19622f0a2 100755 --- a/testing/mozharness/mozharness/mozilla/testing/testbase.py +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -26,7 +26,7 @@ from mozharness.mozilla.proxxy import Proxxy from mozharness.mozilla.structuredlog import StructuredOutputParser from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser -from mozharness.mozilla.testing.try_tools import TryToolsMixin +from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options from mozharness.mozilla.tooltool import TooltoolMixin from mozharness.lib.python.authentication import get_credentials @@ -83,7 +83,7 @@ testing_config_options = [ "choices": ['ondemand', 'true'], "help": "Download and extract crash reporter symbols.", }], -] + copy.deepcopy(virtualenv_config_options) +] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options) # TestingMixin {{{1 @@ -676,6 +676,35 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") return self.minidump_stackwalk_path + def query_options(self, *args, **kwargs): + if "str_format_values" in kwargs: + str_format_values = kwargs.pop("str_format_values") + else: + str_format_values = {} + + arguments = [] + + for arg in args: + if arg is not None: + arguments.extend(argument % str_format_values for argument in arg) + + return arguments + + def query_tests_args(self, *args, **kwargs): + if "str_format_values" in kwargs: + str_format_values = kwargs.pop("str_format_values") + else: + str_format_values = {} + + arguments = [] + + for arg in reversed(args): + if arg: + arguments.append("--") + arguments.extend(argument % str_format_values for argument in arg) + break + + return arguments def _run_cmd_checks(self, suites): if not suites: diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py index ab8edf61b97..0d9b69e0e7d 100644 --- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py @@ -8,17 +8,43 @@ import argparse import os import re +from collections import defaultdict from mozharness.base.script import PostScriptAction from mozharness.base.transfer import TransferMixin +try_config_options = [ + [["--try-message"], + {"action": "store", + "dest": "try_message", + "default": None, + "help": "try syntax string to select tests to run", + }], +] + +test_flavors = { + 'browser-chrome': {}, + 'chrome': {}, + 'devtools-chrome': {}, + 'mochitest': {}, + 'xpcshell' :{}, + 'reftest': { + "path": lambda x: os.path.join("tests", "reftest", "tests", x) + }, + 'crashtest': { + "path": lambda x: os.path.join("tests", "reftest", "tests", x) + }, + 'web-platform-tests': { + "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1]) + } +} class TryToolsMixin(TransferMixin): """Utility functions for an interface between try syntax and out test harnesses. Requires log and script mixins.""" harness_extra_args = None - try_test_paths = [] + try_test_paths = {} known_try_arguments = { '--tag': { 'action': 'append', @@ -29,31 +55,34 @@ class TryToolsMixin(TransferMixin): def _extract_try_message(self): msg = None - if self.buildbot_config['sourcestamp']['changes']: - msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments'] + if "try_message" in self.config and self.config["try_message"]: + msg = self.config["try_message"] + else: + if self.buildbot_config['sourcestamp']['changes']: + msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments'] - if msg is None or len(msg) == 1024: - # This commit message was potentially truncated, get the full message - # from hg. - props = self.buildbot_config['properties'] - rev = props['revision'] - repo = props['repo_path'] - url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev) + if msg is None or len(msg) == 1024: + # This commit message was potentially truncated, get the full message + # from hg. + props = self.buildbot_config['properties'] + rev = props['revision'] + repo = props['repo_path'] + url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev) - pushinfo = self.load_json_from_url(url) - for k, v in pushinfo.items(): - if isinstance(v, dict) and 'changesets' in v: - msg = v['changesets'][-1]['desc'] + pushinfo = self.load_json_from_url(url) + for k, v in pushinfo.items(): + if isinstance(v, dict) and 'changesets' in v: + msg = v['changesets'][-1]['desc'] - if not msg and 'try_syntax' in self.buildbot_config['properties']: - # If we don't find try syntax in the usual place, check for it in an - # alternate property available to tools using self-serve. - msg = self.buildbot_config['properties']['try_syntax'] + if not msg and 'try_syntax' in self.buildbot_config['properties']: + # If we don't find try syntax in the usual place, check for it in an + # alternate property available to tools using self-serve. + msg = self.buildbot_config['properties']['try_syntax'] return msg @PostScriptAction('download-and-extract') - def _set_extra_try_arguments(self, action, success=None): + def set_extra_try_arguments(self, action, success=None): """Finds a commit message and parses it for extra arguments to pass to the test harness command line and test paths used to filter manifests. @@ -107,7 +136,7 @@ class TryToolsMixin(TransferMixin): parser.add_argument('--try-test-paths', nargs='*') (args, _) = parser.parse_known_args(all_try_args) - self.try_test_paths = args.try_test_paths + self.try_test_paths = self._group_test_paths(args.try_test_paths) del args.try_test_paths out_args = [] @@ -127,91 +156,35 @@ class TryToolsMixin(TransferMixin): self.harness_extra_args = out_args - def _resolve_specified_manifests(self): - if not self.try_test_paths: - return None + def _group_test_paths(self, args): + rv = defaultdict(list) - target_manifests = set(self.try_test_paths) + if args is None: + return rv - def filter_ini_manifest(line): - # Lines are formatted as [include:], we care about . - parts = line.split(':') - term = line - if len(parts) == 2: - term = parts[1] - if term.endswith(']'): - term = term[:-1] - if (term in target_manifests or - any(term.startswith(l) for l in target_manifests)): - return True - return False + for item in args: + suite, path = item.split(":", 1) + rv[suite].append(path) + return rv - def filter_list_manifest(line): - # Lines are usually formatted as "include ", we care about . - parts = line.split() - term = line - if len(parts) == 2: - term = parts[1] - # Reftest master manifests also include skip-if lines and relative - # paths we aren't doing to resolve here, so unlike above this is just - # a substring check. - if (term in target_manifests or - any(l in term for l in target_manifests)): - return True - return False - - # The master manifests we need to filter for target manifests. - # TODO: All this needs to go in a config file somewhere and get sewn - # into the job definition so its less likely to break as things are - # modified. One straightforward way to achieve this would be with a key - # in the tree manifest for the master manifest path, however the current - # tree manifests don't distinguish between flavors of mochitests to this - # isn't straightforward. - master_manifests = [ - ('mochitest/chrome/chrome.ini', filter_ini_manifest), - ('mochitest/browser/browser-chrome.ini', filter_ini_manifest), - ('mochitest/tests/mochitest.ini', filter_ini_manifest), - ('xpcshell/tests/all-test-dirs.list', filter_ini_manifest), - ('xpcshell/tests/xpcshell.ini', filter_ini_manifest), - ('reftest/tests/layout/reftests/reftest.list', filter_list_manifest), - ('reftest/tests/testing/crashtest/crashtests.list', filter_list_manifest), - ] - - dirs = self.query_abs_dirs() - tests_dir = dirs.get('abs_test_install_dir', - os.path.join(dirs['abs_work_dir'], 'tests')) - master_manifests = [(os.path.join(tests_dir, name), filter_fn) for (name, filter_fn) in - master_manifests] - for m, filter_fn in master_manifests: - if not os.path.isfile(m): - continue - - self.info("Filtering master manifest at: %s" % m) - lines = self.read_from_file(m).splitlines() - - out_lines = [line for line in lines if filter_fn(line)] - - self.rmtree(m) - self.write_to_file(m, '\n'.join(out_lines)) - - - def append_harness_extra_args(self, cmd): - """Append arguments derived from try syntax to a command.""" + def try_args(self, flavor): + """Get arguments, test_list derived from try syntax to apply to a command""" # TODO: Detect and reject incompatible arguments - extra_args = self.harness_extra_args[:] if self.harness_extra_args else [] - if self.try_test_paths: - self._resolve_specified_manifests() + args = self.harness_extra_args[:] if self.harness_extra_args else [] + + if self.try_test_paths.get(flavor): self.info('TinderboxPrint: Tests will be run from the following ' - 'manifests: %s.' % ','.join(self.try_test_paths)) - extra_args.extend(['--this-chunk=1', '--total-chunks=1']) + 'files: %s.' % ','.join(self.try_test_paths[flavor])) + args.extend(['--this-chunk=1', '--total-chunks=1']) - if not extra_args: - return cmd + path_func = test_flavors[flavor].get("path", lambda x:x) + tests = [path_func(item) for item in self.try_test_paths[flavor]] + else: + tests = [] - out_cmd = cmd[:] - out_cmd.extend(extra_args) - self.info('TinderboxPrint: The following arguments were forwarded from mozharness ' - 'to the test command:\nTinderboxPrint: \t%s' % - extra_args) + if args or tests: + self.info('TinderboxPrint: The following arguments were forwarded from mozharness ' + 'to the test command:\nTinderboxPrint: \t%s -- %s' % + (" ".join(args), " ".join(tests))) - return out_cmd + return args, tests diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index 739e3d6f451..c240f2cca9b 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -483,14 +483,12 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin continue cmd.append(arg) - tests = None - if "tests" in self.test_suite_definitions[self.test_suite]: - tests = self.test_suite_definitions[self.test_suite]["tests"] - elif "tests" in self.config["suite_definitions"][suite_category]: - tests = self.config["suite_definitions"][suite_category]["tests"] - - if tests: - cmd.extend(tests) + try_options, try_tests = self.try_args(suite_category) + cmd.extend(try_options) + cmd.extend(self.query_tests_args( + self.config["suite_definitions"][suite_category].get("tests"), + self.test_suite_definitions[self.test_suite].get("tests"), + try_tests)) return cmd @@ -664,7 +662,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin Run the tests """ cmd = self._build_command() - cmd = self.append_harness_extra_args(cmd) + try: cwd = self._query_tests_dir() except: diff --git a/testing/mozharness/scripts/android_panda.py b/testing/mozharness/scripts/android_panda.py index 695e9a0d978..66271ebc15f 100644 --- a/testing/mozharness/scripts/android_panda.py +++ b/testing/mozharness/scripts/android_panda.py @@ -217,6 +217,20 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu if self._query_specified_suites(cat) is not None] super(PandaTest, self).download_and_extract(suite_categories=target_categories) + def _query_try_flavor(self, category, suite): + flavors = { + "mochitest": [("plain.*", "mochitest"), + ("browser-chrome.*", "browser-chrome"), + ("mochitest-devtools-chrome.*", "devtools-chrome"), + ("chrome", "chrome")], + "xpcshell": [("xpcshell", "xpcshell")], + "reftest": [("reftest", "reftest"), + ("crashtest", "crashtest")] + } + for suite_pattern, flavor in flavors.get(category, []): + if re.compile(suite_pattern).match(suite): + return flavor + def _run_category_suites(self, suite_category, preflight_run_method=None): """run suite(s) to a specific category""" @@ -246,11 +260,13 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu if should_install_app: self._install_app() cmd = abs_base_cmd[:] - replace_dict = {} - for arg in suites[suite]: - cmd.append(arg % replace_dict) - cmd = self.append_harness_extra_args(cmd) + flavor = self._query_try_flavor(suite_category, suite) + try_options, try_tests = self.try_args(flavor) + + cmd.extend(self.query_options(suites[suite], + try_options)) + cmd.extend(self.query_tests_args(try_tests)) tests = self.config["suite_definitions"][suite_category].get("tests", []) cmd += tests diff --git a/testing/mozharness/scripts/b2g_desktop_unittest.py b/testing/mozharness/scripts/b2g_desktop_unittest.py index f6040a1ab83..6fb7f3f2fab 100755 --- a/testing/mozharness/scripts/b2g_desktop_unittest.py +++ b/testing/mozharness/scripts/b2g_desktop_unittest.py @@ -139,6 +139,7 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): '%s_raw.log' % suite) error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], '%s_errorsummary.log' % suite) + str_format_values = { 'application': self.binary_path, 'test_manifest': self.test_manifest, @@ -156,16 +157,15 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): if suite not in self.config["suite_definitions"]: self.fatal("'%s' not defined in the config!" % suite) - options = self.config["suite_definitions"][suite]["options"] - if options: - for option in options: - option = option % str_format_values - if not option.endswith('None'): - cmd.append(option) + try_options, try_tests = self.try_args(suite_name) - tests = self.config["suite_definitions"][suite].get("tests") - if tests: - cmd.extend(tests) + cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite]["options"], + try_options, + str_format_values=str_format_values)) + + cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite].get("tests"), + try_tests, + str_format_values=str_format_values) return cmd @@ -200,7 +200,6 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): self.fatal("Don't know how to run --test-suite '%s'!" % suite) cmd = self._query_abs_base_cmd(suite) - cmd = self.append_harness_extra_args(cmd) cwd = dirs['abs_%s_dir' % suite] diff --git a/testing/mozharness/scripts/b2g_emulator_unittest.py b/testing/mozharness/scripts/b2g_emulator_unittest.py index 9b78ddb53d4..c0dab5d838a 100755 --- a/testing/mozharness/scripts/b2g_emulator_unittest.py +++ b/testing/mozharness/scripts/b2g_emulator_unittest.py @@ -273,18 +273,19 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin): } if suite not in self.config["suite_definitions"]: - self.fatal("Key '%s' not defined in the config!" % suite) + self.fatal("'%s' not defined in the config!" % suite) - options = self.config["suite_definitions"][suite]["options"] - if options: - for option in options: - option = option % str_format_values - if not option.endswith('None'): - cmd.append(option) + try_options, try_tests = self.try_args(suite) - tests = self.config["suite_definitions"][suite].get("tests", []) - if tests: - cmd.extend(tests) + options = self.query_options(self.config["suite_definitions"][suite]["options"], + try_options, + str_format_values=str_format_values) + cmd.extend(opt for opt in options if not opt.endswith('None')) + + tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"), + try_tests, + str_format_values=str_format_values) + cmd.extend(opt for opt in tests if not opt.endswith('None')) return cmd @@ -341,8 +342,6 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin): self.fatal("Don't know how to run --test-suite '%s'!" % suite) cmd = self._query_abs_base_cmd(suite) - cmd = self.append_harness_extra_args(cmd) - cwd = dirs['abs_%s_dir' % suite] # TODO we probably have to move some of the code in diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index 3c74b3c8ecf..d57e3593013 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -34,7 +34,6 @@ from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_opt SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt'] - # DesktopUnittest {{{1 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin): config_options = [ @@ -299,32 +298,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix self.symbols_url = symbols_url return self.symbols_url - def get_webapprt_path(self, res_dir, mochitest_dir): - """Get the path to the webapp runtime binary. - On Mac, we copy the stub from the resources dir to the test app bundle, - since we have to run it from the executable directory of a bundle - in order for its windows to appear. Ideally, the build system would do - this for us at build time, and we should find a way for it to do that. - """ - exe_suffix = self.config.get('exe_suffix', '') - app_name = 'webapprt-stub' + exe_suffix - app_path = os.path.join(res_dir, app_name) - if self._is_darwin(): - mac_dir_name = os.path.join( - mochitest_dir, - 'webapprtChrome', - 'webapprt', - 'test', - 'chrome', - 'TestApp.app', - 'Contents', - 'MacOS') - mac_app_name = 'webapprt' + exe_suffix - mac_app_path = os.path.join(mac_dir_name, mac_app_name) - self.copyfile(app_path, mac_app_path, copystat=True) - return mac_app_path - return app_path - def _query_abs_base_cmd(self, suite_category, suite): if self.binary_path: c = self.config @@ -335,6 +308,11 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix abs_app_dir = self.query_abs_app_dir() abs_res_dir = self.query_abs_res_dir() + webapprt_path = os.path.join(os.path.dirname(self.binary_path), + 'webapprt-stub') + if c.get('exe_suffix'): + webapprt_path += c['exe_suffix'] + raw_log_file = os.path.join(dirs['abs_blob_upload_dir'], '%s_raw.log' % suite) @@ -344,7 +322,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix 'binary_path': self.binary_path, 'symbols_path': self._query_symbols_url(), 'abs_app_dir': abs_app_dir, - 'abs_res_dir': abs_res_dir, + 'app_path': webapprt_path, 'raw_log_file': raw_log_file, 'error_summary_file': error_summary_file, 'gtest_dir': os.path.join(dirs['abs_test_install_dir'], @@ -356,9 +334,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix if self.symbols_path: str_format_values['symbols_path'] = self.symbols_path - if suite_category == 'webapprt': - str_format_values['app_path'] = self.get_webapprt_path(abs_res_dir, dirs['abs_mochitest_dir']) - if c['e10s']: base_cmd.append('--e10s') @@ -398,7 +373,14 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix "please make sure they are specified in your " "config under %s_options" % (suite_category, suite_category)) - return base_cmd + + + for option in options: + option = option % str_format_values + if not option.endswith('None'): + base_cmd.append(option) + + return base_cmd else: self.fatal("'binary_path' could not be determined.\n This should " "be like '/path/build/application/firefox/firefox'" @@ -434,6 +416,20 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix return suites + def _query_try_flavor(self, category, suite): + flavors = { + "mochitest": [("plain.*", "mochitest"), + ("browser-chrome.*", "browser-chrome"), + ("mochitest-devtools-chrome.*", "devtools-chrome"), + ("chrome", "chrome")], + "xpcshell": [("xpcshell", "xpcshell")], + "reftest": [("reftest", "reftest"), + ("crashtest", "crashtest")] + } + for suite_pattern, flavor in flavors.get(category, []): + if re.compile(suite_pattern).match(suite): + return flavor + # Actions {{{2 # clobber defined in BaseScript, deletes mozharness/build if exists @@ -588,15 +584,22 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix options_list = [] env = {} if isinstance(suites[suite], dict): - options_list = suites[suite]['options'] + suites[suite].get("tests", []) + options_list = suites[suite].get('options', []) + tests_list = suites[suite].get('tests', []) env = copy.deepcopy(suites[suite].get('env', {})) else: options_list = suites[suite] + tests_list = [] - for arg in options_list: - cmd.append(arg % replace_dict) + flavor = self._query_try_flavor(suite_category, suite) + try_options, try_tests = self.try_args(flavor) - cmd = self.append_harness_extra_args(cmd) + cmd.extend(self.query_options(options_list, + try_options, + str_format_values=replace_dict)) + cmd.extend(self.query_tests_args(tests_list, + try_tests, + str_format_values=replace_dict)) suite_name = suite_category + '-' + suite tbpl_status, log_level = None, None diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py index 4ae73b59cff..6f84275c80b 100755 --- a/testing/mozharness/scripts/marionette.py +++ b/testing/mozharness/scripts/marionette.py @@ -439,7 +439,10 @@ class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMix self.fatal("Could not create blobber upload directory") cmd.append(manifest) - cmd = self.append_harness_extra_args(cmd) + + try_options, try_tests = self.try_args("marionette") + cmd.extend(self.query_tests_args(try_tests, + str_format_values=config_fmt_args)) env = {} if self.query_minidump_stackwalk(): diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index f1982b2ca0c..0167f487a22 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -115,31 +115,31 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): abs_app_dir = self.query_abs_app_dir() run_file_name = "runtests.py" - base_cmd = [self.query_python_path('python'), '-u'] - base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name)) + cmd = [self.query_python_path('python'), '-u'] + cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name)) # Make sure that the logging directory exists if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1: self.fatal("Could not create blobber upload directory") # Exit - base_cmd += ["--log-raw=-", - "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"], - "wpt_raw.log"), - "--binary=%s" % self.binary_path, - "--symbols-path=%s" % self.query_symbols_url(), - "--stackwalk-binary=%s" % self.query_minidump_stackwalk()] + cmd += ["--log-raw=-", + "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"], + "wpt_raw.log"), + "--binary=%s" % self.binary_path, + "--symbols-path=%s" % self.query_symbols_url(), + "--stackwalk-binary=%s" % self.query_minidump_stackwalk()] for test_type in c.get("test_type", []): - base_cmd.append("--test-type=%s" % test_type) + cmd.append("--test-type=%s" % test_type) if c.get("e10s"): - base_cmd.append("--e10s") + cmd.append("--e10s") for opt in ["total_chunks", "this_chunk"]: val = c.get(opt) if val: - base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val)) + cmd.append("--%s=%s" % (opt.replace("_", "-"), val)) options = list(c.get("options", [])) @@ -151,9 +151,15 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): 'abs_work_dir': dirs["abs_work_dir"] } - opt_cmd = [item % str_format_values for item in options] + try_options, try_tests = self.try_args("web-platform-tests") - return base_cmd + opt_cmd + cmd.extend(self.query_options(options, + try_options, + str_format_values=str_format_values)) + cmd.extend(self.query_tests_args(try_tests, + str_format_values=str_format_values)) + + return cmd def download_and_extract(self): super(WebPlatformTest, self).download_and_extract( @@ -167,7 +173,6 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): def run_tests(self): dirs = self.query_abs_dirs() cmd = self._query_cmd() - cmd = self.append_harness_extra_args(cmd) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj) diff --git a/testing/tools/autotry/autotry.py b/testing/tools/autotry/autotry.py index ca8dd4f5645..217ee62924d 100644 --- a/testing/tools/autotry/autotry.py +++ b/testing/tools/autotry/autotry.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import argparse import sys import os import itertools @@ -10,22 +11,60 @@ import which from collections import defaultdict +import ConfigParser + + +def parser(): + parser = argparse.ArgumentParser() + parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.') + parser.add_argument('-p', dest='platforms', action="append", + help='Platforms to run. (required if not found in the environment)') + parser.add_argument('-u', dest='tests', action="append", + help='Test suites to run in their entirety') + parser.add_argument('-b', dest='builds', default='do', + help='Build types to run (d for debug, o for optimized)') + parser.add_argument('--tag', dest='tags', action='append', + help='Restrict tests to the given tag (may be specified multiple times)') + parser.add_argument('--and', action='store_true', dest="intersection", + help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments') + parser.add_argument('--no-push', dest='push', action='store_false', + help='Do not push to try as a result of running this command (if ' + 'specified this command will only print calculated try ' + 'syntax and selection info).') + parser.add_argument('extra_args', nargs=argparse.REMAINDER, + help='Extra arguments to put in the try push') + parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False, + help='Print detailed information about the resulting test selection ' + 'and commands performed.') + return parser -TRY_SYNTAX_TMPL = """ -try: -b %s -p %s -u %s -t none %s %s -""" class AutoTry(object): - test_flavors = [ - 'browser-chrome', - 'chrome', - 'devtools-chrome', - 'mochitest', - 'xpcshell', - 'reftest', - 'crashtest', - ] + # Maps from flavors to the job names needed to run that flavour + flavor_jobs = { + 'mochitest': ['mochitest-1', 'mochitest-e10s-1'], + 'xpcshell': ['xpcshell'], + 'chrome': ['mochitest-o'], + 'browser-chrome': ['mochitest-browser-chrome-1', + 'mochitest-e10s-browser-chrome-1'], + 'devtools-chrome': ['mochitest-dt', + 'mochitest-e10s-devtools-chrome'], + 'crashtest': ['crashtest', 'crashtest-e10s'], + 'reftest': ['reftest', 'reftest-e10s'], + 'web-platform-tests': ['web-platform-tests-1'], + } + + flavor_suites = { + "mochitest": "mochitests", + "xpcshell": "xpcshell", + "chrome": "mochitest-o", + "browser-chrome": "mochitest-bc", + "devtools-chrome": "mochitest-dt", + "crashtest": "crashtest", + "reftest": "reftest", + "web-platform-tests": "web-platform-tests", + } def __init__(self, topsrcdir, resolver, mach_context): self.topsrcdir = topsrcdir @@ -37,57 +76,69 @@ class AutoTry(object): else: self._use_git = True - def resolve_manifests(self, paths=None, tags=None): - if not (paths or tags): - return {} + def paths_by_flavor(self, paths=None, tags=None): + paths_by_flavor = defaultdict(set) - tests = list(self.resolver.resolve_tests(tags=tags, - paths=paths, - cwd=self.mach_context.cwd)) - manifests_by_flavor = defaultdict(set) + if not (paths or tags): + return dict(paths_by_flavor) + + tests = list(self.resolver.resolve_tests(paths=paths, + tags=tags)) for t in tests: - if t['flavor'] in AutoTry.test_flavors: + if t['flavor'] in self.flavor_suites: flavor = t['flavor'] if 'subsuite' in t and t['subsuite'] == 'devtools': flavor = 'devtools-chrome' - manifest = os.path.relpath(t['manifest'], self.topsrcdir) - manifests_by_flavor[flavor].add(manifest) - return dict(manifests_by_flavor) + for path in paths: + if flavor in ["crashtest", "reftest"]: + manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir) + if manifest_relpath.startswith(path): + paths_by_flavor[flavor].add(manifest_relpath) + else: + if t['file_relpath'].startswith(path): + paths_by_flavor[flavor].add(path) - def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds, - manifests, tags): + return dict(paths_by_flavor) - # Maps from flavors to the try syntax snippets implied by that flavor. - # TODO: put selected tests under their own builder/label to avoid - # confusion reading results on treeherder. - flavor_suites = { - 'mochitest': ['mochitest-1', 'mochitest-e10s-1'], - 'xpcshell': ['xpcshell'], - 'chrome': ['mochitest-o'], - 'browser-chrome': ['mochitest-browser-chrome-1', - 'mochitest-e10s-browser-chrome-1'], - 'devtools-chrome': ['mochitest-dt', - 'mochitest-e10s-devtools-chrome'], - 'crashtest': ['crashtest', 'crashtest-e10s'], - 'reftest': ['reftest', 'reftest-e10s'], - } + def remove_duplicates(self, paths_by_flavor, tests): + rv = {} + for item in paths_by_flavor: + if self.flavor_suites[item] not in tests: + rv[item] = paths_by_flavor[item].copy() + return rv + + def calc_try_syntax(self, platforms, tests, builds, paths_by_flavor, tags, extra_args, + intersection): + parts = ["try:", "-b", builds, "-p", ",".join(platforms)] + + suites = set(tests) if not intersection else set() + paths = set() + for flavor, flavor_tests in paths_by_flavor.iteritems(): + suite = self.flavor_suites[flavor] + if suite not in suites and (not intersection or suite in tests): + for job_name in self.flavor_jobs[flavor]: + for test in flavor_tests: + paths.add("%s:%s" % (flavor, test)) + suites.add(job_name) + + if not suites: + raise ValueError("No tests found matching filters") + + parts.append("-u") + parts.append(",".join(sorted(suites))) if tags: - tags = ' '.join('--tag %s' % t for t in tags) - else: - tags = '' + parts.append(' '.join('--tag %s' % t for t in tags)) - if not tests: - tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors))) - if extra_tests: - tests += ',%s' % (extra_tests) + if extra_args is not None: + parts.extend(extra_args) - manifests = ' '.join(manifests) - if manifests: - manifests = '--try-test-paths %s' % manifests - return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags) + if paths: + parts.append("--try-test-paths %s" % " ".join(sorted(paths))) + + return " ".join(parts) def _run_git(self, *args): args = ['git'] + list(args) From 064ad2defc5964562278005b25f780336181b455 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 29 Jul 2015 17:45:33 +0100 Subject: [PATCH 41/78] Bug 1193264 - Add support for saving and reusing try strings in mach try Adds --save and --preset arguments that can be used to store and reuse frequently used try strings. --- testing/mach_commands.py | 105 +++++++++++++++------- testing/tools/autotry/autotry.py | 148 +++++++++++++++++++++++++++++-- 2 files changed, 213 insertions(+), 40 deletions(-) diff --git a/testing/mach_commands.py b/testing/mach_commands.py index 9b46ad5c9f1..5f1b6ca41a7 100644 --- a/testing/mach_commands.py +++ b/testing/mach_commands.py @@ -10,6 +10,7 @@ import sys import tempfile import subprocess import shutil +from collections import defaultdict from mach.decorators import ( CommandArgument, @@ -437,33 +438,51 @@ class JsapiTestsCommand(MachCommandBase): return jsapi_tests_result def autotry_parser(): - from autotry import parser - return parser() + from autotry import arg_parser + return arg_parser() @CommandProvider class PushToTry(MachCommandBase): + def normalise_list(self, items, allow_subitems=False): + from autotry import parse_arg - def validate_args(self, paths, tests, tags, builds, platforms): - if not any([len(paths), tests, tags]): - print("Paths, tests, or tags must be specified.") + rv = defaultdict(list) + for item in items: + parsed = parse_arg(item) + for key, values in parsed.iteritems(): + rv[key].extend(values) + + if not allow_subitems: + if not all(item == [] for item in rv.itervalues()): + raise ValueError("Unexpected subitems in argument") + return rv.keys() + else: + return rv + + def validate_args(self, **kwargs): + if not kwargs["paths"] and not kwargs["tests"] and not kwargs["tags"]: + print("Paths, tags, or tests must be specified as an argument to autotry.") sys.exit(1) - if platforms is None: - platforms = os.environ['AUTOTRY_PLATFORM_HINT'] + if kwargs["platforms"] is None: + print("Platforms must be specified as an argument to autotry") + sys.exit(1) - rv_platforms = [] - for item in platforms: - for platform in item.split(","): - if platform: - rv_platforms.append(platform) + try: + platforms = self.normalise_list(kwargs["platforms"]) + except ValueError as e: + print("Error parsing -p argument:\n%s" % e.message) + sys.exit(1) - rv_tests = [] - for item in tests: - for test in item.split(","): - if test: - rv_tests.append(test) + try: + tests = (self.normalise_list(kwargs["tests"], allow_subitems=True) + if kwargs["tests"] else {}) + except ValueError as e: + print("Error parsing -u argument:\n%s" % e.message) + sys.exit(1) - for p in paths: + paths = [] + for p in kwargs["paths"]: p = os.path.normpath(os.path.abspath(p)) if not p.startswith(self.topsrcdir): print('Specified path "%s" is outside of the srcdir, unable to' @@ -473,15 +492,23 @@ class PushToTry(MachCommandBase): print('Specified path "%s" is at the top of the srcdir and would' ' select all tests.' % p) sys.exit(1) + paths.append(os.path.relpath(p, self.topsrcdir)) + + try: + tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] + except ValueError as e: + print("Error parsing --tags argument:\n%s" % e.message) + sys.exit(1) + + return kwargs["builds"], platforms, tests, paths, tags, kwargs["extra_args"] - return builds, rv_platforms, rv_tests @Command('try', category='testing', description='Push selected tests to the try server', parser=autotry_parser) - def autotry(self, builds=None, platforms=None, paths=None, verbose=None, - push=None, tags=None, tests=None, extra_args=None, intersection=False): + + def autotry(self, **kwargs): """Autotry is in beta, please file bugs blocking 1149670. Push the current tree to try, with the specified syntax. @@ -527,14 +554,23 @@ class PushToTry(MachCommandBase): from autotry import AutoTry print("mach try is under development, please file bugs blocking 1149670.") - if tests is None: - tests = [] - - builds, platforms, tests = self.validate_args(paths, tests, tags, builds, platforms) resolver = self._spawn(TestResolver) - at = AutoTry(self.topsrcdir, resolver, self._mach_context) - if push and at.find_uncommited_changes(): + + if kwargs["load"] is not None: + defaults = at.load_config(kwargs["load"]) + + if defaults is None: + print("No saved configuration called %s found in autotry.ini" % kwargs["load"], + file=sys.stderr) + + for key, value in kwargs.iteritems(): + if value in (None, []) and key in defaults: + kwargs[key] = defaults[key] + + builds, platforms, tests, paths, tags, extra_args = self.validate_args(**kwargs) + + if kwargs["push"] and at.find_uncommited_changes(): print('ERROR please commit changes before continuing') sys.exit(1) @@ -551,30 +587,31 @@ class PushToTry(MachCommandBase): paths) sys.exit(1) - if not intersection: + if not kwargs["intersection"]: paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests) else: paths_by_flavor = {} try: msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags, - extra_args, intersection) + extra_args, kwargs["intersection"]) except ValueError as e: print(e.message) sys.exit(1) - if verbose and paths_by_flavor: + if kwargs["verbose"] and paths_by_flavor: print('The following tests will be selected: ') for flavor, paths in paths_by_flavor.iteritems(): print("%s: %s" % (flavor, ",".join(paths))) - if verbose or not push: + if kwargs["verbose"] or not kwargs["push"]: print('The following try syntax was calculated:\n%s' % msg) - if push: - at.push_to_try(msg, verbose) + if kwargs["push"]: + at.push_to_try(msg, kwargs["verbose"]) - return + if kwargs["save"] is not None: + at.save_config(kwargs["save"], msg) def get_parser(argv=None): diff --git a/testing/tools/autotry/autotry.py b/testing/tools/autotry/autotry.py index 217ee62924d..4825e441a27 100644 --- a/testing/tools/autotry/autotry.py +++ b/testing/tools/autotry/autotry.py @@ -3,10 +3,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import argparse -import sys -import os import itertools +import os +import re import subprocess +import sys import which from collections import defaultdict @@ -14,7 +15,7 @@ from collections import defaultdict import ConfigParser -def parser(): +def arg_parser(): parser = argparse.ArgumentParser() parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.') parser.add_argument('-p', dest='platforms', action="append", @@ -31,6 +32,10 @@ def parser(): help='Do not push to try as a result of running this command (if ' 'specified this command will only print calculated try ' 'syntax and selection info).') + parser.add_argument('--save', dest="save", action='store', + help="Save the command line arguments for future use with --preset") + parser.add_argument('--preset', dest="load", action='store', + help="Load a saved set of arguments. Additional arguments will override saved ones") parser.add_argument('extra_args', nargs=argparse.REMAINDER, help='Extra arguments to put in the try push') parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False, @@ -38,6 +43,102 @@ def parser(): 'and commands performed.') return parser +class TryArgumentTokenizer(object): + symbols = [("seperator", ","), + ("list_start", "\["), + ("list_end", "\]"), + ("item", "([^,\[\]\s][^,\[\]]+)"), + ("space", "\s+")] + token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) + + def tokenize(self, data): + for match in self.token_re.finditer(data): + symbol = match.lastgroup + data = match.group(symbol) + if symbol == "space": + pass + else: + yield symbol, data + +class TryArgumentParser(object): + """Simple three-state parser for handling expressions + of the from "foo[sub item, another], bar,baz". This takes + input from the TryArgumentTokenizer and runs through a small + state machine, returning a dictionary of {top-level-item:[sub_items]} + i.e. the above would result in + {"foo":["sub item", "another"], "bar": [], "baz": []} + In the case of invalid input a ValueError is raised.""" + + EOF = object() + + def __init__(self): + self.reset() + + def reset(self): + self.tokens = None + self.current_item = None + self.data = {} + self.token = None + self.state = None + + def parse(self, tokens): + self.reset() + self.tokens = tokens + self.consume() + self.state = self.item_state + while self.token[0] != self.EOF: + self.state() + return self.data + + def consume(self): + try: + self.token = self.tokens.next() + except StopIteration: + self.token = (self.EOF, None) + + def expect(self, *types): + if self.token[0] not in types: + raise ValueError("Error parsing try string, unexpected %s" % (self.token[0])) + + def item_state(self): + self.expect("item") + value = self.token[1].strip() + if value not in self.data: + self.data[value] = [] + self.current_item = value + self.consume() + if self.token[0] == "seperator": + self.consume() + elif self.token[0] == "list_start": + self.consume() + self.state = self.subitem_state + elif self.token[0] == self.EOF: + pass + else: + raise ValueError + + def subitem_state(self): + self.expect("item") + value = self.token[1].strip() + self.data[self.current_item].append(value) + self.consume() + if self.token[0] == "seperator": + self.consume() + elif self.token[0] == "list_end": + self.consume() + self.state = self.after_list_end_state + else: + raise ValueError + + def after_list_end_state(self): + self.expect("seperator") + self.consume() + self.state = self.item_state + +def parse_arg(arg): + tokenizer = TryArgumentTokenizer() + parser = TryArgumentParser() + return parser.parse(tokenizer.tokenize(arg)) class AutoTry(object): @@ -76,6 +177,40 @@ class AutoTry(object): else: self._use_git = True + @property + def config_path(self): + return os.path.join(self.mach_context.state_dir, "autotry.ini") + + def load_config(self, name): + config = ConfigParser.RawConfigParser() + success = config.read([self.config_path]) + if not success: + return None + + try: + data = config.get("try", name) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return None + + kwargs = vars(arg_parser().parse_args(data.split())) + + return kwargs + + def save_config(self, name, data): + assert data.startswith("try: ") + data = data[len("try: "):] + + parser = ConfigParser.RawConfigParser() + parser.read([self.config_path]) + + if not parser.has_section("try"): + parser.add_section("try") + + parser.set("try", name, data) + + with open(self.config_path, "w") as f: + parser.write(f) + def paths_by_flavor(self, paths=None, tags=None): paths_by_flavor = defaultdict(set) @@ -113,7 +248,7 @@ class AutoTry(object): intersection): parts = ["try:", "-b", builds, "-p", ",".join(platforms)] - suites = set(tests) if not intersection else set() + suites = tests if not intersection else {} paths = set() for flavor, flavor_tests in paths_by_flavor.iteritems(): suite = self.flavor_suites[flavor] @@ -121,13 +256,14 @@ class AutoTry(object): for job_name in self.flavor_jobs[flavor]: for test in flavor_tests: paths.add("%s:%s" % (flavor, test)) - suites.add(job_name) + suites[job_name] = tests.get(suite, []) if not suites: raise ValueError("No tests found matching filters") parts.append("-u") - parts.append(",".join(sorted(suites))) + parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "") + for k,v in sorted(suites.items()))) if tags: parts.append(' '.join('--tag %s' % t for t in tags)) From e176b422753ff5acdfebf0082e737ccb0a48bf78 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 11 Sep 2015 18:41:45 +0100 Subject: [PATCH 42/78] Bug 1204120 - Allow passing talos arguments to |mach try|. This has the side effect that passing -t none doesn't result in bare -t being passed (because "none" is interpreted as a test path). --- testing/mach_commands.py | 13 ++++++++++--- testing/tools/autotry/autotry.py | 31 ++++++++++++++++++------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/testing/mach_commands.py b/testing/mach_commands.py index 5f1b6ca41a7..ed6fb19ec5d 100644 --- a/testing/mach_commands.py +++ b/testing/mach_commands.py @@ -481,6 +481,12 @@ class PushToTry(MachCommandBase): print("Error parsing -u argument:\n%s" % e.message) sys.exit(1) + try: + talos = self.normalise_list(kwargs["talos"]) if kwargs["talos"] else [] + except ValueError as e: + print("Error parsing -t argument:\n%s" % e.message) + sys.exit(1) + paths = [] for p in kwargs["paths"]: p = os.path.normpath(os.path.abspath(p)) @@ -500,7 +506,7 @@ class PushToTry(MachCommandBase): print("Error parsing --tags argument:\n%s" % e.message) sys.exit(1) - return kwargs["builds"], platforms, tests, paths, tags, kwargs["extra_args"] + return kwargs["builds"], platforms, tests, talos, paths, tags, kwargs["extra_args"] @Command('try', @@ -552,6 +558,7 @@ class PushToTry(MachCommandBase): from mozbuild.testing import TestResolver from mozbuild.controller.building import BuildDriver from autotry import AutoTry + print("mach try is under development, please file bugs blocking 1149670.") resolver = self._spawn(TestResolver) @@ -568,7 +575,7 @@ class PushToTry(MachCommandBase): if value in (None, []) and key in defaults: kwargs[key] = defaults[key] - builds, platforms, tests, paths, tags, extra_args = self.validate_args(**kwargs) + builds, platforms, tests, talos, paths, tags, extra_args = self.validate_args(**kwargs) if kwargs["push"] and at.find_uncommited_changes(): print('ERROR please commit changes before continuing') @@ -593,7 +600,7 @@ class PushToTry(MachCommandBase): paths_by_flavor = {} try: - msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags, + msg = at.calc_try_syntax(platforms, tests, talos, builds, paths_by_flavor, tags, extra_args, kwargs["intersection"]) except ValueError as e: print(e.message) diff --git a/testing/tools/autotry/autotry.py b/testing/tools/autotry/autotry.py index 4825e441a27..dbbd2313106 100644 --- a/testing/tools/autotry/autotry.py +++ b/testing/tools/autotry/autotry.py @@ -18,26 +18,28 @@ import ConfigParser def arg_parser(): parser = argparse.ArgumentParser() parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.') - parser.add_argument('-p', dest='platforms', action="append", - help='Platforms to run. (required if not found in the environment)') - parser.add_argument('-u', dest='tests', action="append", - help='Test suites to run in their entirety') parser.add_argument('-b', dest='builds', default='do', - help='Build types to run (d for debug, o for optimized)') + help='Build types to run (d for debug, o for optimized).') + parser.add_argument('-p', dest='platforms', action="append", + help='Platforms to run (required if not found in the environment).') + parser.add_argument('-u', dest='tests', action="append", + help='Test suites to run in their entirety.') + parser.add_argument('-t', dest="talos", action="append", + help='Talos suites to run.') parser.add_argument('--tag', dest='tags', action='append', - help='Restrict tests to the given tag (may be specified multiple times)') + help='Restrict tests to the given tag (may be specified multiple times).') parser.add_argument('--and', action='store_true', dest="intersection", - help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments') + help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments.') parser.add_argument('--no-push', dest='push', action='store_false', help='Do not push to try as a result of running this command (if ' 'specified this command will only print calculated try ' 'syntax and selection info).') parser.add_argument('--save', dest="save", action='store', - help="Save the command line arguments for future use with --preset") + help="Save the command line arguments for future use with --preset.") parser.add_argument('--preset', dest="load", action='store', - help="Load a saved set of arguments. Additional arguments will override saved ones") + help="Load a saved set of arguments. Additional arguments will override saved ones.") parser.add_argument('extra_args', nargs=argparse.REMAINDER, - help='Extra arguments to put in the try push') + help='Extra arguments to put in the try push.') parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False, help='Print detailed information about the resulting test selection ' 'and commands performed.') @@ -244,8 +246,8 @@ class AutoTry(object): rv[item] = paths_by_flavor[item].copy() return rv - def calc_try_syntax(self, platforms, tests, builds, paths_by_flavor, tags, extra_args, - intersection): + def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags, + extra_args, intersection): parts = ["try:", "-b", builds, "-p", ",".join(platforms)] suites = tests if not intersection else {} @@ -263,7 +265,10 @@ class AutoTry(object): parts.append("-u") parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "") - for k,v in sorted(suites.items()))) + for k,v in sorted(suites.items())) if suites else "none") + + parts.append("-t") + parts.append(",".join(talos) if talos else "none") if tags: parts.append(' '.join('--tag %s' % t for t in tags)) From acf43b1acb87d78b9ed4648dea86832f52a05ae2 Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 19 Sep 2015 12:03:24 +0900 Subject: [PATCH 43/78] Bug 1201869 - Part 0: Rename MNewArray::count to MNewArray::length. r=Waldo --- js/src/jit/CodeGenerator.cpp | 8 ++++---- js/src/jit/IonBuilder.cpp | 4 ++-- js/src/jit/IonBuilder.h | 2 +- js/src/jit/MIR.cpp | 12 ++++++------ js/src/jit/MIR.h | 14 +++++++------- js/src/jit/Recover.cpp | 2 +- js/src/jit/ScalarReplacement.cpp | 6 +++--- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 5f8d9d6007b..6ba71b5dbfe 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -4220,12 +4220,12 @@ CodeGenerator::visitNewArrayCallVM(LNewArray* lir) if (templateObject) { pushArg(Imm32(lir->mir()->convertDoubleElements())); pushArg(ImmGCPtr(templateObject->group())); - pushArg(Imm32(lir->mir()->count())); + pushArg(Imm32(lir->mir()->length())); callVM(NewArrayWithGroupInfo, lir); } else { pushArg(Imm32(GenericObject)); - pushArg(Imm32(lir->mir()->count())); + pushArg(Imm32(lir->mir()->length())); pushArg(ImmPtr(lir->mir()->pc())); pushArg(ImmGCPtr(lir->mir()->block()->info().script())); @@ -4301,9 +4301,9 @@ CodeGenerator::visitNewArray(LNewArray* lir) Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); JSObject* templateObject = lir->mir()->templateObject(); - DebugOnly count = lir->mir()->count(); + DebugOnly length = lir->mir()->length(); - MOZ_ASSERT(count < NativeObject::NELEMENTS_LIMIT); + MOZ_ASSERT(length < NativeObject::NELEMENTS_LIMIT); if (lir->mir()->shouldUseVM()) { visitNewArrayCallVM(lir); diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index 0ae42750a34..54f2b870026 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -6925,7 +6925,7 @@ IonBuilder::compareTrySharedStub(bool* emitted, JSOp op, MDefinition* left, MDef } bool -IonBuilder::jsop_newarray(uint32_t count) +IonBuilder::jsop_newarray(uint32_t length) { JSObject* templateObject = inspector->getTemplateObject(pc); gc::InitialHeap heap; @@ -6940,7 +6940,7 @@ IonBuilder::jsop_newarray(uint32_t count) } current->add(templateConst); - MNewArray* ins = MNewArray::New(alloc(), constraints(), count, templateConst, heap, pc); + MNewArray* ins = MNewArray::New(alloc(), constraints(), length, templateConst, heap, pc); current->add(ins); current->push(ins); diff --git a/js/src/jit/IonBuilder.h b/js/src/jit/IonBuilder.h index 6b8539efbf8..c7fccd2cd7b 100644 --- a/js/src/jit/IonBuilder.h +++ b/js/src/jit/IonBuilder.h @@ -696,7 +696,7 @@ class IonBuilder bool jsop_setprop(PropertyName* name); bool jsop_delprop(PropertyName* name); bool jsop_delelem(); - bool jsop_newarray(uint32_t count); + bool jsop_newarray(uint32_t length); bool jsop_newarray_copyonwrite(); bool jsop_newobject(); bool jsop_initelem(); diff --git a/js/src/jit/MIR.cpp b/js/src/jit/MIR.cpp index fde7eb5d2a2..64e5c330ebc 100644 --- a/js/src/jit/MIR.cpp +++ b/js/src/jit/MIR.cpp @@ -4050,7 +4050,7 @@ MArrayState::MArrayState(MDefinition* arr) // This instruction is only used as a summary for bailout paths. setResultType(MIRType_Object); setRecoveredOnBailout(); - numElements_ = arr->toNewArray()->count(); + numElements_ = arr->toNewArray()->length(); } bool @@ -4090,10 +4090,10 @@ MArrayState::Copy(TempAllocator& alloc, MArrayState* state) return res; } -MNewArray::MNewArray(CompilerConstraintList* constraints, uint32_t count, MConstant* templateConst, +MNewArray::MNewArray(CompilerConstraintList* constraints, uint32_t length, MConstant* templateConst, gc::InitialHeap initialHeap, jsbytecode* pc) : MUnaryInstruction(templateConst), - count_(count), + length_(length), initialHeap_(initialHeap), convertDoubleElements_(false), pc_(pc) @@ -4115,16 +4115,16 @@ MNewArray::shouldUseVM() const return true; if (templateObject()->is()) { - MOZ_ASSERT(templateObject()->as().capacity() >= count()); + MOZ_ASSERT(templateObject()->as().capacity() >= length()); return !templateObject()->as().hasInlineElements(); } - MOZ_ASSERT(count() < NativeObject::NELEMENTS_LIMIT); + MOZ_ASSERT(length() < NativeObject::NELEMENTS_LIMIT); size_t arraySlots = gc::GetGCKindSlots(templateObject()->asTenured().getAllocKind()) - ObjectElements::VALUES_PER_HEADER; - return count() > arraySlots; + return length() > arraySlots; } bool diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index f73ecda3661..9817e8e2f51 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -2905,8 +2905,8 @@ class MNewArray public NoTypePolicy::Data { private: - // Number of space to allocate for the array. - uint32_t count_; + // Number of elements to allocate for the array. + uint32_t length_; // Heap where the array should be allocated. gc::InitialHeap initialHeap_; @@ -2916,21 +2916,21 @@ class MNewArray jsbytecode* pc_; - MNewArray(CompilerConstraintList* constraints, uint32_t count, MConstant* templateConst, + MNewArray(CompilerConstraintList* constraints, uint32_t length, MConstant* templateConst, gc::InitialHeap initialHeap, jsbytecode* pc); public: INSTRUCTION_HEADER(NewArray) static MNewArray* New(TempAllocator& alloc, CompilerConstraintList* constraints, - uint32_t count, MConstant* templateConst, + uint32_t length, MConstant* templateConst, gc::InitialHeap initialHeap, jsbytecode* pc) { - return new(alloc) MNewArray(constraints, count, templateConst, initialHeap, pc); + return new(alloc) MNewArray(constraints, length, templateConst, initialHeap, pc); } - uint32_t count() const { - return count_; + uint32_t length() const { + return length_; } JSObject* templateObject() const { diff --git a/js/src/jit/Recover.cpp b/js/src/jit/Recover.cpp index c25ab460800..b9ad6a87d3d 100644 --- a/js/src/jit/Recover.cpp +++ b/js/src/jit/Recover.cpp @@ -1207,7 +1207,7 @@ MNewArray::writeRecoverData(CompactBufferWriter& writer) const { MOZ_ASSERT(canRecoverOnBailout()); writer.writeUnsigned(uint32_t(RInstruction::Recover_NewArray)); - writer.writeUnsigned(count()); + writer.writeUnsigned(length()); return true; } diff --git a/js/src/jit/ScalarReplacement.cpp b/js/src/jit/ScalarReplacement.cpp index e1444caf694..46a76dfbc1d 100644 --- a/js/src/jit/ScalarReplacement.cpp +++ b/js/src/jit/ScalarReplacement.cpp @@ -895,7 +895,7 @@ IsArrayEscaped(MInstruction* ins) { MOZ_ASSERT(ins->type() == MIRType_Object); MOZ_ASSERT(ins->isNewArray()); - uint32_t count = ins->toNewArray()->count(); + uint32_t length = ins->toNewArray()->length(); JitSpewDef(JitSpew_Escape, "Check array\n", ins); JitSpewIndent spewIndent(JitSpew_Escape); @@ -911,7 +911,7 @@ IsArrayEscaped(MInstruction* ins) return true; } - if (count >= 16) { + if (length >= 16) { JitSpew(JitSpew_Escape, "Array has too many elements"); return true; } @@ -935,7 +935,7 @@ IsArrayEscaped(MInstruction* ins) case MDefinition::Op_Elements: { MElements *elem = def->toElements(); MOZ_ASSERT(elem->object() == ins); - if (IsElementEscaped(elem, count)) { + if (IsElementEscaped(elem, length)) { JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", elem); return true; } From 5bb1ea43027584af46b038a07f050202d70d6a6f Mon Sep 17 00:00:00 2001 From: Jeff Walden Date: Sat, 5 Sep 2015 12:51:18 +0900 Subject: [PATCH 44/78] Bug 1201869 - Part 1: Fix the maximum length of a dense array. r=arai --- js/src/vm/NativeObject.cpp | 135 ++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 71 deletions(-) diff --git a/js/src/vm/NativeObject.cpp b/js/src/vm/NativeObject.cpp index 404e400b80e..ab68fdc4389 100644 --- a/js/src/vm/NativeObject.cpp +++ b/js/src/vm/NativeObject.cpp @@ -6,6 +6,7 @@ #include "vm/NativeObject-inl.h" +#include "mozilla/Casting.h" #include "mozilla/CheckedInt.h" #include "jswatchpoint.h" @@ -654,29 +655,18 @@ NativeObject::maybeDensifySparseElements(js::ExclusiveContext* cx, HandleNativeO return DenseElementResult::Success; } -// Round up |reqAllocated| to a good size. Up to 1 Mebi (i.e. 1,048,576) the -// slot count is usually a power-of-two: +// Given a requested allocation amount (in elements) and (potentially) the +// length of an array for which elements are being allocated, compute an actual +// allocation amount (in elements). (Allocation amounts include space for an +// ObjectElements instance, so a return value of |N| implies +// |N - ObjectElements::VALUES_PER_HEADER| usable elements.) // -// 8, 16, 32, 64, ..., 256 Ki, 512 Ki, 1 Mi +// The requested/actual allocation distinction is meant to: // -// Beyond that, we use this formula: -// -// count(n+1) = Math.ceil(count(n) * 1.125) -// -// where |count(n)| is the size of the nth bucket measured in MiSlots. -// -// These counts lets us add N elements to an array in amortized O(N) time. -// Having the second class means that for bigger arrays the constant factor is -// higher, but we waste less space. -// -// There is one exception to the above rule: for the power-of-two cases, if the -// chosen capacity would be 2/3 or more of the array's length, the chosen -// capacity is adjusted (up or down) to be equal to the array's length -// (assuming length is at least as large as the required capacity). This avoids -// the allocation of excess elements which are unlikely to be needed, either in -// this resizing or a subsequent one. The 2/3 factor is chosen so that -// exceptional resizings will at most triple the capacity, as opposed to the -// usual doubling. +// * preserve amortized O(N) time to add N elements; +// * minimize the number of unused elements beyond an array's length, and +// * provide at least SLOT_CAPACITY_MIN elements no matter what (so adding +// the first several elements to small arrays only needs one allocation). // // Note: the structure and behavior of this method follow along with // UnboxedArrayObject::chooseCapacityIndex. Changes to the allocation strategy @@ -684,62 +674,65 @@ NativeObject::maybeDensifySparseElements(js::ExclusiveContext* cx, HandleNativeO /* static */ uint32_t NativeObject::goodAllocated(uint32_t reqAllocated, uint32_t length = 0) { - static const uint32_t Mebi = 1024 * 1024; + // Handle "small" requests primarily by doubling. + const uint32_t Mebi = 1 << 20; + if (reqAllocated < Mebi) { + uint32_t goodAmount = mozilla::AssertedCast(RoundUpPow2(reqAllocated)); - // This table was generated with this JavaScript code and a small amount - // subsequent reformatting: - // - // for (let n = 1, i = 0; i < 57; i++) { - // print((n * 1024 * 1024) + ', '); - // n = Math.ceil(n * 1.125); - // } - // print('0'); - // - // The final element is a sentinel value. - static const uint32_t BigBuckets[] = { - 1048576, 2097152, 3145728, 4194304, 5242880, 6291456, 7340032, 8388608, - 9437184, 11534336, 13631488, 15728640, 17825792, 20971520, 24117248, - 27262976, 31457280, 35651584, 40894464, 46137344, 52428800, 59768832, - 68157440, 77594624, 88080384, 99614720, 112197632, 126877696, - 143654912, 162529280, 183500800, 206569472, 232783872, 262144000, - 295698432, 333447168, 375390208, 422576128, 476053504, 535822336, - 602931200, 678428672, 763363328, 858783744, 966787072, 1088421888, - 1224736768, 1377828864, 1550843904, 1744830464, 1962934272, 2208301056, - 2485125120, 2796552192, 3146776576, 3541041152, 3984588800, 0 - }; - - // This code relies very much on |goodAllocated| being a uint32_t. - uint32_t goodAllocated = reqAllocated; - if (goodAllocated < Mebi) { - goodAllocated = RoundUpPow2(goodAllocated); - - // Look for the abovementioned exception. - uint32_t goodCapacity = goodAllocated - ObjectElements::VALUES_PER_HEADER; + // If |goodAmount| would be 2/3 or more of the array's length, adjust + // it (up or down) to be equal to the array's length. This avoids + // allocating excess elements that aren't likely to be needed, either + // in this resizing or a subsequent one. The 2/3 factor is chosen so + // that exceptional resizings will at most triple the capacity, as + // opposed to the usual doubling. + uint32_t goodCapacity = goodAmount - ObjectElements::VALUES_PER_HEADER; uint32_t reqCapacity = reqAllocated - ObjectElements::VALUES_PER_HEADER; if (length >= reqCapacity && goodCapacity > (length / 3) * 2) - goodAllocated = length + ObjectElements::VALUES_PER_HEADER; + goodAmount = length + ObjectElements::VALUES_PER_HEADER; - if (goodAllocated < SLOT_CAPACITY_MIN) - goodAllocated = SLOT_CAPACITY_MIN; + if (goodAmount < SLOT_CAPACITY_MIN) + goodAmount = SLOT_CAPACITY_MIN; - } else { - uint32_t i = 0; - while (true) { - uint32_t b = BigBuckets[i++]; - if (b >= goodAllocated) { - // Found the first bucket greater than or equal to - // |goodAllocated|. - goodAllocated = b; - break; - } else if (b == 0) { - // Hit the end; return the maximum possible goodAllocated. - goodAllocated = 0xffffffff; - break; - } - } + return goodAmount; } - return goodAllocated; + // The almost-doubling above wastes a lot of space for larger bucket sizes. + // For large amounts, switch to bucket sizes that obey this formula: + // + // count(n+1) = Math.ceil(count(n) * 1.125) + // + // where |count(n)| is the size of the nth bucket, measured in 2**20 slots. + // These bucket sizes still preserve amortized O(N) time to add N elements, + // just with a larger constant factor. + // + // The bucket size table below was generated with this JavaScript (and + // manual reformatting): + // + // for (let n = 1, i = 0; i < 34; i++) { + // print('0x' + (n * (1 << 20)).toString(16) + ', '); + // n = Math.ceil(n * 1.125); + // } + // print('NELEMENTS_LIMIT - 1'); + // + // Dense array elements can't exceed |NELEMENTS_LIMIT|, so + // |NELEMENTS_LIMIT - 1| is the biggest allowed length. + static const uint32_t BigBuckets[] = { + 0x100000, 0x200000, 0x300000, 0x400000, 0x500000, 0x600000, 0x700000, + 0x800000, 0x900000, 0xb00000, 0xd00000, 0xf00000, 0x1100000, 0x1400000, + 0x1700000, 0x1a00000, 0x1e00000, 0x2200000, 0x2700000, 0x2c00000, + 0x3200000, 0x3900000, 0x4100000, 0x4a00000, 0x5400000, 0x5f00000, + 0x6b00000, 0x7900000, 0x8900000, 0x9b00000, 0xaf00000, 0xc500000, + 0xde00000, 0xfa00000, NELEMENTS_LIMIT - 1 + }; + + // Pick the first bucket that'll fit |reqAllocated|. + for (uint32_t b : BigBuckets) { + if (b >= reqAllocated) + return b; + } + + // Otherwise, return the maximum bucket size. + return NELEMENTS_LIMIT - 1; } bool @@ -779,7 +772,6 @@ NativeObject::growElements(ExclusiveContext* cx, uint32_t reqCapacity) uint32_t newCapacity = newAllocated - ObjectElements::VALUES_PER_HEADER; MOZ_ASSERT(newCapacity > oldCapacity && newCapacity >= reqCapacity); - // Don't let nelements get close to wrapping around uint32_t. if (newCapacity >= NELEMENTS_LIMIT) return false; @@ -828,6 +820,7 @@ NativeObject::shrinkElements(ExclusiveContext* cx, uint32_t reqCapacity) MOZ_ASSERT(newAllocated > ObjectElements::VALUES_PER_HEADER); uint32_t newCapacity = newAllocated - ObjectElements::VALUES_PER_HEADER; + MOZ_ASSERT(newCapacity < NELEMENTS_LIMIT); HeapSlot* oldHeaderSlots = reinterpret_cast(getElementsHeader()); HeapSlot* newHeaderSlots = ReallocateObjectBuffer(cx, this, oldHeaderSlots, From 5cd0d0b6d1398834da96d1cb4a524ed21e88b906 Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 5 Sep 2015 13:24:02 +0900 Subject: [PATCH 45/78] Bug 1201869 - Part 2: Make an array sparse when exceeds the limit of dense array length. r=Waldo --- js/src/frontend/Parser.cpp | 2 +- .../tests/arrays/dense-from-sparse.js | 40 +++++++++++++ js/src/jit/CodeGenerator.cpp | 4 +- js/src/jit/MCallOptimize.cpp | 3 +- js/src/jit/MIR.cpp | 2 +- js/src/jit/RangeAnalysis.cpp | 2 +- js/src/jsgc.h | 9 ++- js/src/jsobjinlines.h | 14 ++--- js/src/vm/NativeObject.cpp | 58 +++++++++---------- js/src/vm/NativeObject.h | 23 ++++++-- 10 files changed, 107 insertions(+), 50 deletions(-) create mode 100644 js/src/jit-test/tests/arrays/dense-from-sparse.js diff --git a/js/src/frontend/Parser.cpp b/js/src/frontend/Parser.cpp index 7e88e1cd3e5..44b3a7d1b73 100644 --- a/js/src/frontend/Parser.cpp +++ b/js/src/frontend/Parser.cpp @@ -8767,7 +8767,7 @@ Parser::arrayInitializer(YieldHandling yieldHandling) uint32_t index = 0; TokenStream::Modifier modifier = TokenStream::Operand; for (; ; index++) { - if (index == NativeObject::NELEMENTS_LIMIT) { + if (index >= NativeObject::MAX_DENSE_ELEMENTS_COUNT) { report(ParseError, false, null(), JSMSG_ARRAY_INIT_TOO_BIG); return null(); } diff --git a/js/src/jit-test/tests/arrays/dense-from-sparse.js b/js/src/jit-test/tests/arrays/dense-from-sparse.js new file mode 100644 index 00000000000..efe56620da2 --- /dev/null +++ b/js/src/jit-test/tests/arrays/dense-from-sparse.js @@ -0,0 +1,40 @@ +// |jit-test| allow-oom +// Appending elements to a dense array should make the array sparse when the +// length exceeds the limit. + +function test() { + const MAX_DENSE_ELEMENTS_ALLOCATION = (1 << 28) - 1; + const VALUES_PER_HEADER = 2; + const MAX_DENSE_ELEMENTS_COUNT = MAX_DENSE_ELEMENTS_ALLOCATION - VALUES_PER_HEADER; + const SPARSE_DENSITY_RATIO = 8; + const MIN_DENSE = MAX_DENSE_ELEMENTS_COUNT / SPARSE_DENSITY_RATIO; + const MARGIN = 16; + + let a = []; + // Fill the beginning of array to make it keep dense until length exceeds + // MAX_DENSE_ELEMENTS_COUNT. + for (let i = 0; i < MIN_DENSE; i++) + a[i] = i; + + // Skip from MIN_DENSE to MAX_DENSE_ELEMENTS_COUNT - MARGIN, to reduce the + // time taken by test. + + // Fill the ending of array to make it sparse at MAX_DENSE_ELEMENTS_COUNT. + for (let i = MAX_DENSE_ELEMENTS_COUNT - MARGIN; i < MAX_DENSE_ELEMENTS_COUNT + MARGIN; i++) + a[i] = i; + + // Make sure the last element is defined. + assertEq(a.length, MAX_DENSE_ELEMENTS_COUNT + MARGIN); + assertEq(a[a.length - 1], MAX_DENSE_ELEMENTS_COUNT + MARGIN - 1); + + // Make sure elements around MAX_DENSE_ELEMENTS_COUNT are also defined. + assertEq(a[MAX_DENSE_ELEMENTS_COUNT - 1], MAX_DENSE_ELEMENTS_COUNT - 1); + assertEq(a[MAX_DENSE_ELEMENTS_COUNT], MAX_DENSE_ELEMENTS_COUNT); + assertEq(a[MAX_DENSE_ELEMENTS_COUNT + 1], MAX_DENSE_ELEMENTS_COUNT + 1); +} + +var config = getBuildConfiguration(); +// Takes too long time on debug build. +if (!config.debug) { + test(); +} diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 6ba71b5dbfe..b4e80fd15a3 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -4303,7 +4303,7 @@ CodeGenerator::visitNewArray(LNewArray* lir) JSObject* templateObject = lir->mir()->templateObject(); DebugOnly length = lir->mir()->length(); - MOZ_ASSERT(length < NativeObject::NELEMENTS_LIMIT); + MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT); if (lir->mir()->shouldUseVM()) { visitNewArrayCallVM(lir); @@ -6946,7 +6946,7 @@ CodeGenerator::visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool) key, &callStub); // Update initialized length. The capacity guard above ensures this won't overflow, - // due to NELEMENTS_LIMIT. + // due to MAX_DENSE_ELEMENTS_COUNT. masm.bumpKey(&key, 1); masm.storeKey(key, Address(elements, ObjectElements::offsetOfInitializedLength())); diff --git a/js/src/jit/MCallOptimize.cpp b/js/src/jit/MCallOptimize.cpp index 92a2cdc66b4..d1dd269ea9f 100644 --- a/js/src/jit/MCallOptimize.cpp +++ b/js/src/jit/MCallOptimize.cpp @@ -442,8 +442,9 @@ IonBuilder::inlineArray(CallInfo& callInfo) // Negative lengths generate a RangeError, unhandled by the inline path. initLength = arg->constantValue().toInt32(); - if (initLength >= NativeObject::NELEMENTS_LIMIT) + if (initLength > NativeObject::MAX_DENSE_ELEMENTS_COUNT) return InliningStatus_NotInlined; + MOZ_ASSERT(initLength <= INT32_MAX); // Make sure initLength matches the template object's length. This is // not guaranteed to be the case, for instance if we're inlining the diff --git a/js/src/jit/MIR.cpp b/js/src/jit/MIR.cpp index 64e5c330ebc..641e7c94c88 100644 --- a/js/src/jit/MIR.cpp +++ b/js/src/jit/MIR.cpp @@ -4119,7 +4119,7 @@ MNewArray::shouldUseVM() const return !templateObject()->as().hasInlineElements(); } - MOZ_ASSERT(length() < NativeObject::NELEMENTS_LIMIT); + MOZ_ASSERT(length() <= NativeObject::MAX_DENSE_ELEMENTS_COUNT); size_t arraySlots = gc::GetGCKindSlots(templateObject()->asTenured().getAllocKind()) - ObjectElements::VALUES_PER_HEADER; diff --git a/js/src/jit/RangeAnalysis.cpp b/js/src/jit/RangeAnalysis.cpp index 25dda5d0de1..66db6f78cc3 100644 --- a/js/src/jit/RangeAnalysis.cpp +++ b/js/src/jit/RangeAnalysis.cpp @@ -1734,7 +1734,7 @@ MArrayLength::computeRange(TempAllocator& alloc) void MInitializedLength::computeRange(TempAllocator& alloc) { - setRange(Range::NewUInt32Range(alloc, 0, NativeObject::NELEMENTS_LIMIT)); + setRange(Range::NewUInt32Range(alloc, 0, NativeObject::MAX_DENSE_ELEMENTS_COUNT)); } void diff --git a/js/src/jsgc.h b/js/src/jsgc.h index 5cf8e9eca2a..7115d84bebb 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -172,7 +172,7 @@ GetGCObjectKind(size_t numSlots) /* As for GetGCObjectKind, but for dense array allocation. */ static inline AllocKind -GetGCArrayKind(size_t numSlots) +GetGCArrayKind(size_t numElements) { /* * Dense arrays can use their fixed slots to hold their elements array @@ -181,9 +181,12 @@ GetGCArrayKind(size_t numSlots) * unused. */ JS_STATIC_ASSERT(ObjectElements::VALUES_PER_HEADER == 2); - if (numSlots > NativeObject::NELEMENTS_LIMIT || numSlots + 2 >= SLOTS_TO_THING_KIND_LIMIT) + if (numElements > NativeObject::MAX_DENSE_ELEMENTS_COUNT || + numElements + ObjectElements::VALUES_PER_HEADER >= SLOTS_TO_THING_KIND_LIMIT) + { return AllocKind::OBJECT2; - return slotsToThingKind[numSlots + 2]; + } + return slotsToThingKind[numElements + ObjectElements::VALUES_PER_HEADER]; } static inline AllocKind diff --git a/js/src/jsobjinlines.h b/js/src/jsobjinlines.h index 3dfadaf2cdd..f4fba5d4aa1 100644 --- a/js/src/jsobjinlines.h +++ b/js/src/jsobjinlines.h @@ -791,23 +791,23 @@ NewObjectWithGroup(ExclusiveContext* cx, HandleObjectGroup group, } /* - * As for gc::GetGCObjectKind, where numSlots is a guess at the final size of + * As for gc::GetGCObjectKind, where numElements is a guess at the final size of * the object, zero if the final size is unknown. This should only be used for * objects that do not require any fixed slots. */ static inline gc::AllocKind -GuessObjectGCKind(size_t numSlots) +GuessObjectGCKind(size_t numElements) { - if (numSlots) - return gc::GetGCObjectKind(numSlots); + if (numElements) + return gc::GetGCObjectKind(numElements); return gc::AllocKind::OBJECT4; } static inline gc::AllocKind -GuessArrayGCKind(size_t numSlots) +GuessArrayGCKind(size_t numElements) { - if (numSlots) - return gc::GetGCArrayKind(numSlots); + if (numElements) + return gc::GetGCArrayKind(numElements); return gc::AllocKind::OBJECT8; } diff --git a/js/src/vm/NativeObject.cpp b/js/src/vm/NativeObject.cpp index ab68fdc4389..06baa7e1c8e 100644 --- a/js/src/vm/NativeObject.cpp +++ b/js/src/vm/NativeObject.cpp @@ -6,8 +6,8 @@ #include "vm/NativeObject-inl.h" +#include "mozilla/ArrayUtils.h" #include "mozilla/Casting.h" -#include "mozilla/CheckedInt.h" #include "jswatchpoint.h" @@ -26,6 +26,7 @@ using namespace js; using JS::GenericNaN; +using mozilla::ArrayLength; using mozilla::DebugOnly; using mozilla::PodCopy; using mozilla::RoundUpPow2; @@ -398,7 +399,7 @@ NativeObject::growSlots(ExclusiveContext* cx, uint32_t oldCount, uint32_t newCou * throttled well before the slot capacity can overflow. */ NativeObject::slotsSizeMustNotOverflow(); - MOZ_ASSERT(newCount < NELEMENTS_LIMIT); + MOZ_ASSERT(newCount <= MAX_SLOTS_COUNT); if (!oldCount) { MOZ_ASSERT(!slots_); @@ -520,7 +521,7 @@ NativeObject::willBeSparseElements(uint32_t requiredCapacity, uint32_t newElemen uint32_t cap = getDenseCapacity(); MOZ_ASSERT(requiredCapacity >= cap); - if (requiredCapacity >= NELEMENTS_LIMIT) + if (requiredCapacity > MAX_DENSE_ELEMENTS_COUNT) return true; uint32_t minimalDenseCount = requiredCapacity / SPARSE_DENSITY_RATIO; @@ -594,7 +595,7 @@ NativeObject::maybeDensifySparseElements(js::ExclusiveContext* cx, HandleNativeO if (numDenseElements * SPARSE_DENSITY_RATIO < newInitializedLength) return DenseElementResult::Incomplete; - if (newInitializedLength >= NELEMENTS_LIMIT) + if (newInitializedLength > MAX_DENSE_ELEMENTS_COUNT) return DenseElementResult::Incomplete; /* @@ -672,7 +673,7 @@ NativeObject::maybeDensifySparseElements(js::ExclusiveContext* cx, HandleNativeO // UnboxedArrayObject::chooseCapacityIndex. Changes to the allocation strategy // in one should generally be matched by the other. /* static */ uint32_t -NativeObject::goodAllocated(uint32_t reqAllocated, uint32_t length = 0) +NativeObject::goodElementsAllocationAmount(uint32_t reqAllocated, uint32_t length = 0) { // Handle "small" requests primarily by doubling. const uint32_t Mebi = 1 << 20; @@ -712,18 +713,19 @@ NativeObject::goodAllocated(uint32_t reqAllocated, uint32_t length = 0) // print('0x' + (n * (1 << 20)).toString(16) + ', '); // n = Math.ceil(n * 1.125); // } - // print('NELEMENTS_LIMIT - 1'); + // print('MAX_DENSE_ELEMENTS_ALLOCATION'); // - // Dense array elements can't exceed |NELEMENTS_LIMIT|, so - // |NELEMENTS_LIMIT - 1| is the biggest allowed length. + // Dense array elements can't exceed |MAX_DENSE_ELEMENTS_ALLOCATION|, so + // |MAX_DENSE_ELEMENTS_ALLOCATION| is the biggest allowed length. static const uint32_t BigBuckets[] = { 0x100000, 0x200000, 0x300000, 0x400000, 0x500000, 0x600000, 0x700000, 0x800000, 0x900000, 0xb00000, 0xd00000, 0xf00000, 0x1100000, 0x1400000, 0x1700000, 0x1a00000, 0x1e00000, 0x2200000, 0x2700000, 0x2c00000, 0x3200000, 0x3900000, 0x4100000, 0x4a00000, 0x5400000, 0x5f00000, 0x6b00000, 0x7900000, 0x8900000, 0x9b00000, 0xaf00000, 0xc500000, - 0xde00000, 0xfa00000, NELEMENTS_LIMIT - 1 + 0xde00000, 0xfa00000, MAX_DENSE_ELEMENTS_ALLOCATION }; + MOZ_ASSERT(BigBuckets[ArrayLength(BigBuckets) - 2] <= MAX_DENSE_ELEMENTS_ALLOCATION); // Pick the first bucket that'll fit |reqAllocated|. for (uint32_t b : BigBuckets) { @@ -732,12 +734,17 @@ NativeObject::goodAllocated(uint32_t reqAllocated, uint32_t length = 0) } // Otherwise, return the maximum bucket size. - return NELEMENTS_LIMIT - 1; + return MAX_DENSE_ELEMENTS_ALLOCATION; } bool NativeObject::growElements(ExclusiveContext* cx, uint32_t reqCapacity) { + if (reqCapacity > MAX_DENSE_ELEMENTS_COUNT) { + ReportOutOfMemory(cx); + return false; + } + MOZ_ASSERT(nonProxyIsExtensible()); MOZ_ASSERT(canHaveNonEmptyElements()); if (denseElementsAreCopyOnWrite()) @@ -746,17 +753,9 @@ NativeObject::growElements(ExclusiveContext* cx, uint32_t reqCapacity) uint32_t oldCapacity = getDenseCapacity(); MOZ_ASSERT(oldCapacity < reqCapacity); - using mozilla::CheckedInt; - - CheckedInt checkedOldAllocated = - CheckedInt(oldCapacity) + ObjectElements::VALUES_PER_HEADER; - CheckedInt checkedReqAllocated = - CheckedInt(reqCapacity) + ObjectElements::VALUES_PER_HEADER; - if (!checkedOldAllocated.isValid() || !checkedReqAllocated.isValid()) - return false; - - uint32_t reqAllocated = checkedReqAllocated.value(); - uint32_t oldAllocated = checkedOldAllocated.value(); + uint32_t reqAllocated = reqCapacity + ObjectElements::VALUES_PER_HEADER; + uint32_t oldAllocated = oldCapacity + ObjectElements::VALUES_PER_HEADER; + MOZ_ASSERT(oldAllocated <= MAX_DENSE_ELEMENTS_ALLOCATION); uint32_t newAllocated; if (is() && !as().lengthIsWritable()) { @@ -766,14 +765,15 @@ NativeObject::growElements(ExclusiveContext* cx, uint32_t reqCapacity) // enforces this requirement. newAllocated = reqAllocated; } else { - newAllocated = goodAllocated(reqAllocated, getElementsHeader()->length); + newAllocated = goodElementsAllocationAmount(reqAllocated, getElementsHeader()->length); } uint32_t newCapacity = newAllocated - ObjectElements::VALUES_PER_HEADER; MOZ_ASSERT(newCapacity > oldCapacity && newCapacity >= reqCapacity); - if (newCapacity >= NELEMENTS_LIMIT) - return false; + // If newCapacity exceeds MAX_DENSE_ELEMENTS_COUNT, the array should become + // sparse. + MOZ_ASSERT(newCapacity <= MAX_DENSE_ELEMENTS_COUNT); uint32_t initlen = getDenseInitializedLength(); @@ -814,13 +814,13 @@ NativeObject::shrinkElements(ExclusiveContext* cx, uint32_t reqCapacity) uint32_t oldAllocated = oldCapacity + ObjectElements::VALUES_PER_HEADER; uint32_t reqAllocated = reqCapacity + ObjectElements::VALUES_PER_HEADER; - uint32_t newAllocated = goodAllocated(reqAllocated); + uint32_t newAllocated = goodElementsAllocationAmount(reqAllocated); if (newAllocated == oldAllocated) return; // Leave elements at its old size. MOZ_ASSERT(newAllocated > ObjectElements::VALUES_PER_HEADER); uint32_t newCapacity = newAllocated - ObjectElements::VALUES_PER_HEADER; - MOZ_ASSERT(newCapacity < NELEMENTS_LIMIT); + MOZ_ASSERT(newCapacity <= MAX_DENSE_ELEMENTS_COUNT); HeapSlot* oldHeaderSlots = reinterpret_cast(getElementsHeader()); HeapSlot* newHeaderSlots = ReallocateObjectBuffer(cx, this, oldHeaderSlots, @@ -845,12 +845,12 @@ NativeObject::CopyElementsForWrite(ExclusiveContext* cx, NativeObject* obj) uint32_t initlen = obj->getDenseInitializedLength(); uint32_t allocated = initlen + ObjectElements::VALUES_PER_HEADER; - uint32_t newAllocated = goodAllocated(allocated); + uint32_t newAllocated = goodElementsAllocationAmount(allocated); uint32_t newCapacity = newAllocated - ObjectElements::VALUES_PER_HEADER; - if (newCapacity >= NELEMENTS_LIMIT) - return false; + // COPY_ON_WRITE flags is set only if obj is a dense array. + MOZ_ASSERT(newCapacity <= MAX_DENSE_ELEMENTS_COUNT); JSObject::writeBarrierPre(obj->getElementsHeader()->ownerObject()); diff --git a/js/src/vm/NativeObject.h b/js/src/vm/NativeObject.h index 3eb575ecf1e..413eca55631 100644 --- a/js/src/vm/NativeObject.h +++ b/js/src/vm/NativeObject.h @@ -556,8 +556,13 @@ class NativeObject : public JSObject bool shadowingShapeChange(ExclusiveContext* cx, const Shape& shape); bool clearFlag(ExclusiveContext* cx, BaseShape::Flag flag); + // The maximum number of slots in an object. + // |MAX_SLOTS_COUNT * sizeof(JS::Value)| shouldn't overflow + // int32_t (see slotsSizeMustNotOverflow). + static const uint32_t MAX_SLOTS_COUNT = (1 << 28) - 1; + static void slotsSizeMustNotOverflow() { - static_assert((NativeObject::NELEMENTS_LIMIT - 1) <= INT32_MAX / sizeof(JS::Value), + static_assert(NativeObject::MAX_SLOTS_COUNT <= INT32_MAX / sizeof(JS::Value), "every caller of this method requires that a slot " "number (or slot count) count multiplied by " "sizeof(Value) can't overflow uint32_t (and sometimes " @@ -882,11 +887,19 @@ class NativeObject : public JSObject /* Elements accessors. */ - /* Upper bound on the number of elements in an object. */ - static const uint32_t NELEMENTS_LIMIT = JS_BIT(28); + // The maximum size, in sizeof(Value), of the allocation used for an + // object's dense elements. (This includes space used to store an + // ObjectElements instance.) + // |MAX_DENSE_ELEMENTS_ALLOCATION * sizeof(JS::Value)| shouldn't overflow + // int32_t (see elementsSizeMustNotOverflow). + static const uint32_t MAX_DENSE_ELEMENTS_ALLOCATION = (1 << 28) - 1; + + // The maximum number of usable dense elements in an object. + static const uint32_t MAX_DENSE_ELEMENTS_COUNT = + MAX_DENSE_ELEMENTS_ALLOCATION - ObjectElements::VALUES_PER_HEADER; static void elementsSizeMustNotOverflow() { - static_assert((NativeObject::NELEMENTS_LIMIT - 1) <= INT32_MAX / sizeof(JS::Value), + static_assert(NativeObject::MAX_DENSE_ELEMENTS_COUNT <= INT32_MAX / sizeof(JS::Value), "every caller of this method require that an element " "count multiplied by sizeof(Value) can't overflow " "uint32_t (and sometimes int32_t ,too)"); @@ -904,7 +917,7 @@ class NativeObject : public JSObject return true; } - static uint32_t goodAllocated(uint32_t n, uint32_t length); + static uint32_t goodElementsAllocationAmount(uint32_t n, uint32_t length); bool growElements(ExclusiveContext* cx, uint32_t newcap); void shrinkElements(ExclusiveContext* cx, uint32_t cap); void setDynamicElements(ObjectElements* header) { From 656eb59dc51eefa7a4677e27fc4f70cd83f1115a Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 29 Aug 2015 17:56:38 +0900 Subject: [PATCH 46/78] Bug 1199345 - Extend JSOP_NEWARRAY/JSOP_INITELEM_ARRAY/JSOP_SPREADCALLARRAY operand to uint32. r=Waldo --- js/src/frontend/BytecodeEmitter.cpp | 38 ++++++++++++++--------------- js/src/jit/BaselineCompiler.cpp | 17 ++++++++++--- js/src/jit/BaselineIC.cpp | 6 ++++- js/src/jit/IonBuilder.cpp | 7 +++--- js/src/vm/Interpreter.cpp | 5 ++-- js/src/vm/Opcodes.h | 13 +++++----- js/src/vm/Xdr.h | 2 +- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/js/src/frontend/BytecodeEmitter.cpp b/js/src/frontend/BytecodeEmitter.cpp index 5eb6f49490f..b72760aee2f 100644 --- a/js/src/frontend/BytecodeEmitter.cpp +++ b/js/src/frontend/BytecodeEmitter.cpp @@ -3889,13 +3889,8 @@ BytecodeEmitter::emitDestructuringOpsArrayHelper(ParseNode* pattern, VarEmitOpti if (elem->isKind(PNK_SPREAD)) { /* Create a new array with the rest of the iterator */ - ptrdiff_t off; - if (!emitN(JSOP_NEWARRAY, 3, &off)) // ... OBJ? ITER ARRAY + if (!emitUint32Operand(JSOP_NEWARRAY, 0)) // ... OBJ? ITER ARRAY return false; - checkTypeSet(JSOP_NEWARRAY); - jsbytecode* pc = code(off); - SET_UINT24(pc, 0); - if (!emitNumberOp(0)) // ... OBJ? ITER ARRAY INDEX return false; if (!emitSpread()) // ... OBJ? ARRAY INDEX @@ -7214,29 +7209,35 @@ BytecodeEmitter::emitArray(ParseNode* pn, uint32_t count, JSOp op) */ MOZ_ASSERT(op == JSOP_NEWARRAY || op == JSOP_SPREADCALLARRAY); - int32_t nspread = 0; + uint32_t nspread = 0; for (ParseNode* elt = pn; elt; elt = elt->pn_next) { if (elt->isKind(PNK_SPREAD)) nspread++; } - ptrdiff_t off; - if (!emitN(op, 3, &off)) // ARRAY - return false; - checkTypeSet(op); - jsbytecode* pc = code(off); + // Array literal's length is limited to NELEMENTS_LIMIT in parser. + static_assert(NativeObject::MAX_DENSE_ELEMENTS_COUNT <= INT32_MAX, + "array literals' maximum length must not exceed limits " + "required by BaselineCompiler::emit_JSOP_NEWARRAY, " + "BaselineCompiler::emit_JSOP_INITELEM_ARRAY, " + "and DoSetElemFallback's handling of JSOP_INITELEM_ARRAY"); + MOZ_ASSERT(count >= nspread); + MOZ_ASSERT(count <= NativeObject::MAX_DENSE_ELEMENTS_COUNT, + "the parser must throw an error if the array exceeds maximum " + "length"); // For arrays with spread, this is a very pessimistic allocation, the // minimum possible final size. - SET_UINT24(pc, count - nspread); + if (!emitUint32Operand(op, count - nspread)) // ARRAY + return false; ParseNode* pn2 = pn; - jsatomid atomIndex; + uint32_t index; bool afterSpread = false; - for (atomIndex = 0; pn2; atomIndex++, pn2 = pn2->pn_next) { + for (index = 0; pn2; index++, pn2 = pn2->pn_next) { if (!afterSpread && pn2->isKind(PNK_SPREAD)) { afterSpread = true; - if (!emitNumberOp(atomIndex)) // ARRAY INDEX + if (!emitNumberOp(index)) // ARRAY INDEX return false; } if (!updateSourceCoordNotes(pn2->pn_pos.begin)) @@ -7262,12 +7263,11 @@ BytecodeEmitter::emitArray(ParseNode* pn, uint32_t count, JSOp op) if (!emit1(JSOP_INITELEM_INC)) return false; } else { - if (!emitN(JSOP_INITELEM_ARRAY, 3, &off)) + if (!emitUint32Operand(JSOP_INITELEM_ARRAY, index)) return false; - SET_UINT24(code(off), atomIndex); } } - MOZ_ASSERT(atomIndex == count); + MOZ_ASSERT(index == count); if (afterSpread) { if (!emit1(JSOP_POP)) // ARRAY return false; diff --git a/js/src/jit/BaselineCompiler.cpp b/js/src/jit/BaselineCompiler.cpp index 8b49a5b643a..8128d097c54 100644 --- a/js/src/jit/BaselineCompiler.cpp +++ b/js/src/jit/BaselineCompiler.cpp @@ -6,6 +6,7 @@ #include "jit/BaselineCompiler.h" +#include "mozilla/Casting.h" #include "mozilla/UniquePtr.h" #include "jit/BaselineIC.h" @@ -32,6 +33,8 @@ using namespace js; using namespace js::jit; +using mozilla::AssertedCast; + BaselineCompiler::BaselineCompiler(JSContext* cx, TempAllocator& alloc, JSScript* script) : BaselineCompilerSpecific(cx, alloc, script), yieldOffsets_(cx), @@ -1792,10 +1795,13 @@ BaselineCompiler::emit_JSOP_NEWARRAY() { frame.syncStack(0); - uint32_t length = GET_UINT24(pc); + uint32_t length = GET_UINT32(pc); + MOZ_ASSERT(length <= INT32_MAX, + "the bytecode emitter must fail to compile code that would " + "produce JSOP_NEWARRAY with a length exceeding int32_t range"); // Pass length in R0. - masm.move32(Imm32(length), R0.scratchReg()); + masm.move32(Imm32(AssertedCast(length)), R0.scratchReg()); ObjectGroup* group = ObjectGroup::allocationSiteGroup(cx, script, pc, JSProto_Array); if (!group) @@ -1849,7 +1855,12 @@ BaselineCompiler::emit_JSOP_INITELEM_ARRAY() // Load object in R0, index in R1. masm.loadValue(frame.addressOfStackValue(frame.peek(-2)), R0); - masm.moveValue(Int32Value(GET_UINT24(pc)), R1); + uint32_t index = GET_UINT32(pc); + MOZ_ASSERT(index <= INT32_MAX, + "the bytecode emitter must fail to compile code that would " + "produce JSOP_INITELEM_ARRAY with a length exceeding " + "int32_t range"); + masm.moveValue(Int32Value(AssertedCast(index)), R1); // Call IC. ICSetElem_Fallback::Compiler stubCompiler(cx); diff --git a/js/src/jit/BaselineIC.cpp b/js/src/jit/BaselineIC.cpp index de214f09dd6..950abffed20 100644 --- a/js/src/jit/BaselineIC.cpp +++ b/js/src/jit/BaselineIC.cpp @@ -3689,7 +3689,11 @@ DoSetElemFallback(JSContext* cx, BaselineFrame* frame, ICSetElem_Fallback* stub_ if (!InitElemOperation(cx, obj, index, rhs)) return false; } else if (op == JSOP_INITELEM_ARRAY) { - MOZ_ASSERT(uint32_t(index.toInt32()) == GET_UINT24(pc)); + MOZ_ASSERT(uint32_t(index.toInt32()) <= INT32_MAX, + "the bytecode emitter must fail to compile code that would " + "produce JSOP_INITELEM_ARRAY with an index exceeding " + "int32_t range"); + MOZ_ASSERT(uint32_t(index.toInt32()) == GET_UINT32(pc)); if (!InitArrayElemOperation(cx, pc, obj, index.toInt32(), rhs)) return false; } else if (op == JSOP_INITELEM_INC) { diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index 54f2b870026..519564e6b69 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -1794,7 +1794,7 @@ IonBuilder::inspectOpcode(JSOp op) return jsop_newobject(); case JSOP_NEWARRAY: - return jsop_newarray(GET_UINT24(pc)); + return jsop_newarray(GET_UINT32(pc)); case JSOP_NEWARRAY_COPYONWRITE: return jsop_newarray_copyonwrite(); @@ -7052,13 +7052,14 @@ IonBuilder::jsop_initelem_array() } } + uint32_t index = GET_UINT32(pc); if (needStub) { - MCallInitElementArray* store = MCallInitElementArray::New(alloc(), obj, GET_UINT24(pc), value); + MCallInitElementArray* store = MCallInitElementArray::New(alloc(), obj, index, value); current->add(store); return resumeAfter(store); } - return initializeArrayElement(obj, GET_UINT24(pc), value, unboxedType, /* addResumePoint = */ true); + return initializeArrayElement(obj, index, value, unboxedType, /* addResumePoint = */ true); } bool diff --git a/js/src/vm/Interpreter.cpp b/js/src/vm/Interpreter.cpp index a0057a5831d..5a32a10e138 100644 --- a/js/src/vm/Interpreter.cpp +++ b/js/src/vm/Interpreter.cpp @@ -3609,7 +3609,8 @@ END_CASE(JSOP_NEWINIT) CASE(JSOP_NEWARRAY) CASE(JSOP_SPREADCALLARRAY) { - JSObject* obj = NewArrayOperation(cx, script, REGS.pc, GET_UINT24(REGS.pc)); + uint32_t length = GET_UINT32(REGS.pc); + JSObject* obj = NewArrayOperation(cx, script, REGS.pc, length); if (!obj) goto error; PUSH_OBJECT(*obj); @@ -3705,7 +3706,7 @@ CASE(JSOP_INITELEM_ARRAY) ReservedRooted obj(&rootObject0, ®S.sp[-2].toObject()); - uint32_t index = GET_UINT24(REGS.pc); + uint32_t index = GET_UINT32(REGS.pc); if (!InitArrayElemOperation(cx, REGS.pc, obj, index, val)) goto error; diff --git a/js/src/vm/Opcodes.h b/js/src/vm/Opcodes.h index 9f88e22b8df..fc5d0e0a791 100644 --- a/js/src/vm/Opcodes.h +++ b/js/src/vm/Opcodes.h @@ -847,10 +847,10 @@ * This opcode takes the final length, which is preallocated. * Category: Literals * Type: Array - * Operands: uint24_t length + * Operands: uint32_t length * Stack: => obj */ \ - macro(JSOP_NEWARRAY, 90, "newarray", NULL, 4, 0, 1, JOF_UINT24) \ + macro(JSOP_NEWARRAY, 90, "newarray", NULL, 5, 0, 1, JOF_UINT32) \ /* * Pushes newly created object onto the stack. * @@ -920,10 +920,10 @@ * property of 'obj' as 'val', pushes 'obj' onto the stack. * Category: Literals * Type: Array - * Operands: uint24_t index + * Operands: uint32_t index * Stack: obj, val => obj */ \ - macro(JSOP_INITELEM_ARRAY,96, "initelem_array", NULL, 4, 2, 1, JOF_UINT24|JOF_ELEM|JOF_SET|JOF_DETECTING) \ + macro(JSOP_INITELEM_ARRAY,96, "initelem_array", NULL, 5, 2, 1, JOF_UINT32|JOF_ELEM|JOF_SET|JOF_DETECTING) \ \ /* * Initialize a getter in an object literal. @@ -1277,13 +1277,12 @@ * the same semantics as JSOP_NEWARRAY, but is distinguished to avoid * using unboxed arrays in spread calls, which would make compiling spread * calls in baseline more complex. - * * Category: Literals * Type: Array - * Operands: uint24_t length + * Operands: uint32_t length * Stack: => obj */ \ - macro(JSOP_SPREADCALLARRAY, 126, "spreadcallarray", NULL, 4, 0, 1, JOF_UINT24) \ + macro(JSOP_SPREADCALLARRAY, 126, "spreadcallarray", NULL, 5, 0, 1, JOF_UINT32) \ \ /* * Defines the given function on the current scope. diff --git a/js/src/vm/Xdr.h b/js/src/vm/Xdr.h index fb0eba1e3a3..d8730ac2298 100644 --- a/js/src/vm/Xdr.h +++ b/js/src/vm/Xdr.h @@ -29,7 +29,7 @@ namespace js { * * https://developer.mozilla.org/en-US/docs/SpiderMonkey/Internals/Bytecode */ -static const uint32_t XDR_BYTECODE_VERSION_SUBTRAHEND = 309; +static const uint32_t XDR_BYTECODE_VERSION_SUBTRAHEND = 310; static const uint32_t XDR_BYTECODE_VERSION = uint32_t(0xb973c0de - XDR_BYTECODE_VERSION_SUBTRAHEND); From cb05cd4e6c952ae34993d254cbb74bad7a691bb8 Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Thu, 24 Sep 2015 12:30:24 +0100 Subject: [PATCH 47/78] Bug 1193519 followup - annotate test for antialiasing fuzz on OS X. --- layout/reftests/writing-mode/reftest.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layout/reftests/writing-mode/reftest.list b/layout/reftests/writing-mode/reftest.list index fdd0a45d081..39aeb0cc9e3 100644 --- a/layout/reftests/writing-mode/reftest.list +++ b/layout/reftests/writing-mode/reftest.list @@ -158,7 +158,7 @@ test-pref(dom.meta-viewport.enabled,true) test-pref(font.size.inflation.emPerLin == 1193519-sideways-lr-2.html 1193519-sideways-lr-2-ref.html == 1193519-sideways-lr-3.html 1193519-sideways-lr-3-ref.html == 1193519-sideways-lr-4.html 1193519-sideways-lr-4-ref.html -fuzzy-if(gtkWidget||B2G,255,6) == 1193519-sideways-lr-decoration-1.html 1193519-sideways-lr-decoration-1-ref.html +fuzzy-if(gtkWidget||B2G,255,6) fuzzy-if(cocoaWidget,65,69) == 1193519-sideways-lr-decoration-1.html 1193519-sideways-lr-decoration-1-ref.html == 1196887-1-computed-display-inline-block.html 1196887-1-computed-display-inline-block-ref.html == 1205787-legacy-svg-values-1.html 1205787-legacy-svg-values-1-ref.html From e46a3453c1fb8d373652d78bf23bcfb2f05afe43 Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Thu, 27 Aug 2015 11:43:45 -0400 Subject: [PATCH 48/78] Bug 1188483 - desktop_l10n.py should support --locale $locale:$revision. r=jlund --- .../mozharness/mozharness/mozilla/buildbot.py | 2 +- .../mozharness/mozilla/l10n/locales.py | 79 +++++++++++++++---- testing/mozharness/scripts/desktop_l10n.py | 10 ++- 3 files changed, 72 insertions(+), 19 deletions(-) diff --git a/testing/mozharness/mozharness/mozilla/buildbot.py b/testing/mozharness/mozharness/mozilla/buildbot.py index 97189aa2be4..733c57e18cb 100755 --- a/testing/mozharness/mozharness/mozilla/buildbot.py +++ b/testing/mozharness/mozharness/mozilla/buildbot.py @@ -49,7 +49,7 @@ TBPL_WORST_LEVEL_TUPLE = (TBPL_RETRY, TBPL_EXCEPTION, TBPL_FAILURE, class BuildbotMixin(object): - buildbot_config = None + buildbot_config = {} buildbot_properties = {} worst_buildbot_status = TBPL_SUCCESS diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py index ebe8643f2d7..bfaf863dfbb 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -29,34 +29,80 @@ class LocalesMixin(ChunkingMixin): self.abs_dirs = None self.locales = None self.gecko_locale_revisions = None + self.l10n_revisions = {} def query_locales(self): if self.locales is not None: return self.locales c = self.config - locales = c.get("locales", None) ignore_locales = c.get("ignore_locales", []) additional_locales = c.get("additional_locales", []) + # List of locales can be set by using different methods in the + # following order: + # 1. "locales" buildbot property: a string of locale:revision separated + # by space + # 2. "MOZ_LOCALES" env variable: a string of locale:revision separated + # by space + # 3. self.config["locales"] which can be either coming from the config + # or from --locale command line argument + # 4. using self.config["locales_file"] l10n changesets file + locales = None + + # Buildbot property + if hasattr(self, 'read_buildbot_config'): + self.read_buildbot_config() + locales = self.buildbot_config.get("locales") + if locales: + self.info("Using locales from buildbot: %s" % locales) + locales = locales.split() + else: + self.info("'read_buildbot_config()' is missing, ignoring buildbot" + " properties") + + # Environment variable + if not locales and "MOZ_LOCALES" in os.environ: + self.debug("Using locales from environment: %s" % + os.environ["MOZ_LOCALES"]) + locales = os.environ["MOZ_LOCALES"].split() + + # Command line or config + if not locales and c.get("locales", None): + locales = c["locales"] + self.debug("Using locales from config/CLI: %s" % locales) + + # parse locale:revision if set + if locales: + for l in locales: + if ":" in l: + # revision specified in locale string + locale, revision = l.split(":", 1) + self.debug("Using %s:%s" % (locale, revision)) + self.l10n_revisions[locale] = revision + # clean up locale by removing revisions + locales = [l.split(":")[0] for l in locales] + + if not locales and 'locales_file' in c: + locales_file = os.path.join(c['base_work_dir'], c['work_dir'], + c['locales_file']) + locales = self.parse_locales_file(locales_file) + + if not locales: + self.fatal("No locales set!") - if locales is None: - locales = [] - if 'locales_file' in c: - # Best way to get abs/relative path to this? - locales_file = os.path.join(c['base_work_dir'], c['work_dir'], - c['locales_file']) - locales = self.parse_locales_file(locales_file) - else: - self.fatal("No way to determine locales!") for locale in ignore_locales: if locale in locales: self.debug("Ignoring locale %s." % locale) locales.remove(locale) + if locale in self.l10n_revisions: + del self.l10n_revisions[locale] + for locale in additional_locales: if locale not in locales: self.debug("Adding locale %s." % locale) locales.append(locale) - if locales is None: - return + + if not locales: + return None if 'total_locale_chunks' and 'this_locale_chunk' in c: self.debug("Pre-chunking locale list: %s" % str(locales)) locales = self.query_chunked_list(locales, @@ -79,15 +125,14 @@ class LocalesMixin(ChunkingMixin): if locales_file.endswith('json'): locales_json = parse_config_file(locales_file) - self.locale_dict = {} for locale in locales_json.keys(): if isinstance(locales_json[locale], dict): if platform and platform not in locales_json[locale]['platforms']: continue - self.locale_dict[locale] = locales_json[locale]['revision'] + self.l10n_revisions[locale] = locales_json[locale]['revision'] else: # some other way of getting this? - self.locale_dict[locale] = 'default' + self.l10n_revisions[locale] = 'default' locales.append(locale) else: locales = self.read_from_file(locales_file).split() @@ -177,8 +222,8 @@ class LocalesMixin(ChunkingMixin): hg_l10n_base = hg_l10n_base % {"user_repo_override": c["user_repo_override"]} for locale in locales: tag = c.get('hg_l10n_tag', 'default') - if hasattr(self, 'locale_dict'): - tag = self.locale_dict[locale] + if self.l10n_revisions.get(locale): + tag = self.l10n_revisions[locale] locale_repos.append({ 'repo': "%s/%s" % (hg_l10n_base, locale), 'tag': tag, diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 160d9383b2f..58ff83fc669 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -107,7 +107,8 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, {"action": "extend", "dest": "locales", "type": "string", - "help": "Specify the locale(s) to sign and update"} + "help": "Specify the locale(s) to sign and update. Optionally pass" + " revision separated by colon, en-GB:default."} ], [ ['--locales-file', ], {"action": "store", @@ -333,6 +334,13 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, if config.get('en_us_binary_url') and \ config.get('release_config_file'): bootstrap_env['EN_US_BINARY_URL'] = config['en_us_binary_url'] + # Override en_us_binary_url if passed as a buildbot property + self.read_buildbot_config() + if self.buildbot_config.get("en_us_binary_url"): + self.info("Overriding en_us_binary_url with %s" % + self.buildbot_config["en_us_binary_url"]) + bootstrap_env['EN_US_BINARY_URL'] = \ + self.buildbot_config["en_us_binary_url"] if 'MOZ_SIGNING_SERVERS' in os.environ: sign_cmd = self.query_moz_sign_cmd(formats=None) sign_cmd = subprocess.list2cmdline(sign_cmd) From 0c926fe887de264ad43d0f9914b8e08726d12af1 Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Thu, 24 Sep 2015 13:58:20 +0200 Subject: [PATCH 49/78] Backed out 4 changesets (bug 1193264, bug 1193215, bug 1188483, bug 1204120) for test bustage and bustage on a CLOSED TREE Backed out changeset a9a4292b2df3 (bug 1188483) Backed out changeset 9748ff0d2836 (bug 1204120) Backed out changeset a0b952bb6620 (bug 1193264) Backed out changeset 63c61416c2f7 (bug 1193215) --- testing/mach_commands.py | 204 +++++------- .../configs/b2g/emulator_automation_config.py | 4 +- .../configs/unittests/linux_unittest.py | 2 +- .../mozharness/mozharness/mozilla/buildbot.py | 2 +- .../mozharness/mozilla/l10n/locales.py | 79 +---- .../mozharness/mozilla/testing/testbase.py | 33 +- .../mozharness/mozilla/testing/try_tools.py | 173 +++++----- .../scripts/android_emulator_unittest.py | 16 +- testing/mozharness/scripts/android_panda.py | 24 +- .../scripts/b2g_desktop_unittest.py | 19 +- .../scripts/b2g_emulator_unittest.py | 23 +- testing/mozharness/scripts/desktop_l10n.py | 10 +- .../mozharness/scripts/desktop_unittest.py | 75 +++-- testing/mozharness/scripts/marionette.py | 5 +- .../mozharness/scripts/web_platform_tests.py | 33 +- testing/tools/autotry/autotry.py | 296 +++--------------- 16 files changed, 332 insertions(+), 666 deletions(-) diff --git a/testing/mach_commands.py b/testing/mach_commands.py index ed6fb19ec5d..1d525f37d12 100644 --- a/testing/mach_commands.py +++ b/testing/mach_commands.py @@ -10,7 +10,6 @@ import sys import tempfile import subprocess import shutil -from collections import defaultdict from mach.decorators import ( CommandArgument, @@ -437,58 +436,19 @@ class JsapiTestsCommand(MachCommandBase): return jsapi_tests_result -def autotry_parser(): - from autotry import arg_parser - return arg_parser() @CommandProvider class PushToTry(MachCommandBase): - def normalise_list(self, items, allow_subitems=False): - from autotry import parse_arg - rv = defaultdict(list) - for item in items: - parsed = parse_arg(item) - for key, values in parsed.iteritems(): - rv[key].extend(values) - - if not allow_subitems: - if not all(item == [] for item in rv.itervalues()): - raise ValueError("Unexpected subitems in argument") - return rv.keys() - else: - return rv - - def validate_args(self, **kwargs): - if not kwargs["paths"] and not kwargs["tests"] and not kwargs["tags"]: - print("Paths, tags, or tests must be specified as an argument to autotry.") + def validate_args(self, paths, tests, tags, builds, platforms): + if not any([len(paths), tests, tags]): + print("Paths, tests, or tags must be specified.") sys.exit(1) - if kwargs["platforms"] is None: - print("Platforms must be specified as an argument to autotry") - sys.exit(1) + if platforms is None: + platforms = os.environ['AUTOTRY_PLATFORM_HINT'] - try: - platforms = self.normalise_list(kwargs["platforms"]) - except ValueError as e: - print("Error parsing -p argument:\n%s" % e.message) - sys.exit(1) - - try: - tests = (self.normalise_list(kwargs["tests"], allow_subitems=True) - if kwargs["tests"] else {}) - except ValueError as e: - print("Error parsing -u argument:\n%s" % e.message) - sys.exit(1) - - try: - talos = self.normalise_list(kwargs["talos"]) if kwargs["talos"] else [] - except ValueError as e: - print("Error parsing -t argument:\n%s" % e.message) - sys.exit(1) - - paths = [] - for p in kwargs["paths"]: + for p in paths: p = os.path.normpath(os.path.abspath(p)) if not p.startswith(self.topsrcdir): print('Specified path "%s" is outside of the srcdir, unable to' @@ -498,86 +458,70 @@ class PushToTry(MachCommandBase): print('Specified path "%s" is at the top of the srcdir and would' ' select all tests.' % p) sys.exit(1) - paths.append(os.path.relpath(p, self.topsrcdir)) - try: - tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] - except ValueError as e: - print("Error parsing --tags argument:\n%s" % e.message) - sys.exit(1) + return builds, platforms - return kwargs["builds"], platforms, tests, talos, paths, tags, kwargs["extra_args"] + @Command('try', category='testing', description='Push selected tests to the try server') + @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.') + @CommandArgument('-n', dest='verbose', action='store_true', default=False, + help='Print detailed information about the resulting test selection ' + 'and commands performed.') + @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ, + help='Platforms to run. (required if not found in the environment)') + @CommandArgument('-u', dest='tests', + help='Test jobs to run. These will be used in place of suites ' + 'determined by test paths, if any.') + @CommandArgument('--extra', dest='extra_tests', + help='Additional tests to run. These will be added to suites ' + 'determined by test paths, if any.') + @CommandArgument('-b', dest='builds', default='do', + help='Build types to run (d for debug, o for optimized)') + @CommandArgument('--tag', dest='tags', action='append', + help='Restrict tests to the given tag (may be specified multiple times)') + @CommandArgument('--no-push', dest='push', action='store_false', + help='Do not push to try as a result of running this command (if ' + 'specified this command will only print calculated try ' + 'syntax and selection info).') + def autotry(self, builds=None, platforms=None, paths=None, verbose=None, + extra_tests=None, push=None, tags=None, tests=None): + """mach try is under development, please file bugs blocking 1149670. + Pushes the specified tests to try. The simplest way to specify tests is + by using the -u argument, which will behave as usual for try syntax. + This command also provides a mechanism to select test jobs and tests + within a job by path based on tests present in the tree under that + path. Mochitests, xpcshell tests, and reftests are eligible for + selection by this mechanism. Selected tests will be run in a single + chunk of the relevant suite, at this time in chunk 1. - @Command('try', - category='testing', - description='Push selected tests to the try server', - parser=autotry_parser) + Specifying platforms is still required with the -p argument (a default + is taken from the AUTOTRY_PLATFORM_HINT environment variable if set). - def autotry(self, **kwargs): - """Autotry is in beta, please file bugs blocking 1149670. + Tests may be further filtered by passing one or more --tag to the + command. If one or more --tag is specified with out paths or -u, + tests with the given tags will be run in a single chunk of + applicable suites. - Push the current tree to try, with the specified syntax. - - Build options, platforms and regression tests may be selected - using the usual try options (-b, -p and -u respectively). In - addition, tests in a given directory may be automatically - selected by passing that directory as a positional argument to the - command. For example: - - mach try -b d -p linux64 dom testing/web-platform/tests/dom - - would schedule a try run for linux64 debug consisting of all - tests under dom/ and testing/web-platform/tests/dom. - - Test selection using positional arguments is available for - mochitests, reftests, xpcshell tests and web-platform-tests. - - Tests may be also filtered by passing --tag to the command, - which will run only tests marked as having the specified - tags e.g. - - mach try -b d -p win64 --tag media - - would run all tests tagged 'media' on Windows 64. - - If both positional arguments or tags and -u are supplied, the - suites in -u will be run in full. Where tests are selected by - positional argument they will be run in a single chunk. - - If no build option is selected, both debug and opt will be - scheduled. If no platform is selected a default is taken from - the AUTOTRY_PLATFORM_HINT environment variable, if set. + To run suites in addition to those determined from the tree, they + can be passed to the --extra arguent. The command requires either its own mercurial extension ("push-to-try", installable from mach mercurial-setup) or a git repo using git-cinnabar (available at https://github.com/glandium/git-cinnabar). - """ from mozbuild.testing import TestResolver from mozbuild.controller.building import BuildDriver from autotry import AutoTry + import pprint print("mach try is under development, please file bugs blocking 1149670.") + builds, platforms = self.validate_args(paths, tests, tags, builds, platforms) resolver = self._spawn(TestResolver) + at = AutoTry(self.topsrcdir, resolver, self._mach_context) - - if kwargs["load"] is not None: - defaults = at.load_config(kwargs["load"]) - - if defaults is None: - print("No saved configuration called %s found in autotry.ini" % kwargs["load"], - file=sys.stderr) - - for key, value in kwargs.iteritems(): - if value in (None, []) and key in defaults: - kwargs[key] = defaults[key] - - builds, platforms, tests, talos, paths, tags, extra_args = self.validate_args(**kwargs) - - if kwargs["push"] and at.find_uncommited_changes(): + if at.find_uncommited_changes(): print('ERROR please commit changes before continuing') sys.exit(1) @@ -585,40 +529,32 @@ class PushToTry(MachCommandBase): driver = self._spawn(BuildDriver) driver.install_tests(remove=False) - paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) - for item in paths] - paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags) + manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags) - if not paths_by_flavor and not tests: - print("No tests were found when attempting to resolve paths:\n\n\t%s" % - paths) - sys.exit(1) - - if not kwargs["intersection"]: - paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests) - else: - paths_by_flavor = {} - - try: - msg = at.calc_try_syntax(platforms, tests, talos, builds, paths_by_flavor, tags, - extra_args, kwargs["intersection"]) - except ValueError as e: - print(e.message) + if not manifests_by_flavor and not tests: + print("No tests were found when attempting to resolve paths:\n\n\t%s" % + paths) sys.exit(1) - if kwargs["verbose"] and paths_by_flavor: - print('The following tests will be selected: ') - for flavor, paths in paths_by_flavor.iteritems(): - print("%s: %s" % (flavor, ",".join(paths))) + all_manifests = set() + for m in manifests_by_flavor.values(): + all_manifests |= m + all_manifests = list(all_manifests) - if kwargs["verbose"] or not kwargs["push"]: - print('The following try syntax was calculated:\n%s' % msg) + msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests, + extra_tests, builds, all_manifests, tags) - if kwargs["push"]: - at.push_to_try(msg, kwargs["verbose"]) + if verbose and manifests_by_flavor: + print('Tests from the following manifests will be selected: ') + pprint.pprint(manifests_by_flavor) - if kwargs["save"] is not None: - at.save_config(kwargs["save"], msg) + if verbose or not push: + print('The following try syntax was calculated:\n\n\t%s\n' % msg) + + if push: + at.push_to_try(msg, verbose) + + return def get_parser(argv=None): diff --git a/testing/mozharness/configs/b2g/emulator_automation_config.py b/testing/mozharness/configs/b2g/emulator_automation_config.py index a80053769c7..03a39952c02 100644 --- a/testing/mozharness/configs/b2g/emulator_automation_config.py +++ b/testing/mozharness/configs/b2g/emulator_automation_config.py @@ -118,8 +118,8 @@ config = { "--log-errorsummary=%(error_summary_file)s", "--certificate-path=%(certificate_path)s", "--screenshot-on-fail", + "%(test_path)s" ], - "tests": ["%(test_path)s"], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" }, @@ -142,8 +142,8 @@ config = { "--log-errorsummary=%(error_summary_file)s", "--certificate-path=%(certificate_path)s", "--screenshot-on-fail", + "%(test_path)s" ], - "tests": ["%(test_path)s"], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" }, diff --git a/testing/mozharness/configs/unittests/linux_unittest.py b/testing/mozharness/configs/unittests/linux_unittest.py index 2b1c5f357fc..5d0f0969dda 100644 --- a/testing/mozharness/configs/unittests/linux_unittest.py +++ b/testing/mozharness/configs/unittests/linux_unittest.py @@ -256,7 +256,7 @@ config = { }, }, "all_cppunittest_suites": { - "cppunittest": {"tests": ["tests/cppunittest"]} + "cppunittest": ["tests/cppunittest"] }, "all_gtest_suites": { "gtest": [] diff --git a/testing/mozharness/mozharness/mozilla/buildbot.py b/testing/mozharness/mozharness/mozilla/buildbot.py index 733c57e18cb..97189aa2be4 100755 --- a/testing/mozharness/mozharness/mozilla/buildbot.py +++ b/testing/mozharness/mozharness/mozilla/buildbot.py @@ -49,7 +49,7 @@ TBPL_WORST_LEVEL_TUPLE = (TBPL_RETRY, TBPL_EXCEPTION, TBPL_FAILURE, class BuildbotMixin(object): - buildbot_config = {} + buildbot_config = None buildbot_properties = {} worst_buildbot_status = TBPL_SUCCESS diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py index bfaf863dfbb..ebe8643f2d7 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -29,80 +29,34 @@ class LocalesMixin(ChunkingMixin): self.abs_dirs = None self.locales = None self.gecko_locale_revisions = None - self.l10n_revisions = {} def query_locales(self): if self.locales is not None: return self.locales c = self.config + locales = c.get("locales", None) ignore_locales = c.get("ignore_locales", []) additional_locales = c.get("additional_locales", []) - # List of locales can be set by using different methods in the - # following order: - # 1. "locales" buildbot property: a string of locale:revision separated - # by space - # 2. "MOZ_LOCALES" env variable: a string of locale:revision separated - # by space - # 3. self.config["locales"] which can be either coming from the config - # or from --locale command line argument - # 4. using self.config["locales_file"] l10n changesets file - locales = None - - # Buildbot property - if hasattr(self, 'read_buildbot_config'): - self.read_buildbot_config() - locales = self.buildbot_config.get("locales") - if locales: - self.info("Using locales from buildbot: %s" % locales) - locales = locales.split() - else: - self.info("'read_buildbot_config()' is missing, ignoring buildbot" - " properties") - - # Environment variable - if not locales and "MOZ_LOCALES" in os.environ: - self.debug("Using locales from environment: %s" % - os.environ["MOZ_LOCALES"]) - locales = os.environ["MOZ_LOCALES"].split() - - # Command line or config - if not locales and c.get("locales", None): - locales = c["locales"] - self.debug("Using locales from config/CLI: %s" % locales) - - # parse locale:revision if set - if locales: - for l in locales: - if ":" in l: - # revision specified in locale string - locale, revision = l.split(":", 1) - self.debug("Using %s:%s" % (locale, revision)) - self.l10n_revisions[locale] = revision - # clean up locale by removing revisions - locales = [l.split(":")[0] for l in locales] - - if not locales and 'locales_file' in c: - locales_file = os.path.join(c['base_work_dir'], c['work_dir'], - c['locales_file']) - locales = self.parse_locales_file(locales_file) - - if not locales: - self.fatal("No locales set!") + if locales is None: + locales = [] + if 'locales_file' in c: + # Best way to get abs/relative path to this? + locales_file = os.path.join(c['base_work_dir'], c['work_dir'], + c['locales_file']) + locales = self.parse_locales_file(locales_file) + else: + self.fatal("No way to determine locales!") for locale in ignore_locales: if locale in locales: self.debug("Ignoring locale %s." % locale) locales.remove(locale) - if locale in self.l10n_revisions: - del self.l10n_revisions[locale] - for locale in additional_locales: if locale not in locales: self.debug("Adding locale %s." % locale) locales.append(locale) - - if not locales: - return None + if locales is None: + return if 'total_locale_chunks' and 'this_locale_chunk' in c: self.debug("Pre-chunking locale list: %s" % str(locales)) locales = self.query_chunked_list(locales, @@ -125,14 +79,15 @@ class LocalesMixin(ChunkingMixin): if locales_file.endswith('json'): locales_json = parse_config_file(locales_file) + self.locale_dict = {} for locale in locales_json.keys(): if isinstance(locales_json[locale], dict): if platform and platform not in locales_json[locale]['platforms']: continue - self.l10n_revisions[locale] = locales_json[locale]['revision'] + self.locale_dict[locale] = locales_json[locale]['revision'] else: # some other way of getting this? - self.l10n_revisions[locale] = 'default' + self.locale_dict[locale] = 'default' locales.append(locale) else: locales = self.read_from_file(locales_file).split() @@ -222,8 +177,8 @@ class LocalesMixin(ChunkingMixin): hg_l10n_base = hg_l10n_base % {"user_repo_override": c["user_repo_override"]} for locale in locales: tag = c.get('hg_l10n_tag', 'default') - if self.l10n_revisions.get(locale): - tag = self.l10n_revisions[locale] + if hasattr(self, 'locale_dict'): + tag = self.locale_dict[locale] locale_repos.append({ 'repo': "%s/%s" % (hg_l10n_base, locale), 'tag': tag, diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py index fc19622f0a2..aa69546be2e 100755 --- a/testing/mozharness/mozharness/mozilla/testing/testbase.py +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -26,7 +26,7 @@ from mozharness.mozilla.proxxy import Proxxy from mozharness.mozilla.structuredlog import StructuredOutputParser from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser -from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options +from mozharness.mozilla.testing.try_tools import TryToolsMixin from mozharness.mozilla.tooltool import TooltoolMixin from mozharness.lib.python.authentication import get_credentials @@ -83,7 +83,7 @@ testing_config_options = [ "choices": ['ondemand', 'true'], "help": "Download and extract crash reporter symbols.", }], -] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options) +] + copy.deepcopy(virtualenv_config_options) # TestingMixin {{{1 @@ -676,35 +676,6 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") return self.minidump_stackwalk_path - def query_options(self, *args, **kwargs): - if "str_format_values" in kwargs: - str_format_values = kwargs.pop("str_format_values") - else: - str_format_values = {} - - arguments = [] - - for arg in args: - if arg is not None: - arguments.extend(argument % str_format_values for argument in arg) - - return arguments - - def query_tests_args(self, *args, **kwargs): - if "str_format_values" in kwargs: - str_format_values = kwargs.pop("str_format_values") - else: - str_format_values = {} - - arguments = [] - - for arg in reversed(args): - if arg: - arguments.append("--") - arguments.extend(argument % str_format_values for argument in arg) - break - - return arguments def _run_cmd_checks(self, suites): if not suites: diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py index 0d9b69e0e7d..ab8edf61b97 100644 --- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py @@ -8,43 +8,17 @@ import argparse import os import re -from collections import defaultdict from mozharness.base.script import PostScriptAction from mozharness.base.transfer import TransferMixin -try_config_options = [ - [["--try-message"], - {"action": "store", - "dest": "try_message", - "default": None, - "help": "try syntax string to select tests to run", - }], -] - -test_flavors = { - 'browser-chrome': {}, - 'chrome': {}, - 'devtools-chrome': {}, - 'mochitest': {}, - 'xpcshell' :{}, - 'reftest': { - "path": lambda x: os.path.join("tests", "reftest", "tests", x) - }, - 'crashtest': { - "path": lambda x: os.path.join("tests", "reftest", "tests", x) - }, - 'web-platform-tests': { - "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1]) - } -} class TryToolsMixin(TransferMixin): """Utility functions for an interface between try syntax and out test harnesses. Requires log and script mixins.""" harness_extra_args = None - try_test_paths = {} + try_test_paths = [] known_try_arguments = { '--tag': { 'action': 'append', @@ -55,34 +29,31 @@ class TryToolsMixin(TransferMixin): def _extract_try_message(self): msg = None - if "try_message" in self.config and self.config["try_message"]: - msg = self.config["try_message"] - else: - if self.buildbot_config['sourcestamp']['changes']: - msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments'] + if self.buildbot_config['sourcestamp']['changes']: + msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments'] - if msg is None or len(msg) == 1024: - # This commit message was potentially truncated, get the full message - # from hg. - props = self.buildbot_config['properties'] - rev = props['revision'] - repo = props['repo_path'] - url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev) + if msg is None or len(msg) == 1024: + # This commit message was potentially truncated, get the full message + # from hg. + props = self.buildbot_config['properties'] + rev = props['revision'] + repo = props['repo_path'] + url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev) - pushinfo = self.load_json_from_url(url) - for k, v in pushinfo.items(): - if isinstance(v, dict) and 'changesets' in v: - msg = v['changesets'][-1]['desc'] + pushinfo = self.load_json_from_url(url) + for k, v in pushinfo.items(): + if isinstance(v, dict) and 'changesets' in v: + msg = v['changesets'][-1]['desc'] - if not msg and 'try_syntax' in self.buildbot_config['properties']: - # If we don't find try syntax in the usual place, check for it in an - # alternate property available to tools using self-serve. - msg = self.buildbot_config['properties']['try_syntax'] + if not msg and 'try_syntax' in self.buildbot_config['properties']: + # If we don't find try syntax in the usual place, check for it in an + # alternate property available to tools using self-serve. + msg = self.buildbot_config['properties']['try_syntax'] return msg @PostScriptAction('download-and-extract') - def set_extra_try_arguments(self, action, success=None): + def _set_extra_try_arguments(self, action, success=None): """Finds a commit message and parses it for extra arguments to pass to the test harness command line and test paths used to filter manifests. @@ -136,7 +107,7 @@ class TryToolsMixin(TransferMixin): parser.add_argument('--try-test-paths', nargs='*') (args, _) = parser.parse_known_args(all_try_args) - self.try_test_paths = self._group_test_paths(args.try_test_paths) + self.try_test_paths = args.try_test_paths del args.try_test_paths out_args = [] @@ -156,35 +127,91 @@ class TryToolsMixin(TransferMixin): self.harness_extra_args = out_args - def _group_test_paths(self, args): - rv = defaultdict(list) + def _resolve_specified_manifests(self): + if not self.try_test_paths: + return None - if args is None: - return rv + target_manifests = set(self.try_test_paths) - for item in args: - suite, path = item.split(":", 1) - rv[suite].append(path) - return rv + def filter_ini_manifest(line): + # Lines are formatted as [include:], we care about . + parts = line.split(':') + term = line + if len(parts) == 2: + term = parts[1] + if term.endswith(']'): + term = term[:-1] + if (term in target_manifests or + any(term.startswith(l) for l in target_manifests)): + return True + return False - def try_args(self, flavor): - """Get arguments, test_list derived from try syntax to apply to a command""" + def filter_list_manifest(line): + # Lines are usually formatted as "include ", we care about . + parts = line.split() + term = line + if len(parts) == 2: + term = parts[1] + # Reftest master manifests also include skip-if lines and relative + # paths we aren't doing to resolve here, so unlike above this is just + # a substring check. + if (term in target_manifests or + any(l in term for l in target_manifests)): + return True + return False + + # The master manifests we need to filter for target manifests. + # TODO: All this needs to go in a config file somewhere and get sewn + # into the job definition so its less likely to break as things are + # modified. One straightforward way to achieve this would be with a key + # in the tree manifest for the master manifest path, however the current + # tree manifests don't distinguish between flavors of mochitests to this + # isn't straightforward. + master_manifests = [ + ('mochitest/chrome/chrome.ini', filter_ini_manifest), + ('mochitest/browser/browser-chrome.ini', filter_ini_manifest), + ('mochitest/tests/mochitest.ini', filter_ini_manifest), + ('xpcshell/tests/all-test-dirs.list', filter_ini_manifest), + ('xpcshell/tests/xpcshell.ini', filter_ini_manifest), + ('reftest/tests/layout/reftests/reftest.list', filter_list_manifest), + ('reftest/tests/testing/crashtest/crashtests.list', filter_list_manifest), + ] + + dirs = self.query_abs_dirs() + tests_dir = dirs.get('abs_test_install_dir', + os.path.join(dirs['abs_work_dir'], 'tests')) + master_manifests = [(os.path.join(tests_dir, name), filter_fn) for (name, filter_fn) in + master_manifests] + for m, filter_fn in master_manifests: + if not os.path.isfile(m): + continue + + self.info("Filtering master manifest at: %s" % m) + lines = self.read_from_file(m).splitlines() + + out_lines = [line for line in lines if filter_fn(line)] + + self.rmtree(m) + self.write_to_file(m, '\n'.join(out_lines)) + + + def append_harness_extra_args(self, cmd): + """Append arguments derived from try syntax to a command.""" # TODO: Detect and reject incompatible arguments - args = self.harness_extra_args[:] if self.harness_extra_args else [] - - if self.try_test_paths.get(flavor): + extra_args = self.harness_extra_args[:] if self.harness_extra_args else [] + if self.try_test_paths: + self._resolve_specified_manifests() self.info('TinderboxPrint: Tests will be run from the following ' - 'files: %s.' % ','.join(self.try_test_paths[flavor])) - args.extend(['--this-chunk=1', '--total-chunks=1']) + 'manifests: %s.' % ','.join(self.try_test_paths)) + extra_args.extend(['--this-chunk=1', '--total-chunks=1']) - path_func = test_flavors[flavor].get("path", lambda x:x) - tests = [path_func(item) for item in self.try_test_paths[flavor]] - else: - tests = [] + if not extra_args: + return cmd - if args or tests: - self.info('TinderboxPrint: The following arguments were forwarded from mozharness ' - 'to the test command:\nTinderboxPrint: \t%s -- %s' % - (" ".join(args), " ".join(tests))) + out_cmd = cmd[:] + out_cmd.extend(extra_args) + self.info('TinderboxPrint: The following arguments were forwarded from mozharness ' + 'to the test command:\nTinderboxPrint: \t%s' % + extra_args) - return args, tests + return out_cmd diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index c240f2cca9b..739e3d6f451 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -483,12 +483,14 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin continue cmd.append(arg) - try_options, try_tests = self.try_args(suite_category) - cmd.extend(try_options) - cmd.extend(self.query_tests_args( - self.config["suite_definitions"][suite_category].get("tests"), - self.test_suite_definitions[self.test_suite].get("tests"), - try_tests)) + tests = None + if "tests" in self.test_suite_definitions[self.test_suite]: + tests = self.test_suite_definitions[self.test_suite]["tests"] + elif "tests" in self.config["suite_definitions"][suite_category]: + tests = self.config["suite_definitions"][suite_category]["tests"] + + if tests: + cmd.extend(tests) return cmd @@ -662,7 +664,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin Run the tests """ cmd = self._build_command() - + cmd = self.append_harness_extra_args(cmd) try: cwd = self._query_tests_dir() except: diff --git a/testing/mozharness/scripts/android_panda.py b/testing/mozharness/scripts/android_panda.py index 66271ebc15f..695e9a0d978 100644 --- a/testing/mozharness/scripts/android_panda.py +++ b/testing/mozharness/scripts/android_panda.py @@ -217,20 +217,6 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu if self._query_specified_suites(cat) is not None] super(PandaTest, self).download_and_extract(suite_categories=target_categories) - def _query_try_flavor(self, category, suite): - flavors = { - "mochitest": [("plain.*", "mochitest"), - ("browser-chrome.*", "browser-chrome"), - ("mochitest-devtools-chrome.*", "devtools-chrome"), - ("chrome", "chrome")], - "xpcshell": [("xpcshell", "xpcshell")], - "reftest": [("reftest", "reftest"), - ("crashtest", "crashtest")] - } - for suite_pattern, flavor in flavors.get(category, []): - if re.compile(suite_pattern).match(suite): - return flavor - def _run_category_suites(self, suite_category, preflight_run_method=None): """run suite(s) to a specific category""" @@ -260,13 +246,11 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu if should_install_app: self._install_app() cmd = abs_base_cmd[:] + replace_dict = {} + for arg in suites[suite]: + cmd.append(arg % replace_dict) - flavor = self._query_try_flavor(suite_category, suite) - try_options, try_tests = self.try_args(flavor) - - cmd.extend(self.query_options(suites[suite], - try_options)) - cmd.extend(self.query_tests_args(try_tests)) + cmd = self.append_harness_extra_args(cmd) tests = self.config["suite_definitions"][suite_category].get("tests", []) cmd += tests diff --git a/testing/mozharness/scripts/b2g_desktop_unittest.py b/testing/mozharness/scripts/b2g_desktop_unittest.py index 6fb7f3f2fab..f6040a1ab83 100755 --- a/testing/mozharness/scripts/b2g_desktop_unittest.py +++ b/testing/mozharness/scripts/b2g_desktop_unittest.py @@ -139,7 +139,6 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): '%s_raw.log' % suite) error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], '%s_errorsummary.log' % suite) - str_format_values = { 'application': self.binary_path, 'test_manifest': self.test_manifest, @@ -157,15 +156,16 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): if suite not in self.config["suite_definitions"]: self.fatal("'%s' not defined in the config!" % suite) - try_options, try_tests = self.try_args(suite_name) + options = self.config["suite_definitions"][suite]["options"] + if options: + for option in options: + option = option % str_format_values + if not option.endswith('None'): + cmd.append(option) - cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite]["options"], - try_options, - str_format_values=str_format_values)) - - cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite].get("tests"), - try_tests, - str_format_values=str_format_values) + tests = self.config["suite_definitions"][suite].get("tests") + if tests: + cmd.extend(tests) return cmd @@ -200,6 +200,7 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript): self.fatal("Don't know how to run --test-suite '%s'!" % suite) cmd = self._query_abs_base_cmd(suite) + cmd = self.append_harness_extra_args(cmd) cwd = dirs['abs_%s_dir' % suite] diff --git a/testing/mozharness/scripts/b2g_emulator_unittest.py b/testing/mozharness/scripts/b2g_emulator_unittest.py index c0dab5d838a..9b78ddb53d4 100755 --- a/testing/mozharness/scripts/b2g_emulator_unittest.py +++ b/testing/mozharness/scripts/b2g_emulator_unittest.py @@ -273,19 +273,18 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin): } if suite not in self.config["suite_definitions"]: - self.fatal("'%s' not defined in the config!" % suite) + self.fatal("Key '%s' not defined in the config!" % suite) - try_options, try_tests = self.try_args(suite) + options = self.config["suite_definitions"][suite]["options"] + if options: + for option in options: + option = option % str_format_values + if not option.endswith('None'): + cmd.append(option) - options = self.query_options(self.config["suite_definitions"][suite]["options"], - try_options, - str_format_values=str_format_values) - cmd.extend(opt for opt in options if not opt.endswith('None')) - - tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"), - try_tests, - str_format_values=str_format_values) - cmd.extend(opt for opt in tests if not opt.endswith('None')) + tests = self.config["suite_definitions"][suite].get("tests", []) + if tests: + cmd.extend(tests) return cmd @@ -342,6 +341,8 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin): self.fatal("Don't know how to run --test-suite '%s'!" % suite) cmd = self._query_abs_base_cmd(suite) + cmd = self.append_harness_extra_args(cmd) + cwd = dirs['abs_%s_dir' % suite] # TODO we probably have to move some of the code in diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 58ff83fc669..160d9383b2f 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -107,8 +107,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, {"action": "extend", "dest": "locales", "type": "string", - "help": "Specify the locale(s) to sign and update. Optionally pass" - " revision separated by colon, en-GB:default."} + "help": "Specify the locale(s) to sign and update"} ], [ ['--locales-file', ], {"action": "store", @@ -334,13 +333,6 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, if config.get('en_us_binary_url') and \ config.get('release_config_file'): bootstrap_env['EN_US_BINARY_URL'] = config['en_us_binary_url'] - # Override en_us_binary_url if passed as a buildbot property - self.read_buildbot_config() - if self.buildbot_config.get("en_us_binary_url"): - self.info("Overriding en_us_binary_url with %s" % - self.buildbot_config["en_us_binary_url"]) - bootstrap_env['EN_US_BINARY_URL'] = \ - self.buildbot_config["en_us_binary_url"] if 'MOZ_SIGNING_SERVERS' in os.environ: sign_cmd = self.query_moz_sign_cmd(formats=None) sign_cmd = subprocess.list2cmdline(sign_cmd) diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index d57e3593013..3c74b3c8ecf 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -34,6 +34,7 @@ from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_opt SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt'] + # DesktopUnittest {{{1 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin): config_options = [ @@ -298,6 +299,32 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix self.symbols_url = symbols_url return self.symbols_url + def get_webapprt_path(self, res_dir, mochitest_dir): + """Get the path to the webapp runtime binary. + On Mac, we copy the stub from the resources dir to the test app bundle, + since we have to run it from the executable directory of a bundle + in order for its windows to appear. Ideally, the build system would do + this for us at build time, and we should find a way for it to do that. + """ + exe_suffix = self.config.get('exe_suffix', '') + app_name = 'webapprt-stub' + exe_suffix + app_path = os.path.join(res_dir, app_name) + if self._is_darwin(): + mac_dir_name = os.path.join( + mochitest_dir, + 'webapprtChrome', + 'webapprt', + 'test', + 'chrome', + 'TestApp.app', + 'Contents', + 'MacOS') + mac_app_name = 'webapprt' + exe_suffix + mac_app_path = os.path.join(mac_dir_name, mac_app_name) + self.copyfile(app_path, mac_app_path, copystat=True) + return mac_app_path + return app_path + def _query_abs_base_cmd(self, suite_category, suite): if self.binary_path: c = self.config @@ -308,11 +335,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix abs_app_dir = self.query_abs_app_dir() abs_res_dir = self.query_abs_res_dir() - webapprt_path = os.path.join(os.path.dirname(self.binary_path), - 'webapprt-stub') - if c.get('exe_suffix'): - webapprt_path += c['exe_suffix'] - raw_log_file = os.path.join(dirs['abs_blob_upload_dir'], '%s_raw.log' % suite) @@ -322,7 +344,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix 'binary_path': self.binary_path, 'symbols_path': self._query_symbols_url(), 'abs_app_dir': abs_app_dir, - 'app_path': webapprt_path, + 'abs_res_dir': abs_res_dir, 'raw_log_file': raw_log_file, 'error_summary_file': error_summary_file, 'gtest_dir': os.path.join(dirs['abs_test_install_dir'], @@ -334,6 +356,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix if self.symbols_path: str_format_values['symbols_path'] = self.symbols_path + if suite_category == 'webapprt': + str_format_values['app_path'] = self.get_webapprt_path(abs_res_dir, dirs['abs_mochitest_dir']) + if c['e10s']: base_cmd.append('--e10s') @@ -373,14 +398,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix "please make sure they are specified in your " "config under %s_options" % (suite_category, suite_category)) - - - for option in options: - option = option % str_format_values - if not option.endswith('None'): - base_cmd.append(option) - - return base_cmd + return base_cmd else: self.fatal("'binary_path' could not be determined.\n This should " "be like '/path/build/application/firefox/firefox'" @@ -416,20 +434,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix return suites - def _query_try_flavor(self, category, suite): - flavors = { - "mochitest": [("plain.*", "mochitest"), - ("browser-chrome.*", "browser-chrome"), - ("mochitest-devtools-chrome.*", "devtools-chrome"), - ("chrome", "chrome")], - "xpcshell": [("xpcshell", "xpcshell")], - "reftest": [("reftest", "reftest"), - ("crashtest", "crashtest")] - } - for suite_pattern, flavor in flavors.get(category, []): - if re.compile(suite_pattern).match(suite): - return flavor - # Actions {{{2 # clobber defined in BaseScript, deletes mozharness/build if exists @@ -584,22 +588,15 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix options_list = [] env = {} if isinstance(suites[suite], dict): - options_list = suites[suite].get('options', []) - tests_list = suites[suite].get('tests', []) + options_list = suites[suite]['options'] + suites[suite].get("tests", []) env = copy.deepcopy(suites[suite].get('env', {})) else: options_list = suites[suite] - tests_list = [] - flavor = self._query_try_flavor(suite_category, suite) - try_options, try_tests = self.try_args(flavor) + for arg in options_list: + cmd.append(arg % replace_dict) - cmd.extend(self.query_options(options_list, - try_options, - str_format_values=replace_dict)) - cmd.extend(self.query_tests_args(tests_list, - try_tests, - str_format_values=replace_dict)) + cmd = self.append_harness_extra_args(cmd) suite_name = suite_category + '-' + suite tbpl_status, log_level = None, None diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py index 6f84275c80b..4ae73b59cff 100755 --- a/testing/mozharness/scripts/marionette.py +++ b/testing/mozharness/scripts/marionette.py @@ -439,10 +439,7 @@ class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMix self.fatal("Could not create blobber upload directory") cmd.append(manifest) - - try_options, try_tests = self.try_args("marionette") - cmd.extend(self.query_tests_args(try_tests, - str_format_values=config_fmt_args)) + cmd = self.append_harness_extra_args(cmd) env = {} if self.query_minidump_stackwalk(): diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index 0167f487a22..f1982b2ca0c 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -115,31 +115,31 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): abs_app_dir = self.query_abs_app_dir() run_file_name = "runtests.py" - cmd = [self.query_python_path('python'), '-u'] - cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name)) + base_cmd = [self.query_python_path('python'), '-u'] + base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name)) # Make sure that the logging directory exists if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1: self.fatal("Could not create blobber upload directory") # Exit - cmd += ["--log-raw=-", - "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"], - "wpt_raw.log"), - "--binary=%s" % self.binary_path, - "--symbols-path=%s" % self.query_symbols_url(), - "--stackwalk-binary=%s" % self.query_minidump_stackwalk()] + base_cmd += ["--log-raw=-", + "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"], + "wpt_raw.log"), + "--binary=%s" % self.binary_path, + "--symbols-path=%s" % self.query_symbols_url(), + "--stackwalk-binary=%s" % self.query_minidump_stackwalk()] for test_type in c.get("test_type", []): - cmd.append("--test-type=%s" % test_type) + base_cmd.append("--test-type=%s" % test_type) if c.get("e10s"): - cmd.append("--e10s") + base_cmd.append("--e10s") for opt in ["total_chunks", "this_chunk"]: val = c.get(opt) if val: - cmd.append("--%s=%s" % (opt.replace("_", "-"), val)) + base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val)) options = list(c.get("options", [])) @@ -151,15 +151,9 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): 'abs_work_dir': dirs["abs_work_dir"] } - try_options, try_tests = self.try_args("web-platform-tests") + opt_cmd = [item % str_format_values for item in options] - cmd.extend(self.query_options(options, - try_options, - str_format_values=str_format_values)) - cmd.extend(self.query_tests_args(try_tests, - str_format_values=str_format_values)) - - return cmd + return base_cmd + opt_cmd def download_and_extract(self): super(WebPlatformTest, self).download_and_extract( @@ -173,6 +167,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin): def run_tests(self): dirs = self.query_abs_dirs() cmd = self._query_cmd() + cmd = self.append_harness_extra_args(cmd) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj) diff --git a/testing/tools/autotry/autotry.py b/testing/tools/autotry/autotry.py index dbbd2313106..ca8dd4f5645 100644 --- a/testing/tools/autotry/autotry.py +++ b/testing/tools/autotry/autotry.py @@ -2,172 +2,30 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import argparse -import itertools -import os -import re -import subprocess import sys +import os +import itertools +import subprocess import which from collections import defaultdict -import ConfigParser - -def arg_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.') - parser.add_argument('-b', dest='builds', default='do', - help='Build types to run (d for debug, o for optimized).') - parser.add_argument('-p', dest='platforms', action="append", - help='Platforms to run (required if not found in the environment).') - parser.add_argument('-u', dest='tests', action="append", - help='Test suites to run in their entirety.') - parser.add_argument('-t', dest="talos", action="append", - help='Talos suites to run.') - parser.add_argument('--tag', dest='tags', action='append', - help='Restrict tests to the given tag (may be specified multiple times).') - parser.add_argument('--and', action='store_true', dest="intersection", - help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments.') - parser.add_argument('--no-push', dest='push', action='store_false', - help='Do not push to try as a result of running this command (if ' - 'specified this command will only print calculated try ' - 'syntax and selection info).') - parser.add_argument('--save', dest="save", action='store', - help="Save the command line arguments for future use with --preset.") - parser.add_argument('--preset', dest="load", action='store', - help="Load a saved set of arguments. Additional arguments will override saved ones.") - parser.add_argument('extra_args', nargs=argparse.REMAINDER, - help='Extra arguments to put in the try push.') - parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False, - help='Print detailed information about the resulting test selection ' - 'and commands performed.') - return parser - -class TryArgumentTokenizer(object): - symbols = [("seperator", ","), - ("list_start", "\["), - ("list_end", "\]"), - ("item", "([^,\[\]\s][^,\[\]]+)"), - ("space", "\s+")] - token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) - - def tokenize(self, data): - for match in self.token_re.finditer(data): - symbol = match.lastgroup - data = match.group(symbol) - if symbol == "space": - pass - else: - yield symbol, data - -class TryArgumentParser(object): - """Simple three-state parser for handling expressions - of the from "foo[sub item, another], bar,baz". This takes - input from the TryArgumentTokenizer and runs through a small - state machine, returning a dictionary of {top-level-item:[sub_items]} - i.e. the above would result in - {"foo":["sub item", "another"], "bar": [], "baz": []} - In the case of invalid input a ValueError is raised.""" - - EOF = object() - - def __init__(self): - self.reset() - - def reset(self): - self.tokens = None - self.current_item = None - self.data = {} - self.token = None - self.state = None - - def parse(self, tokens): - self.reset() - self.tokens = tokens - self.consume() - self.state = self.item_state - while self.token[0] != self.EOF: - self.state() - return self.data - - def consume(self): - try: - self.token = self.tokens.next() - except StopIteration: - self.token = (self.EOF, None) - - def expect(self, *types): - if self.token[0] not in types: - raise ValueError("Error parsing try string, unexpected %s" % (self.token[0])) - - def item_state(self): - self.expect("item") - value = self.token[1].strip() - if value not in self.data: - self.data[value] = [] - self.current_item = value - self.consume() - if self.token[0] == "seperator": - self.consume() - elif self.token[0] == "list_start": - self.consume() - self.state = self.subitem_state - elif self.token[0] == self.EOF: - pass - else: - raise ValueError - - def subitem_state(self): - self.expect("item") - value = self.token[1].strip() - self.data[self.current_item].append(value) - self.consume() - if self.token[0] == "seperator": - self.consume() - elif self.token[0] == "list_end": - self.consume() - self.state = self.after_list_end_state - else: - raise ValueError - - def after_list_end_state(self): - self.expect("seperator") - self.consume() - self.state = self.item_state - -def parse_arg(arg): - tokenizer = TryArgumentTokenizer() - parser = TryArgumentParser() - return parser.parse(tokenizer.tokenize(arg)) +TRY_SYNTAX_TMPL = """ +try: -b %s -p %s -u %s -t none %s %s +""" class AutoTry(object): - # Maps from flavors to the job names needed to run that flavour - flavor_jobs = { - 'mochitest': ['mochitest-1', 'mochitest-e10s-1'], - 'xpcshell': ['xpcshell'], - 'chrome': ['mochitest-o'], - 'browser-chrome': ['mochitest-browser-chrome-1', - 'mochitest-e10s-browser-chrome-1'], - 'devtools-chrome': ['mochitest-dt', - 'mochitest-e10s-devtools-chrome'], - 'crashtest': ['crashtest', 'crashtest-e10s'], - 'reftest': ['reftest', 'reftest-e10s'], - 'web-platform-tests': ['web-platform-tests-1'], - } - - flavor_suites = { - "mochitest": "mochitests", - "xpcshell": "xpcshell", - "chrome": "mochitest-o", - "browser-chrome": "mochitest-bc", - "devtools-chrome": "mochitest-dt", - "crashtest": "crashtest", - "reftest": "reftest", - "web-platform-tests": "web-platform-tests", - } + test_flavors = [ + 'browser-chrome', + 'chrome', + 'devtools-chrome', + 'mochitest', + 'xpcshell', + 'reftest', + 'crashtest', + ] def __init__(self, topsrcdir, resolver, mach_context): self.topsrcdir = topsrcdir @@ -179,107 +37,57 @@ class AutoTry(object): else: self._use_git = True - @property - def config_path(self): - return os.path.join(self.mach_context.state_dir, "autotry.ini") - - def load_config(self, name): - config = ConfigParser.RawConfigParser() - success = config.read([self.config_path]) - if not success: - return None - - try: - data = config.get("try", name) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return None - - kwargs = vars(arg_parser().parse_args(data.split())) - - return kwargs - - def save_config(self, name, data): - assert data.startswith("try: ") - data = data[len("try: "):] - - parser = ConfigParser.RawConfigParser() - parser.read([self.config_path]) - - if not parser.has_section("try"): - parser.add_section("try") - - parser.set("try", name, data) - - with open(self.config_path, "w") as f: - parser.write(f) - - def paths_by_flavor(self, paths=None, tags=None): - paths_by_flavor = defaultdict(set) - + def resolve_manifests(self, paths=None, tags=None): if not (paths or tags): - return dict(paths_by_flavor) + return {} - tests = list(self.resolver.resolve_tests(paths=paths, - tags=tags)) + tests = list(self.resolver.resolve_tests(tags=tags, + paths=paths, + cwd=self.mach_context.cwd)) + manifests_by_flavor = defaultdict(set) for t in tests: - if t['flavor'] in self.flavor_suites: + if t['flavor'] in AutoTry.test_flavors: flavor = t['flavor'] if 'subsuite' in t and t['subsuite'] == 'devtools': flavor = 'devtools-chrome' + manifest = os.path.relpath(t['manifest'], self.topsrcdir) + manifests_by_flavor[flavor].add(manifest) - for path in paths: - if flavor in ["crashtest", "reftest"]: - manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir) - if manifest_relpath.startswith(path): - paths_by_flavor[flavor].add(manifest_relpath) - else: - if t['file_relpath'].startswith(path): - paths_by_flavor[flavor].add(path) + return dict(manifests_by_flavor) - return dict(paths_by_flavor) + def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds, + manifests, tags): - def remove_duplicates(self, paths_by_flavor, tests): - rv = {} - for item in paths_by_flavor: - if self.flavor_suites[item] not in tests: - rv[item] = paths_by_flavor[item].copy() - return rv - - def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags, - extra_args, intersection): - parts = ["try:", "-b", builds, "-p", ",".join(platforms)] - - suites = tests if not intersection else {} - paths = set() - for flavor, flavor_tests in paths_by_flavor.iteritems(): - suite = self.flavor_suites[flavor] - if suite not in suites and (not intersection or suite in tests): - for job_name in self.flavor_jobs[flavor]: - for test in flavor_tests: - paths.add("%s:%s" % (flavor, test)) - suites[job_name] = tests.get(suite, []) - - if not suites: - raise ValueError("No tests found matching filters") - - parts.append("-u") - parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "") - for k,v in sorted(suites.items())) if suites else "none") - - parts.append("-t") - parts.append(",".join(talos) if talos else "none") + # Maps from flavors to the try syntax snippets implied by that flavor. + # TODO: put selected tests under their own builder/label to avoid + # confusion reading results on treeherder. + flavor_suites = { + 'mochitest': ['mochitest-1', 'mochitest-e10s-1'], + 'xpcshell': ['xpcshell'], + 'chrome': ['mochitest-o'], + 'browser-chrome': ['mochitest-browser-chrome-1', + 'mochitest-e10s-browser-chrome-1'], + 'devtools-chrome': ['mochitest-dt', + 'mochitest-e10s-devtools-chrome'], + 'crashtest': ['crashtest', 'crashtest-e10s'], + 'reftest': ['reftest', 'reftest-e10s'], + } if tags: - parts.append(' '.join('--tag %s' % t for t in tags)) + tags = ' '.join('--tag %s' % t for t in tags) + else: + tags = '' - if extra_args is not None: - parts.extend(extra_args) + if not tests: + tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors))) + if extra_tests: + tests += ',%s' % (extra_tests) - if paths: - parts.append("--try-test-paths %s" % " ".join(sorted(paths))) - - return " ".join(parts) + manifests = ' '.join(manifests) + if manifests: + manifests = '--try-test-paths %s' % manifests + return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags) def _run_git(self, *args): args = ['git'] + list(args) From d784c195753089921db4390c248bdf7ab83cdbf9 Mon Sep 17 00:00:00 2001 From: Ehsan Akhgari Date: Thu, 24 Sep 2015 07:58:35 -0400 Subject: [PATCH 50/78] Bug 1206458 follow-up: Continue to use the LOAD_FLAGS_BYPASS_CACHE flag for the Request Desktop Mode feature As per the discussion happened on the mailing list: https://mail.mozilla.org/pipermail/mobile-firefox-dev/2015-September/001518.html --- mobile/android/chrome/content/browser.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mobile/android/chrome/content/browser.js b/mobile/android/chrome/content/browser.js index c9c813d0470..8ef99f0a478 100644 --- a/mobile/android/chrome/content/browser.js +++ b/mobile/android/chrome/content/browser.js @@ -3602,7 +3602,11 @@ Tab.prototype = { return; let url = currentURI.spec; - let flags = Ci.nsIWebNavigation.LOAD_FLAGS_REPLACE_HISTORY; + // We need LOAD_FLAGS_BYPASS_CACHE here since we're changing the User-Agent + // string, and servers typically don't use the Vary: User-Agent header, so + // not doing this means that we'd get some of the previously cached content. + let flags = Ci.nsIWebNavigation.LOAD_FLAGS_BYPASS_CACHE | + Ci.nsIWebNavigation.LOAD_FLAGS_REPLACE_HISTORY; if (this.originalURI && !this.originalURI.equals(currentURI)) { // We were redirected; reload the original URL url = this.originalURI.spec; From f5ffd8414181710e4b95ed3a86f02b1050a10f0f Mon Sep 17 00:00:00 2001 From: Neil Deakin Date: Thu, 24 Sep 2015 09:09:54 -0400 Subject: [PATCH 51/78] Bug 1204944, convert drag position coordinates properly, r=mstange --- widget/cocoa/nsChildView.mm | 4 +++- widget/nsBaseDragService.cpp | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/widget/cocoa/nsChildView.mm b/widget/cocoa/nsChildView.mm index e04275bc515..c421a1277a0 100644 --- a/widget/cocoa/nsChildView.mm +++ b/widget/cocoa/nsChildView.mm @@ -5889,7 +5889,9 @@ PanGestureTypeForEvent(NSEvent* aEvent) if (dragService) { NSPoint pnt = [NSEvent mouseLocation]; FlipCocoaScreenCoordinate(pnt); - dragService->DragMoved(NSToIntRound(pnt.x), NSToIntRound(pnt.y)); + + nsIntPoint devPoint = mGeckoChild->CocoaPointsToDevPixels(pnt); + dragService->DragMoved(devPoint.x, devPoint.y); } } diff --git a/widget/nsBaseDragService.cpp b/widget/nsBaseDragService.cpp index 7ec2bd319e4..bf7c4481c6e 100644 --- a/widget/nsBaseDragService.cpp +++ b/widget/nsBaseDragService.cpp @@ -432,8 +432,8 @@ nsBaseDragService::DragMoved(int32_t aX, int32_t aY) nsIFrame* frame = mDragPopup->GetPrimaryFrame(); if (frame && frame->GetType() == nsGkAtoms::menuPopupFrame) { nsPresContext* presContext = frame->PresContext(); - int32_t x = presContext->DevPixelsToIntCSSPixels(aX - mImageX); - int32_t y = presContext->DevPixelsToIntCSSPixels(aY - mImageY); + int32_t x = presContext->DevPixelsToIntCSSPixels(aX) - mImageX; + int32_t y = presContext->DevPixelsToIntCSSPixels(aY) - mImageY; (static_cast(frame))->MoveTo(x, y, true); } } From 48b42d258929f960716b0cbd8c00c4592b88bc82 Mon Sep 17 00:00:00 2001 From: Randell Jesup Date: Thu, 24 Sep 2015 09:23:37 -0400 Subject: [PATCH 52/78] Bug 953265: Update webrtc/getUserMedia default audio capture rate to 32KHz r=padenot --- dom/media/webrtc/MediaEngine.h | 6 ++++++ dom/media/webrtc/MediaEngineDefault.cpp | 2 +- dom/media/webrtc/MediaEngineWebRTC.h | 2 ++ dom/media/webrtc/MediaEngineWebRTCAudio.cpp | 17 ++++++++++------- .../src/mediapipeline/MediaPipeline.cpp | 5 ++++- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/dom/media/webrtc/MediaEngine.h b/dom/media/webrtc/MediaEngine.h index fd71e8f0a48..336d80501c6 100644 --- a/dom/media/webrtc/MediaEngine.h +++ b/dom/media/webrtc/MediaEngine.h @@ -54,6 +54,12 @@ public: static const int DEFAULT_169_VIDEO_HEIGHT = 720; static const int DEFAULT_AUDIO_TIMER_MS = 10; +#ifndef MOZ_B2G + static const int DEFAULT_SAMPLE_RATE = 32000; +#else + static const int DEFAULT_SAMPLE_RATE = 16000; +#endif + /* Populate an array of video sources in the nsTArray. Also include devices * that are currently unavailable. */ virtual void EnumerateVideoDevices(dom::MediaSourceEnum, diff --git a/dom/media/webrtc/MediaEngineDefault.cpp b/dom/media/webrtc/MediaEngineDefault.cpp index 612255e3380..e7467ad1870 100644 --- a/dom/media/webrtc/MediaEngineDefault.cpp +++ b/dom/media/webrtc/MediaEngineDefault.cpp @@ -26,7 +26,7 @@ #include "YuvStamper.h" #endif -#define AUDIO_RATE 16000 +#define AUDIO_RATE mozilla::MediaEngine::DEFAULT_SAMPLE_RATE #define AUDIO_FRAME_LENGTH ((AUDIO_RATE * MediaEngine::DEFAULT_AUDIO_TIMER_MS) / 1000) namespace mozilla { diff --git a/dom/media/webrtc/MediaEngineWebRTC.h b/dom/media/webrtc/MediaEngineWebRTC.h index 6cf470727dc..137a76ae40d 100644 --- a/dom/media/webrtc/MediaEngineWebRTC.h +++ b/dom/media/webrtc/MediaEngineWebRTC.h @@ -137,6 +137,7 @@ public: , mChannel(-1) , mInitDone(false) , mStarted(false) + , mSampleFrequency(MediaEngine::DEFAULT_SAMPLE_RATE) , mEchoOn(false), mAgcOn(false), mNoiseOn(false) , mEchoCancel(webrtc::kEcDefault) , mAGC(webrtc::kAgcDefault) @@ -225,6 +226,7 @@ private: nsString mDeviceName; nsCString mDeviceUUID; + uint32_t mSampleFrequency; bool mEchoOn, mAgcOn, mNoiseOn; webrtc::EcModes mEchoCancel; webrtc::AgcModes mAGC; diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp index da3f2d86572..52897d4d961 100644 --- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -19,9 +19,8 @@ #define ENCODING "L16" #define DEFAULT_PORT 5555 -#define SAMPLE_RATE 256000 -#define SAMPLE_FREQUENCY 16000 -#define SAMPLE_LENGTH ((SAMPLE_FREQUENCY*10)/1000) +#define SAMPLE_RATE(freq) ((freq)*2*8) // bps, 16-bit samples +#define SAMPLE_LENGTH(freq) (((freq)*10)/1000) // These are restrictions from the webrtc.org code #define MAX_CHANNELS 2 @@ -345,7 +344,7 @@ MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream, } AudioSegment* segment = new AudioSegment(); - aStream->AddAudioTrack(aID, SAMPLE_FREQUENCY, 0, segment, SourceMediaStream::ADDTRACK_QUEUED); + aStream->AddAudioTrack(aID, mSampleFrequency, 0, segment, SourceMediaStream::ADDTRACK_QUEUED); // XXX Make this based on the pref. aStream->RegisterForAudioMixing(); @@ -470,6 +469,9 @@ MediaEngineWebRTCMicrophoneSource::Init() return; } + mSampleFrequency = MediaEngine::DEFAULT_SAMPLE_RATE; + LOG(("%s: sampling rate %u", __FUNCTION__, mSampleFrequency)); + // Check for availability. ScopedCustomReleasePtr ptrVoEHw(webrtc::VoEHardware::GetInterface(mVoiceEngine)); if (!ptrVoEHw || ptrVoEHw->SetRecordingDevice(mCapIndex)) { @@ -495,9 +497,10 @@ MediaEngineWebRTCMicrophoneSource::Init() webrtc::CodecInst codec; strcpy(codec.plname, ENCODING); codec.channels = CHANNELS; - codec.rate = SAMPLE_RATE; - codec.plfreq = SAMPLE_FREQUENCY; - codec.pacsize = SAMPLE_LENGTH; + MOZ_ASSERT(mSampleFrequency == 16000 || mSampleFrequency == 32000); + codec.rate = SAMPLE_RATE(mSampleFrequency); + codec.plfreq = mSampleFrequency; + codec.pacsize = SAMPLE_LENGTH(mSampleFrequency); codec.pltype = 0; // Default payload type if (!ptrVoECodec->SetSendCodec(mChannel, codec)) { diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 132a1f244f0..4cd5df0be83 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -48,6 +48,9 @@ #include "logging.h" +// Should come from MediaEngineWebRTC.h, but that's a pain to include here +#define DEFAULT_SAMPLE_RATE 32000 + using namespace mozilla; using namespace mozilla::gfx; using namespace mozilla::layers; @@ -1332,7 +1335,7 @@ void GenericReceiveListener::AddSelf(MediaSegment* segment) { MediaPipelineReceiveAudio::PipelineListener::PipelineListener( SourceMediaStream * source, TrackID track_id, const RefPtr& conduit, bool queue_track) - : GenericReceiveListener(source, track_id, 16000, queue_track), // XXX rate assumption + : GenericReceiveListener(source, track_id, DEFAULT_SAMPLE_RATE, queue_track), // XXX rate assumption conduit_(conduit) { MOZ_ASSERT(track_rate_%100 == 0); From 7789b473784c2210b62412b29ede9d9bf24424d6 Mon Sep 17 00:00:00 2001 From: Randell Jesup Date: Thu, 24 Sep 2015 09:23:37 -0400 Subject: [PATCH 53/78] Bug 953265: Adjust Opus bitrate in WebRTC to pass >8KHz audio, and comment r=bwc --- .../signaling/src/jsep/JsepSessionImpl.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp b/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp index 486b9cb7799..f72382659bd 100644 --- a/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp +++ b/media/webrtc/signaling/src/jsep/JsepSessionImpl.cpp @@ -2006,13 +2006,27 @@ void JsepSessionImpl::SetupDefaultCodecs() { // Supported audio codecs. + // Per jmspeex on IRC: + // For 32KHz sampling, 28 is ok, 32 is good, 40 should be really good + // quality. Note that 1-2Kbps will be wasted on a stereo Opus channel + // with mono input compared to configuring it for mono. + // If we reduce bitrate enough Opus will low-pass us; 16000 will kill a + // 9KHz tone. This should be adaptive when we're at the low-end of video + // bandwidth (say <100Kbps), and if we're audio-only, down to 8 or + // 12Kbps. mSupportedCodecs.values.push_back(new JsepAudioCodecDescription( "109", "opus", 48000, 2, 960, - 16000)); +#ifdef WEBRTC_GONK + // TODO Move this elsewhere to be adaptive to rate - Bug 1207925 + 16000 // B2G uses lower capture sampling rate +#else + 40000 +#endif + )); mSupportedCodecs.values.push_back(new JsepAudioCodecDescription( "9", From 36f1c10adededdcf61a278456747ebff11bc248b Mon Sep 17 00:00:00 2001 From: Randell Jesup Date: Thu, 24 Sep 2015 09:23:37 -0400 Subject: [PATCH 54/78] Bug 953265: make getUserMedia fake audio tones configurable in frequency via pref r=jib --- dom/media/MediaManager.cpp | 7 +++++-- dom/media/webrtc/MediaEngine.h | 1 + dom/media/webrtc/MediaEngineDefault.cpp | 18 ++++++++++-------- modules/libpref/init/all.js | 5 +++-- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 2c25016ce5f..aa6fd9a2850 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -1485,6 +1485,7 @@ MediaManager::MediaManager() : mMediaThread(nullptr) , mMutex("mozilla::MediaManager") , mBackend(nullptr) { + mPrefs.mFreq = 1000; // 1KHz test tone mPrefs.mWidth = 0; // adaptive default mPrefs.mHeight = 0; // adaptive default mPrefs.mFPS = MediaEngine::DEFAULT_VIDEO_FPS; @@ -1497,8 +1498,8 @@ MediaManager::MediaManager() GetPrefs(branch, nullptr); } } - LOG(("%s: default prefs: %dx%d @%dfps (min %d)", __FUNCTION__, - mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mMinFPS)); + LOG(("%s: default prefs: %dx%d @%dfps (min %d), %dHz test tones", __FUNCTION__, + mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mMinFPS, mPrefs.mFreq)); } NS_IMPL_ISUPPORTS(MediaManager, nsIMediaManagerService, nsIObserver) @@ -2512,6 +2513,7 @@ MediaManager::GetPrefs(nsIPrefBranch *aBranch, const char *aData) GetPref(aBranch, "media.navigator.video.default_height", aData, &mPrefs.mHeight); GetPref(aBranch, "media.navigator.video.default_fps", aData, &mPrefs.mFPS); GetPref(aBranch, "media.navigator.video.default_minfps", aData, &mPrefs.mMinFPS); + GetPref(aBranch, "media.navigator.audio.fake_frequency", aData, &mPrefs.mFreq); } nsresult @@ -2544,6 +2546,7 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic, prefs->RemoveObserver("media.navigator.video.default_height", this); prefs->RemoveObserver("media.navigator.video.default_fps", this); prefs->RemoveObserver("media.navigator.video.default_minfps", this); + prefs->RemoveObserver("media.navigator.audio.fake_frequency", this); } // Close off any remaining active windows. diff --git a/dom/media/webrtc/MediaEngine.h b/dom/media/webrtc/MediaEngine.h index 336d80501c6..477f57414bb 100644 --- a/dom/media/webrtc/MediaEngine.h +++ b/dom/media/webrtc/MediaEngine.h @@ -203,6 +203,7 @@ public: int32_t mHeight; int32_t mFPS; int32_t mMinFPS; + int32_t mFreq; // for test tones (fake:true) // mWidth and/or mHeight may be zero (=adaptive default), so use functions. diff --git a/dom/media/webrtc/MediaEngineDefault.cpp b/dom/media/webrtc/MediaEngineDefault.cpp index e7467ad1870..e760fd1af6f 100644 --- a/dom/media/webrtc/MediaEngineDefault.cpp +++ b/dom/media/webrtc/MediaEngineDefault.cpp @@ -302,15 +302,16 @@ class SineWaveGenerator { public: static const int bytesPerSample = 2; - static const int millisecondsPerSecond = 1000; - static const int frequency = 1000; + static const int millisecondsPerSecond = PR_MSEC_PER_SEC; - explicit SineWaveGenerator(int aSampleRate) : - mTotalLength(aSampleRate / frequency), + explicit SineWaveGenerator(uint32_t aSampleRate, uint32_t aFrequency) : + mTotalLength(aSampleRate / aFrequency), mReadLength(0) { - MOZ_ASSERT(mTotalLength * frequency == aSampleRate); + // If we allow arbitrary frequencies, there's no guarantee we won't get rounded here + // We could include an error term and adjust for it in generation; not worth the trouble + //MOZ_ASSERT(mTotalLength * aFrequency == aSampleRate); mAudioBuffer = new int16_t[mTotalLength]; - for(int i = 0; i < mTotalLength; i++) { + for (int i = 0; i < mTotalLength; i++) { // Set volume to -20db. It's from 32768.0 * 10^(-20/20) = 3276.8 mAudioBuffer[i] = (3276.8f * sin(2 * M_PI * i / mTotalLength)); } @@ -397,8 +398,9 @@ MediaEngineDefaultAudioSource::Allocate(const dom::MediaTrackConstraints &aConst } mState = kAllocated; - // generate 1Khz sine wave - mSineGenerator = new SineWaveGenerator(AUDIO_RATE); + // generate sine wave (default 1KHz) + mSineGenerator = new SineWaveGenerator(AUDIO_RATE, + static_cast(aPrefs.mFreq ? aPrefs.mFreq : 1000)); return NS_OK; } diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 2b2dfef2c01..3b82e349390 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -365,8 +365,8 @@ pref("media.webrtc.debug.log_file", ""); pref("media.webrtc.debug.aec_dump_max_size", 4194304); // 4MB #ifdef MOZ_WIDGET_GONK -pref("media.navigator.video.default_width",320); -pref("media.navigator.video.default_height",240); +pref("media.navigator.video.default_width", 320); +pref("media.navigator.video.default_height", 240); pref("media.peerconnection.enabled", true); pref("media.peerconnection.video.enabled", true); pref("media.navigator.video.max_fs", 1200); // 640x480 == 1200mb @@ -400,6 +400,7 @@ pref("media.peerconnection.video.min_bitrate", 200); pref("media.peerconnection.video.start_bitrate", 300); pref("media.peerconnection.video.max_bitrate", 2000); #endif +pref("media.navigator.audio.fake_frequency", 1000); pref("media.navigator.permission.disabled", false); pref("media.peerconnection.default_iceservers", "[]"); pref("media.peerconnection.ice.loopback", false); // Set only for testing in offline environments. From d0a7993b8d48cfa1cd8e372161f18753e9c2ca53 Mon Sep 17 00:00:00 2001 From: Ehsan Akhgari Date: Mon, 14 Sep 2015 13:29:52 -0400 Subject: [PATCH 55/78] Bug 1204581 - Add a deprecation warning for the usage of AppCache when service worker fetch interception is enabled; r=mcmanus,baku --- docshell/base/nsDocShell.cpp | 11 +++++++++ docshell/base/nsDocShell.h | 3 +++ dom/base/nsDeprecatedOperationList.h | 1 + dom/locales/en-US/chrome/dom/dom.properties | 4 +++- netwerk/base/moz.build | 1 + netwerk/base/nsIDeprecationWarner.idl | 23 ++++++++++++++++++ netwerk/protocol/http/HttpChannelChild.cpp | 13 +++++++++++ netwerk/protocol/http/HttpChannelChild.h | 3 +++ netwerk/protocol/http/HttpChannelParent.cpp | 10 +++++++- netwerk/protocol/http/HttpChannelParent.h | 3 +++ netwerk/protocol/http/PHttpChannel.ipdl | 3 +++ netwerk/protocol/http/nsHttpChannel.cpp | 26 +++++++++++++++++---- netwerk/protocol/http/nsHttpChannel.h | 2 ++ 13 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 netwerk/base/nsIDeprecationWarner.idl diff --git a/docshell/base/nsDocShell.cpp b/docshell/base/nsDocShell.cpp index 7c841de6ed4..e83797e5181 100644 --- a/docshell/base/nsDocShell.cpp +++ b/docshell/base/nsDocShell.cpp @@ -942,6 +942,7 @@ NS_INTERFACE_MAP_BEGIN(nsDocShell) NS_INTERFACE_MAP_ENTRY(nsIClipboardCommands) NS_INTERFACE_MAP_ENTRY(nsIDOMStorageManager) NS_INTERFACE_MAP_ENTRY(nsINetworkInterceptController) + NS_INTERFACE_MAP_ENTRY(nsIDeprecationWarner) NS_INTERFACE_MAP_END_INHERITING(nsDocLoader) NS_IMETHODIMP @@ -14060,3 +14061,13 @@ nsDocShell::InFrameSwap() } while (shell); return false; } + +NS_IMETHODIMP +nsDocShell::IssueWarning(uint32_t aWarning, bool aAsError) +{ + nsCOMPtr doc = mContentViewer->GetDocument(); + if (doc) { + doc->WarnOnceAbout(nsIDocument::DeprecatedOperations(aWarning), aAsError); + } + return NS_OK; +} diff --git a/docshell/base/nsDocShell.h b/docshell/base/nsDocShell.h index 943876232b3..3bf4a88bc3b 100644 --- a/docshell/base/nsDocShell.h +++ b/docshell/base/nsDocShell.h @@ -58,6 +58,7 @@ #include "prtime.h" #include "nsRect.h" #include "Units.h" +#include "nsIDeprecationWarner.h" namespace mozilla { namespace dom { @@ -145,6 +146,7 @@ class nsDocShell final , public nsIClipboardCommands , public nsIDOMStorageManager , public nsINetworkInterceptController + , public nsIDeprecationWarner , public mozilla::SupportsWeakPtr { friend class nsDSURIContentListener; @@ -176,6 +178,7 @@ public: NS_DECL_NSICLIPBOARDCOMMANDS NS_DECL_NSIWEBSHELLSERVICES NS_DECL_NSINETWORKINTERCEPTCONTROLLER + NS_DECL_NSIDEPRECATIONWARNER NS_FORWARD_SAFE_NSIDOMSTORAGEMANAGER(TopSessionStorageManager()) NS_IMETHOD Stop() override diff --git a/dom/base/nsDeprecatedOperationList.h b/dom/base/nsDeprecatedOperationList.h index fed7c176cc4..3adab2d940a 100644 --- a/dom/base/nsDeprecatedOperationList.h +++ b/dom/base/nsDeprecatedOperationList.h @@ -41,3 +41,4 @@ DEPRECATED_OPERATION(DataContainerEvent) DEPRECATED_OPERATION(Window_Controllers) DEPRECATED_OPERATION(ImportXULIntoContent) DEPRECATED_OPERATION(PannerNodeDoppler) +DEPRECATED_OPERATION(AppCache) diff --git a/dom/locales/en-US/chrome/dom/dom.properties b/dom/locales/en-US/chrome/dom/dom.properties index 2af33eba20f..1bf43d138c8 100644 --- a/dom/locales/en-US/chrome/dom/dom.properties +++ b/dom/locales/en-US/chrome/dom/dom.properties @@ -158,8 +158,10 @@ IndexedDBTransactionAbortNavigation=An IndexedDB transaction that was not yet co IgnoringWillChangeOverBudgetWarning=Will-change memory consumption is too high. Budget limit is the document surface area multiplied by %1$S (%2$S px). Occurrences of will-change over the budget will be ignored. # LOCALIZATION NOTE: Do not translate "ServiceWorker". HittingMaxWorkersPerDomain=A ServiceWorker could not be started immediately because other documents in the same origin are already using the maximum number of workers. The ServiceWorker is now queued and will be started after some of the other workers have completed. -# LOCALIZATION NOTE: Do no translate "setVelocity", "PannerNode", "AudioListener", "speedOfSound" and "dopplerFactor" +# LOCALIZATION NOTE: Do not translate "setVelocity", "PannerNode", "AudioListener", "speedOfSound" and "dopplerFactor" PannerNodeDopplerWarning=Use of setVelocity on the PannerNode and AudioListener, and speedOfSound and dopplerFactor on the AudioListener are deprecated and those members will be removed. For more help https://developer.mozilla.org/en-US/docs/Web/API/AudioListener#Deprecated_features +# LOCALIZATION NOTE: Do not translate "Application Cache API", "AppCache" and "ServiceWorker". +AppCacheWarning=The Application Cache API (AppCache) is deprecated and will be removed at a future date. Please consider using ServiceWorker for offline support. # LOCALIZATION NOTE: Do not translate "Worker". EmptyWorkerSourceWarning=Attempting to create a Worker from an empty source. This is probably unintentional. # LOCALIZATION NOTE: Do not translate "ServiceWorker". diff --git a/netwerk/base/moz.build b/netwerk/base/moz.build index d7e38fcae81..61ac28d1175 100644 --- a/netwerk/base/moz.build +++ b/netwerk/base/moz.build @@ -38,6 +38,7 @@ XPIDL_SOURCES += [ 'nsICryptoHMAC.idl', 'nsIDashboard.idl', 'nsIDashboardEventNotifier.idl', + 'nsIDeprecationWarner.idl', 'nsIDivertableChannel.idl', 'nsIDownloader.idl', 'nsIEncodedChannel.idl', diff --git a/netwerk/base/nsIDeprecationWarner.idl b/netwerk/base/nsIDeprecationWarner.idl new file mode 100644 index 00000000000..72303b81567 --- /dev/null +++ b/netwerk/base/nsIDeprecationWarner.idl @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * Interface for warning about deprecated operations. Consumers should + * attach this interface to the channel's notification callbacks/loadgroup. + */ +[uuid(665c5124-2c52-41ba-ae72-2393f8e76c25)] +interface nsIDeprecationWarner : nsISupports +{ + /** + * Issue a deprecation warning. + * + * @param aWarning a warning code as declared in nsDeprecatedOperationList.h. + * @param aAsError optional boolean flag indicating whether the warning + * should be treated as an error. + */ + void issueWarning(in uint32_t aWarning, [optional] in bool aAsError); +}; diff --git a/netwerk/protocol/http/HttpChannelChild.cpp b/netwerk/protocol/http/HttpChannelChild.cpp index e5b611fd3f7..ef012331ad5 100644 --- a/netwerk/protocol/http/HttpChannelChild.cpp +++ b/netwerk/protocol/http/HttpChannelChild.cpp @@ -34,6 +34,7 @@ #include "nsPerformance.h" #include "mozIThirdPartyUtil.h" #include "nsContentSecurityManager.h" +#include "nsIDeprecationWarner.h" #ifdef OS_POSIX #include "chrome/common/file_descriptor_set_posix.h" @@ -2319,5 +2320,17 @@ HttpChannelChild::ForceIntercepted() return NS_OK; } +bool +HttpChannelChild::RecvIssueDeprecationWarning(const uint32_t& warning, + const bool& asError) +{ + nsCOMPtr warner; + GetCallback(warner); + if (warner) { + warner->IssueWarning(warning, asError); + } + return true; +} + } // namespace net } // namespace mozilla diff --git a/netwerk/protocol/http/HttpChannelChild.h b/netwerk/protocol/http/HttpChannelChild.h index 79406a91c6a..541dbec1d92 100644 --- a/netwerk/protocol/http/HttpChannelChild.h +++ b/netwerk/protocol/http/HttpChannelChild.h @@ -148,6 +148,9 @@ protected: bool RecvReportSecurityMessage(const nsString& messageTag, const nsString& messageCategory) override; + bool RecvIssueDeprecationWarning(const uint32_t& warning, + const bool& asError) override; + bool GetAssociatedContentSecurity(nsIAssociatedContentSecurity** res = nullptr); virtual void DoNotifyListenerCleanup() override; diff --git a/netwerk/protocol/http/HttpChannelParent.cpp b/netwerk/protocol/http/HttpChannelParent.cpp index 7100e213795..1a4b24e7500 100644 --- a/netwerk/protocol/http/HttpChannelParent.cpp +++ b/netwerk/protocol/http/HttpChannelParent.cpp @@ -153,7 +153,8 @@ NS_IMPL_ISUPPORTS(HttpChannelParent, nsIParentChannel, nsIAuthPromptProvider, nsIParentRedirectingChannel, - nsINetworkInterceptController) + nsINetworkInterceptController, + nsIDeprecationWarner) NS_IMETHODIMP HttpChannelParent::ShouldPrepareForIntercept(nsIURI* aURI, bool aIsNavigate, @@ -1579,5 +1580,12 @@ HttpChannelParent::ReportSecurityMessage(const nsAString& aMessageTag, return NS_OK; } +NS_IMETHODIMP +HttpChannelParent::IssueWarning(uint32_t aWarning, bool aAsError) +{ + unused << SendIssueDeprecationWarning(aWarning, aAsError); + return NS_OK; +} + } // namespace net } // namespace mozilla diff --git a/netwerk/protocol/http/HttpChannelParent.h b/netwerk/protocol/http/HttpChannelParent.h index d91b8d72332..fb4105184c3 100644 --- a/netwerk/protocol/http/HttpChannelParent.h +++ b/netwerk/protocol/http/HttpChannelParent.h @@ -21,6 +21,7 @@ #include "nsIAuthPromptProvider.h" #include "mozilla/dom/ipc/IdType.h" #include "nsINetworkInterceptController.h" +#include "nsIDeprecationWarner.h" class nsICacheEntry; class nsIAssociatedContentSecurity; @@ -43,6 +44,7 @@ class HttpChannelParent final : public PHttpChannelParent , public ADivertableParentChannel , public nsIAuthPromptProvider , public nsINetworkInterceptController + , public nsIDeprecationWarner , public DisconnectableParent , public HttpChannelSecurityWarningReporter { @@ -58,6 +60,7 @@ public: NS_DECL_NSIINTERFACEREQUESTOR NS_DECL_NSIAUTHPROMPTPROVIDER NS_DECL_NSINETWORKINTERCEPTCONTROLLER + NS_DECL_NSIDEPRECATIONWARNER HttpChannelParent(const dom::PBrowserOrId& iframeEmbedding, nsILoadContext* aLoadContext, diff --git a/netwerk/protocol/http/PHttpChannel.ipdl b/netwerk/protocol/http/PHttpChannel.ipdl index 1456b961acf..04a24579a72 100644 --- a/netwerk/protocol/http/PHttpChannel.ipdl +++ b/netwerk/protocol/http/PHttpChannel.ipdl @@ -154,6 +154,9 @@ child: // Tell child to delete channel (all IPDL deletes must be done from child to // avoid races: see bug 591708). DeleteSelf(); + + // Tell the child to issue a deprecation warning. + IssueDeprecationWarning(uint32_t warning, bool asError); }; diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp index e606851f522..75444f2325c 100644 --- a/netwerk/protocol/http/nsHttpChannel.cpp +++ b/netwerk/protocol/http/nsHttpChannel.cpp @@ -87,6 +87,8 @@ #include "ScopedNSSTypes.h" #include "nsNullPrincipal.h" #include "nsIPackagedAppService.h" +#include "nsIDeprecationWarner.h" +#include "nsIDocument.h" namespace mozilla { namespace net { @@ -2822,8 +2824,7 @@ nsHttpChannel::ContinueProcessFallback(nsresult rv) NS_ENSURE_SUCCESS(rv, rv); if (mLoadFlags & LOAD_INITIAL_DOCUMENT_URI) { - Telemetry::Accumulate(Telemetry::HTTP_OFFLINE_CACHE_DOCUMENT_LOAD, - true); + MaybeWarnAboutAppCache(); } // close down this channel @@ -3623,8 +3624,7 @@ nsHttpChannel::OnOfflineCacheEntryAvailable(nsICacheEntry *aEntry, mCacheEntryIsWriteOnly = false; if (mLoadFlags & LOAD_INITIAL_DOCUMENT_URI && !mApplicationCacheForWrite) { - Telemetry::Accumulate(Telemetry::HTTP_OFFLINE_CACHE_DOCUMENT_LOAD, - true); + MaybeWarnAboutAppCache(); } return NS_OK; @@ -7026,5 +7026,23 @@ nsHttpChannel::OnPreflightFailed(nsresult aError) return NS_OK; } +void +nsHttpChannel::MaybeWarnAboutAppCache() +{ + // First, accumulate a telemetry ping about appcache usage. + Telemetry::Accumulate(Telemetry::HTTP_OFFLINE_CACHE_DOCUMENT_LOAD, + true); + + // Then, issue a deprecation warning if service worker interception is + // enabled. + if (nsContentUtils::ServiceWorkerInterceptionEnabled()) { + nsCOMPtr warner; + GetCallback(warner); + if (warner) { + warner->IssueWarning(nsIDocument::eAppCache, false); + } + } +} + } // namespace net } // namespace mozilla diff --git a/netwerk/protocol/http/nsHttpChannel.h b/netwerk/protocol/http/nsHttpChannel.h index 13d7ed7cec9..9ce5049131c 100644 --- a/netwerk/protocol/http/nsHttpChannel.h +++ b/netwerk/protocol/http/nsHttpChannel.h @@ -398,6 +398,8 @@ private: void SetPushedStream(Http2PushedStream *stream); + void MaybeWarnAboutAppCache(); + private: nsCOMPtr mProxyRequest; From b71b7c6d4dffbd18524618015af754964e766dd3 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Wed, 23 Sep 2015 23:07:29 -0400 Subject: [PATCH 56/78] Bug 1207245 - part 0 - fix why-did-we-allow-that tests in TestRefPtr.cpp RefPtr.h's byref permits callees to see the incoming value of the outparam; XPCOM's getter_AddRefs zeros outparams prior to the call, so information doesn't leak through inadvertently. Given this difference, we need to eliminate tests that depended on this (arguably dangerous) behavior. The numerous assertion fixups are required because we're removing construction and destruction of objects along the way. --- mfbt/tests/TestRefPtr.cpp | 35 +++++------------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/mfbt/tests/TestRefPtr.cpp b/mfbt/tests/TestRefPtr.cpp index 44e1b416bef..2ccb0e5bbb1 100644 --- a/mfbt/tests/TestRefPtr.cpp +++ b/mfbt/tests/TestRefPtr.cpp @@ -54,23 +54,12 @@ GetNewFoo(Foo** aFoo) (*aFoo)->AddRef(); } -void -GetPassedFoo(Foo** aFoo) -{ - // Kids, don't try this at home - (*aFoo)->AddRef(); -} - void GetNewFoo(RefPtr* aFoo) { *aFoo = new Bar(); } -void -GetPassedFoo(RefPtr* aFoo) -{} - already_AddRefed GetNullFoo() { @@ -127,37 +116,23 @@ main() } MOZ_RELEASE_ASSERT(8 == Foo::sNumDestroyed); - { - RefPtr f = new Foo(); - GetPassedFoo(byRef(f)); - MOZ_RELEASE_ASSERT(8 == Foo::sNumDestroyed); - } - MOZ_RELEASE_ASSERT(9 == Foo::sNumDestroyed); - { RefPtr f = new Foo(); GetNewFoo(&f); - MOZ_RELEASE_ASSERT(10 == Foo::sNumDestroyed); + MOZ_RELEASE_ASSERT(9 == Foo::sNumDestroyed); } - MOZ_RELEASE_ASSERT(11 == Foo::sNumDestroyed); - - { - RefPtr f = new Foo(); - GetPassedFoo(&f); - MOZ_RELEASE_ASSERT(11 == Foo::sNumDestroyed); - } - MOZ_RELEASE_ASSERT(12 == Foo::sNumDestroyed); + MOZ_RELEASE_ASSERT(10 == Foo::sNumDestroyed); { RefPtr f1 = new Bar(); } - MOZ_RELEASE_ASSERT(13 == Foo::sNumDestroyed); + MOZ_RELEASE_ASSERT(11 == Foo::sNumDestroyed); { RefPtr f = GetNullFoo(); - MOZ_RELEASE_ASSERT(13 == Foo::sNumDestroyed); + MOZ_RELEASE_ASSERT(11 == Foo::sNumDestroyed); } - MOZ_RELEASE_ASSERT(13 == Foo::sNumDestroyed); + MOZ_RELEASE_ASSERT(11 == Foo::sNumDestroyed); return 0; } From 490e851ea9b4dac48ec1a12e5cd67318f292691b Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Tue, 22 Sep 2015 21:27:34 -0400 Subject: [PATCH 57/78] Bug 1207245 - part 1 - move RefCounted to its own file Various bits depend on RefPtr.h to provide RefCounted and RefPtr. It will be easier to manage an automatic conversion from RefPtr to nsRefPtr if we split out the dependency on RefCounted first. --- devtools/shared/heapsnapshot/HeapSnapshot.h | 1 + gfx/2d/GenericRefCounted.h | 1 + image/SourceBuffer.h | 1 + memory/volatile/VolatileBuffer.h | 1 + mfbt/RefCounted.h | 215 ++++++++++++++++++++ mfbt/RefPtr.h | 182 +---------------- mfbt/WeakPtr.h | 1 + mfbt/moz.build | 1 + mfbt/tests/TestRefPtr.cpp | 1 + tools/profiler/core/ProfileBuffer.h | 1 + 10 files changed, 224 insertions(+), 181 deletions(-) create mode 100644 mfbt/RefCounted.h diff --git a/devtools/shared/heapsnapshot/HeapSnapshot.h b/devtools/shared/heapsnapshot/HeapSnapshot.h index 5a3d4de5ce8..8c4de6fb2a2 100644 --- a/devtools/shared/heapsnapshot/HeapSnapshot.h +++ b/devtools/shared/heapsnapshot/HeapSnapshot.h @@ -12,6 +12,7 @@ #include "mozilla/dom/BindingDeclarations.h" #include "mozilla/HashFunctions.h" #include "mozilla/Maybe.h" +#include "mozilla/RefCounted.h" #include "mozilla/RefPtr.h" #include "mozilla/TimeStamp.h" #include "mozilla/UniquePtr.h" diff --git a/gfx/2d/GenericRefCounted.h b/gfx/2d/GenericRefCounted.h index b1de3f754ba..bee792b4de7 100644 --- a/gfx/2d/GenericRefCounted.h +++ b/gfx/2d/GenericRefCounted.h @@ -11,6 +11,7 @@ #define MOZILLA_GENERICREFCOUNTED_H_ #include "mozilla/RefPtr.h" +#include "mozilla/RefCounted.h" namespace mozilla { diff --git a/image/SourceBuffer.h b/image/SourceBuffer.h index cbac16b074d..66458823ec0 100644 --- a/image/SourceBuffer.h +++ b/image/SourceBuffer.h @@ -17,6 +17,7 @@ #include "mozilla/Move.h" #include "mozilla/MemoryReporting.h" #include "mozilla/RefPtr.h" +#include "mozilla/RefCounted.h" #include "mozilla/UniquePtr.h" #include "mozilla/nsRefPtr.h" #include "nsTArray.h" diff --git a/memory/volatile/VolatileBuffer.h b/memory/volatile/VolatileBuffer.h index 9fe692f4705..afb075c7253 100644 --- a/memory/volatile/VolatileBuffer.h +++ b/memory/volatile/VolatileBuffer.h @@ -9,6 +9,7 @@ #include "mozilla/Mutex.h" #include "mozilla/RefPtr.h" #include "mozilla/MemoryReporting.h" +#include "mozilla/RefCounted.h" /* VolatileBuffer * diff --git a/mfbt/RefCounted.h b/mfbt/RefCounted.h new file mode 100644 index 00000000000..5532cffc4dc --- /dev/null +++ b/mfbt/RefCounted.h @@ -0,0 +1,215 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* CRTP refcounting templates. Do not use unless you are an Expert. */ + +#ifndef mozilla_RefCounted_h +#define mozilla_RefCounted_h + +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/Move.h" +#include "mozilla/RefCountType.h" +#include "mozilla/TypeTraits.h" + +#if defined(MOZILLA_INTERNAL_API) +#include "nsXPCOM.h" +#endif + +#if defined(MOZILLA_INTERNAL_API) && \ + !defined(MOZILLA_XPCOMRT_API) && \ + (defined(DEBUG) || defined(FORCE_BUILD_REFCNT_LOGGING)) +#define MOZ_REFCOUNTED_LEAK_CHECKING +#endif + +namespace mozilla { + +template class RefPtr; + +/** + * RefCounted is a sort of a "mixin" for a class T. RefCounted + * manages, well, refcounting for T, and because RefCounted is + * parameterized on T, RefCounted can call T's destructor directly. + * This means T doesn't need to have a virtual dtor and so doesn't + * need a vtable. + * + * RefCounted is created with refcount == 0. Newly-allocated + * RefCounted must immediately be assigned to a RefPtr to make the + * refcount > 0. It's an error to allocate and free a bare + * RefCounted, i.e. outside of the RefPtr machinery. Attempts to + * do so will abort DEBUG builds. + * + * Live RefCounted have refcount > 0. The lifetime (refcounts) of + * live RefCounted are controlled by nsRefPtr and + * nsRefPtr. Upon a transition from refcounted==1 + * to 0, the RefCounted "dies" and is destroyed. The "destroyed" + * state is represented in DEBUG builds by refcount==0xffffdead. This + * state distinguishes use-before-ref (refcount==0) from + * use-after-destroy (refcount==0xffffdead). + * + * Note that when deriving from RefCounted or AtomicRefCounted, you + * should add MOZ_DECLARE_REFCOUNTED_TYPENAME(ClassName) to the public + * section of your class, where ClassName is the name of your class. + */ +namespace detail { +const MozRefCountType DEAD = 0xffffdead; + +// When building code that gets compiled into Gecko, try to use the +// trace-refcount leak logging facilities. +#ifdef MOZ_REFCOUNTED_LEAK_CHECKING +class RefCountLogger +{ +public: + static void logAddRef(const void* aPointer, MozRefCountType aRefCount, + const char* aTypeName, uint32_t aInstanceSize) + { + MOZ_ASSERT(aRefCount != DEAD); + NS_LogAddRef(const_cast(aPointer), aRefCount, aTypeName, + aInstanceSize); + } + + static void logRelease(const void* aPointer, MozRefCountType aRefCount, + const char* aTypeName) + { + MOZ_ASSERT(aRefCount != DEAD); + NS_LogRelease(const_cast(aPointer), aRefCount, aTypeName); + } +}; +#endif + +// This is used WeakPtr.h as well as this file. +enum RefCountAtomicity +{ + AtomicRefCount, + NonAtomicRefCount +}; + +template +class RefCounted +{ + friend class RefPtr; + +protected: + RefCounted() : mRefCnt(0) {} + ~RefCounted() { MOZ_ASSERT(mRefCnt == detail::DEAD); } + +public: + // Compatibility with nsRefPtr. + void AddRef() const + { + // Note: this method must be thread safe for AtomicRefCounted. + MOZ_ASSERT(int32_t(mRefCnt) >= 0); +#ifndef MOZ_REFCOUNTED_LEAK_CHECKING + ++mRefCnt; +#else + const char* type = static_cast(this)->typeName(); + uint32_t size = static_cast(this)->typeSize(); + const void* ptr = static_cast(this); + MozRefCountType cnt = ++mRefCnt; + detail::RefCountLogger::logAddRef(ptr, cnt, type, size); +#endif + } + + void Release() const + { + // Note: this method must be thread safe for AtomicRefCounted. + MOZ_ASSERT(int32_t(mRefCnt) > 0); +#ifndef MOZ_REFCOUNTED_LEAK_CHECKING + MozRefCountType cnt = --mRefCnt; +#else + const char* type = static_cast(this)->typeName(); + const void* ptr = static_cast(this); + MozRefCountType cnt = --mRefCnt; + // Note: it's not safe to touch |this| after decrementing the refcount, + // except for below. + detail::RefCountLogger::logRelease(ptr, cnt, type); +#endif + if (0 == cnt) { + // Because we have atomically decremented the refcount above, only + // one thread can get a 0 count here, so as long as we can assume that + // everything else in the system is accessing this object through + // RefPtrs, it's safe to access |this| here. +#ifdef DEBUG + mRefCnt = detail::DEAD; +#endif + delete static_cast(this); + } + } + + // Compatibility with wtf::RefPtr. + void ref() { AddRef(); } + void deref() { Release(); } + MozRefCountType refCount() const { return mRefCnt; } + bool hasOneRef() const + { + MOZ_ASSERT(mRefCnt > 0); + return mRefCnt == 1; + } + +private: + mutable typename Conditional, + MozRefCountType>::Type mRefCnt; +}; + +#ifdef MOZ_REFCOUNTED_LEAK_CHECKING +// Passing override for the optional argument marks the typeName and +// typeSize functions defined by this macro as overrides. +#define MOZ_DECLARE_REFCOUNTED_VIRTUAL_TYPENAME(T, ...) \ + virtual const char* typeName() const __VA_ARGS__ { return #T; } \ + virtual size_t typeSize() const __VA_ARGS__ { return sizeof(*this); } +#else +#define MOZ_DECLARE_REFCOUNTED_VIRTUAL_TYPENAME(T, ...) +#endif + +// Note that this macro is expanded unconditionally because it declares only +// two small inline functions which will hopefully get eliminated by the linker +// in non-leak-checking builds. +#define MOZ_DECLARE_REFCOUNTED_TYPENAME(T) \ + const char* typeName() const { return #T; } \ + size_t typeSize() const { return sizeof(*this); } + +} // namespace detail + +template +class RefCounted : public detail::RefCounted +{ +public: + ~RefCounted() + { + static_assert(IsBaseOf::value, + "T must derive from RefCounted"); + } +}; + +namespace external { + +/** + * AtomicRefCounted is like RefCounted, with an atomically updated + * reference counter. + * + * NOTE: Please do not use this class, use NS_INLINE_DECL_THREADSAFE_REFCOUNTING + * instead. + */ +template +class AtomicRefCounted : + public mozilla::detail::RefCounted +{ +public: + ~AtomicRefCounted() + { + static_assert(IsBaseOf::value, + "T must derive from AtomicRefCounted"); + } +}; + +} // namespace external + +} // namespace mozilla + +#endif // mozilla_RefCounted_h diff --git a/mfbt/RefPtr.h b/mfbt/RefPtr.h index 51a269258fd..48aa0c215ae 100644 --- a/mfbt/RefPtr.h +++ b/mfbt/RefPtr.h @@ -14,6 +14,7 @@ #include "mozilla/Atomics.h" #include "mozilla/Attributes.h" #include "mozilla/Move.h" +#include "mozilla/RefCounted.h" #include "mozilla/RefCountType.h" #include "mozilla/nsRefPtr.h" #include "mozilla/TypeTraits.h" @@ -29,190 +30,9 @@ namespace mozilla { -template class RefCounted; -template class RefPtr; template class OutParamRef; template OutParamRef byRef(RefPtr&); -/** - * RefCounted is a sort of a "mixin" for a class T. RefCounted - * manages, well, refcounting for T, and because RefCounted is - * parameterized on T, RefCounted can call T's destructor directly. - * This means T doesn't need to have a virtual dtor and so doesn't - * need a vtable. - * - * RefCounted is created with refcount == 0. Newly-allocated - * RefCounted must immediately be assigned to a RefPtr to make the - * refcount > 0. It's an error to allocate and free a bare - * RefCounted, i.e. outside of the RefPtr machinery. Attempts to - * do so will abort DEBUG builds. - * - * Live RefCounted have refcount > 0. The lifetime (refcounts) of - * live RefCounted are controlled by RefPtr and - * RefPtr. Upon a transition from refcounted==1 - * to 0, the RefCounted "dies" and is destroyed. The "destroyed" - * state is represented in DEBUG builds by refcount==0xffffdead. This - * state distinguishes use-before-ref (refcount==0) from - * use-after-destroy (refcount==0xffffdead). - * - * Note that when deriving from RefCounted or AtomicRefCounted, you - * should add MOZ_DECLARE_REFCOUNTED_TYPENAME(ClassName) to the public - * section of your class, where ClassName is the name of your class. - */ -namespace detail { -const MozRefCountType DEAD = 0xffffdead; - -// When building code that gets compiled into Gecko, try to use the -// trace-refcount leak logging facilities. -#ifdef MOZ_REFCOUNTED_LEAK_CHECKING -class RefCountLogger -{ -public: - static void logAddRef(const void* aPointer, MozRefCountType aRefCount, - const char* aTypeName, uint32_t aInstanceSize) - { - MOZ_ASSERT(aRefCount != DEAD); - NS_LogAddRef(const_cast(aPointer), aRefCount, aTypeName, - aInstanceSize); - } - - static void logRelease(const void* aPointer, MozRefCountType aRefCount, - const char* aTypeName) - { - MOZ_ASSERT(aRefCount != DEAD); - NS_LogRelease(const_cast(aPointer), aRefCount, aTypeName); - } -}; -#endif - -// This is used WeakPtr.h as well as this file. -enum RefCountAtomicity -{ - AtomicRefCount, - NonAtomicRefCount -}; - -template -class RefCounted -{ - friend class RefPtr; - -protected: - RefCounted() : mRefCnt(0) {} - ~RefCounted() { MOZ_ASSERT(mRefCnt == detail::DEAD); } - -public: - // Compatibility with nsRefPtr. - void AddRef() const - { - // Note: this method must be thread safe for AtomicRefCounted. - MOZ_ASSERT(int32_t(mRefCnt) >= 0); -#ifndef MOZ_REFCOUNTED_LEAK_CHECKING - ++mRefCnt; -#else - const char* type = static_cast(this)->typeName(); - uint32_t size = static_cast(this)->typeSize(); - const void* ptr = static_cast(this); - MozRefCountType cnt = ++mRefCnt; - detail::RefCountLogger::logAddRef(ptr, cnt, type, size); -#endif - } - - void Release() const - { - // Note: this method must be thread safe for AtomicRefCounted. - MOZ_ASSERT(int32_t(mRefCnt) > 0); -#ifndef MOZ_REFCOUNTED_LEAK_CHECKING - MozRefCountType cnt = --mRefCnt; -#else - const char* type = static_cast(this)->typeName(); - const void* ptr = static_cast(this); - MozRefCountType cnt = --mRefCnt; - // Note: it's not safe to touch |this| after decrementing the refcount, - // except for below. - detail::RefCountLogger::logRelease(ptr, cnt, type); -#endif - if (0 == cnt) { - // Because we have atomically decremented the refcount above, only - // one thread can get a 0 count here, so as long as we can assume that - // everything else in the system is accessing this object through - // RefPtrs, it's safe to access |this| here. -#ifdef DEBUG - mRefCnt = detail::DEAD; -#endif - delete static_cast(this); - } - } - - // Compatibility with wtf::RefPtr. - void ref() { AddRef(); } - void deref() { Release(); } - MozRefCountType refCount() const { return mRefCnt; } - bool hasOneRef() const - { - MOZ_ASSERT(mRefCnt > 0); - return mRefCnt == 1; - } - -private: - mutable typename Conditional, - MozRefCountType>::Type mRefCnt; -}; - -#ifdef MOZ_REFCOUNTED_LEAK_CHECKING -// Passing override for the optional argument marks the typeName and -// typeSize functions defined by this macro as overrides. -#define MOZ_DECLARE_REFCOUNTED_VIRTUAL_TYPENAME(T, ...) \ - virtual const char* typeName() const __VA_ARGS__ { return #T; } \ - virtual size_t typeSize() const __VA_ARGS__ { return sizeof(*this); } -#else -#define MOZ_DECLARE_REFCOUNTED_VIRTUAL_TYPENAME(T, ...) -#endif - -// Note that this macro is expanded unconditionally because it declares only -// two small inline functions which will hopefully get eliminated by the linker -// in non-leak-checking builds. -#define MOZ_DECLARE_REFCOUNTED_TYPENAME(T) \ - const char* typeName() const { return #T; } \ - size_t typeSize() const { return sizeof(*this); } - -} // namespace detail - -template -class RefCounted : public detail::RefCounted -{ -public: - ~RefCounted() - { - static_assert(IsBaseOf::value, - "T must derive from RefCounted"); - } -}; - -namespace external { - -/** - * AtomicRefCounted is like RefCounted, with an atomically updated - * reference counter. - * - * NOTE: Please do not use this class, use NS_INLINE_DECL_THREADSAFE_REFCOUNTING - * instead. - */ -template -class AtomicRefCounted : - public mozilla::detail::RefCounted -{ -public: - ~AtomicRefCounted() - { - static_assert(IsBaseOf::value, - "T must derive from AtomicRefCounted"); - } -}; - -} // namespace external - /** * RefPtr points to a refcounted thing that has AddRef and Release * methods to increase/decrease the refcount, respectively. After a diff --git a/mfbt/WeakPtr.h b/mfbt/WeakPtr.h index 22ba20e0652..8b1ee0e228d 100644 --- a/mfbt/WeakPtr.h +++ b/mfbt/WeakPtr.h @@ -70,6 +70,7 @@ #include "mozilla/ArrayUtils.h" #include "mozilla/Assertions.h" #include "mozilla/Attributes.h" +#include "mozilla/RefCounted.h" #include "mozilla/RefPtr.h" #include "mozilla/TypeTraits.h" diff --git a/mfbt/moz.build b/mfbt/moz.build index 5de80c48de7..c7d23819b52 100644 --- a/mfbt/moz.build +++ b/mfbt/moz.build @@ -67,6 +67,7 @@ EXPORTS.mozilla = [ 'Range.h', 'RangedArray.h', 'RangedPtr.h', + 'RefCounted.h', 'RefCountType.h', 'ReentrancyGuard.h', 'RefPtr.h', diff --git a/mfbt/tests/TestRefPtr.cpp b/mfbt/tests/TestRefPtr.cpp index 2ccb0e5bbb1..316f91df2bc 100644 --- a/mfbt/tests/TestRefPtr.cpp +++ b/mfbt/tests/TestRefPtr.cpp @@ -5,6 +5,7 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/RefPtr.h" +#include "mozilla/RefCounted.h" using mozilla::RefCounted; using mozilla::RefPtr; diff --git a/tools/profiler/core/ProfileBuffer.h b/tools/profiler/core/ProfileBuffer.h index bc69abfe4a9..1426908f209 100644 --- a/tools/profiler/core/ProfileBuffer.h +++ b/tools/profiler/core/ProfileBuffer.h @@ -10,6 +10,7 @@ #include "platform.h" #include "ProfileJSONWriter.h" #include "mozilla/RefPtr.h" +#include "mozilla/RefCounted.h" class ProfileBuffer : public mozilla::RefCounted { public: From 19a323acbd0a1e16765dd566b42944cf97275dd1 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Tue, 22 Sep 2015 22:23:10 -0400 Subject: [PATCH 58/78] Bug 1207245 - part 2 - move MakeAndAddRef to nsRefPtr.h A number of places depend on RefPtr.h providing this function. When we s/RefPtr/nsRefPtr/, such places still need to be able to see this function. Moving it to nsRefPtr.h makes it still visible before we switch (since RefPtr.h includes nsRefPtr.h), and after we switch (since every place that #includes RefPtr.h will now be #including nsRefPtr.h). --- mfbt/RefPtr.h | 17 ----------------- mfbt/nsRefPtr.h | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/mfbt/RefPtr.h b/mfbt/RefPtr.h index 48aa0c215ae..a1ad86031ee 100644 --- a/mfbt/RefPtr.h +++ b/mfbt/RefPtr.h @@ -190,23 +190,6 @@ byRef(RefPtr& aPtr) return OutParamRef(aPtr); } -/** - * Helper function to be able to conveniently write things like: - * - * already_AddRefed - * f(...) - * { - * return MakeAndAddRef(...); - * } - */ -template -already_AddRefed -MakeAndAddRef(Args&&... aArgs) -{ - RefPtr p(new T(Forward(aArgs)...)); - return p.forget(); -} - } // namespace mozilla // Declared in nsRefPtr.h diff --git a/mfbt/nsRefPtr.h b/mfbt/nsRefPtr.h index c3440b289c8..0ba5a69c2c9 100644 --- a/mfbt/nsRefPtr.h +++ b/mfbt/nsRefPtr.h @@ -591,4 +591,25 @@ do_AddRef(T*&& aObj) return ref.forget(); } +namespace mozilla { + +/** + * Helper function to be able to conveniently write things like: + * + * already_AddRefed + * f(...) + * { + * return MakeAndAddRef(...); + * } + */ +template +already_AddRefed +MakeAndAddRef(Args&&... aArgs) +{ + nsRefPtr p(new T(Forward(aArgs)...)); + return p.forget(); +} + +} // namespace mozilla + #endif /* mozilla_nsRefPtr_h */ From ddd6a49e603c3d4541feeab769526108ae29edc0 Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Fri, 18 Sep 2015 11:50:37 +0800 Subject: [PATCH 59/78] Bug 1178518 - Packaged App Utils. r=valentin --- b2g/installer/package-manifest.in | 2 + browser/installer/package-manifest.in | 2 + .../b2gdroid/installer/package-manifest.in | 2 + mobile/android/installer/package-manifest.in | 2 + netwerk/base/moz.build | 1 + netwerk/base/nsIPackagedAppUtils.idl | 68 +++++++++ netwerk/protocol/http/PackagedAppUtils.js | 109 ++++++++++++++ .../protocol/http/PackagedAppUtils.manifest | 3 + netwerk/protocol/http/moz.build | 5 + netwerk/test/unit/test_packaged_app_utils.js | 135 ++++++++++++++++++ netwerk/test/unit/xpcshell.ini | 1 + 11 files changed, 330 insertions(+) create mode 100644 netwerk/base/nsIPackagedAppUtils.idl create mode 100644 netwerk/protocol/http/PackagedAppUtils.js create mode 100644 netwerk/protocol/http/PackagedAppUtils.manifest create mode 100644 netwerk/test/unit/test_packaged_app_utils.js diff --git a/b2g/installer/package-manifest.in b/b2g/installer/package-manifest.in index 7951bb470ff..a8c6c3ad3e6 100644 --- a/b2g/installer/package-manifest.in +++ b/b2g/installer/package-manifest.in @@ -410,6 +410,8 @@ @RESPATH@/components/AlarmsManager.manifest @RESPATH@/components/FeedProcessor.manifest @RESPATH@/components/FeedProcessor.js +@RESPATH@/components/PackagedAppUtils.manifest +@RESPATH@/components/PackagedAppUtils.js @RESPATH@/components/BrowserFeeds.manifest @RESPATH@/components/FeedConverter.js @RESPATH@/components/FeedWriter.js diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index aaf28ca4a54..f006c903dcb 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -369,6 +369,8 @@ @RESPATH@/components/BrowserElementParent.js @RESPATH@/components/FeedProcessor.manifest @RESPATH@/components/FeedProcessor.js +@RESPATH@/components/PackagedAppUtils.js +@RESPATH@/components/PackagedAppUtils.manifest @RESPATH@/browser/components/BrowserFeeds.manifest @RESPATH@/browser/components/FeedConverter.js @RESPATH@/browser/components/FeedWriter.js diff --git a/mobile/android/b2gdroid/installer/package-manifest.in b/mobile/android/b2gdroid/installer/package-manifest.in index 94ff64b2f38..e8d92b78568 100644 --- a/mobile/android/b2gdroid/installer/package-manifest.in +++ b/mobile/android/b2gdroid/installer/package-manifest.in @@ -308,6 +308,8 @@ @BINPATH@/components/SettingsService.manifest @BINPATH@/components/BrowserElementParent.manifest @BINPATH@/components/BrowserElementParent.js +@BINPATH@/components/PackagedAppUtils.manifest +@BINPATH@/components/PackagedAppUtils.js @BINPATH@/components/BrowserFeeds.manifest @BINPATH@/components/FeedConverter.js @BINPATH@/components/FeedWriter.js diff --git a/mobile/android/installer/package-manifest.in b/mobile/android/installer/package-manifest.in index ce422257b08..831b0394889 100644 --- a/mobile/android/installer/package-manifest.in +++ b/mobile/android/installer/package-manifest.in @@ -311,6 +311,8 @@ @BINPATH@/components/BrowserElementParent.js @BINPATH@/components/FeedProcessor.manifest @BINPATH@/components/FeedProcessor.js +@BINPATH@/components/PackagedAppUtils.manifest +@BINPATH@/components/PackagedAppUtils.js @BINPATH@/components/BrowserFeeds.manifest @BINPATH@/components/FeedConverter.js @BINPATH@/components/FeedWriter.js diff --git a/netwerk/base/moz.build b/netwerk/base/moz.build index 61ac28d1175..ba946368eef 100644 --- a/netwerk/base/moz.build +++ b/netwerk/base/moz.build @@ -70,6 +70,7 @@ XPIDL_SOURCES += [ 'nsINullChannel.idl', 'nsIPACGenerator.idl', 'nsIPackagedAppService.idl', + 'nsIPackagedAppUtils.idl', 'nsIPackagedAppVerifier.idl', 'nsIParentChannel.idl', 'nsIParentRedirectingChannel.idl', diff --git a/netwerk/base/nsIPackagedAppUtils.idl b/netwerk/base/nsIPackagedAppUtils.idl new file mode 100644 index 00000000000..5080b4d9eeb --- /dev/null +++ b/netwerk/base/nsIPackagedAppUtils.idl @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIVerificationCallback; + +%{C++ +#define NS_PACKAGEDAPPUTILS_CONTRACTID "@mozilla.org/network/packaged-app-utils;1" +%} + +/** + * A package using privileged APIs should be signed by marketplace or trust- + * worthy developers. When Necko receives such a package, it has to + * extract the manifest and the signature and calls verifyManifest(...) to verify + * the manifest. nsIPackagedAppUtils will parse the manifest and + * store the hash values of each resource. When a resource is ready, Necko + * will calculate its hash value (including the header like Content-Location: xxx), + * and calls checkIntegrity(...) to verify the integrity. + * + * For more detail: + * https://wiki.mozilla.org/FirefoxOS/New_security_model/Packaging + */ + +[scriptable, uuid(d0a98a69-a215-4cf9-abb3-7a0b9237cd27)] +interface nsIPackagedAppUtils : nsISupports +{ + /** + * @aHeader is the package's header including + * - "manifest-signature: xxxxxx" (base64 encoding) + * @aManifest is the manifest of the package + * - the multipart header is included + * - manifest must be the first resource of the package + * @aCallback is the callback, see comments of nsIVerificationCallback below + */ + void verifyManifest(in ACString aHeader, + in ACString aManifest, + in nsIVerificationCallback aVerifier); + + /** + * @aFileName is the name of a resource in the package + * @aHashValue is the hash value of this resource named aFileName + * - aHashValue should be computed by the caller of this method + * @aCallback is the callback, see comments of nsIVerificationCallback below + */ + void checkIntegrity(in ACString aFileName, + in ACString aHashValue, + in nsIVerificationCallback aVerifier); +}; + +/** + * The callback passed to verifyManifest and checkIntegrity + */ +[scriptable, uuid(e1912028-93e5-4378-aa3f-a58702937169)] +interface nsIVerificationCallback : nsISupports +{ + /** + * @aForManifest + * - true if it's called by verifyManifest + * - false if it's called by checkIntegrity + * @aSuccess + * - true if the verification succeeds, false otherwise + */ + void fireVerifiedEvent(in boolean aForManifest, + in boolean aSuccess); +}; diff --git a/netwerk/protocol/http/PackagedAppUtils.js b/netwerk/protocol/http/PackagedAppUtils.js new file mode 100644 index 00000000000..7854260e105 --- /dev/null +++ b/netwerk/protocol/http/PackagedAppUtils.js @@ -0,0 +1,109 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, +* You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict'; + +const { classes: Cc, interfaces: Ci, utils: Cu } = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +const PACKAGEDAPPUTILS_CONTRACTID = "@mozilla.org/network/packaged-app-utils;1"; +const PACKAGEDAPPUTILS_CID = Components.ID("{fe8f1c2e-3c13-11e5-9a3f-bbf47d1e6697}"); + +function PackagedAppUtils() { + +} + +let DEBUG = 0 +function debug(s) { + if (DEBUG) { + dump("-*- PackagedAppUtils: " + s + "\n"); + } +} + +PackagedAppUtils.prototype = { + classID: PACKAGEDAPPUTILS_CID, + contractID: PACKAGEDAPPUTILS_CONTRACTID, + classDescription: "Packaged App Utils", + QueryInterface: XPCOMUtils.generateQI([Ci.nsIPackagedAppUtils]), + + verifyManifest: function(aHeader, aManifest, aCallback) { + debug("Manifest: " + aManifest); + + // parse signature from header + let signature; + const signatureField = "manifest-signature: "; + for (let item of aHeader.split('\r\n')) { + if (item.substr(0, signatureField.length) == signatureField) { + signature = item.substr(signatureField.length); + break; + } + } + if (!signature) { + debug("No signature in header"); + aCallback.fireVerifiedEvent(true, false); + return; + } + debug("Signature: " + signature); + + try { + // Base64 decode + signature = atob(signature); + + // Remove header + let manifestBody = aManifest.substr(aManifest.indexOf('\r\n\r\n') + 4); + debug("manifestBody: " + manifestBody); + + // Parse manifest, store resource hashes + this.resources = JSON.parse(manifestBody)["moz-resources"]; + } catch (e) { + debug("JSON parsing failure"); + aCallback.fireVerifiedEvent(true, false); + return; + } + + let manifestStream = Cc["@mozilla.org/io/string-input-stream;1"] + .createInstance(Ci.nsIStringInputStream); + let signatureStream = Cc["@mozilla.org/io/string-input-stream;1"] + .createInstance(Ci.nsIStringInputStream); + manifestStream.setData(aManifest, aManifest.length); + signatureStream.setData(signature, signature.length); + + let certDb; + try { + certDb = Cc["@mozilla.org/security/x509certdb;1"] + .getService(Ci.nsIX509CertDB); + } catch (e) { + debug("nsIX509CertDB error: " + e); + // unrecoverable error, don't bug the user + throw "CERTDB_ERROR"; + } + + certDb.verifySignedManifestAsync( + Ci.nsIX509CertDB.PrivilegedPackageRoot, manifestStream, signatureStream, + function(aRv, aCert) { + aCallback.fireVerifiedEvent(true, Components.isSuccessCode(aRv)); + }); + }, + + checkIntegrity: function(aFileName, aHashValue, aCallback) { + debug("checkIntegrity() " + aFileName + ": " + aHashValue + "\n"); + if (!this.resources) { + debug("resource hashes not found"); + aCallback.fireVerifiedEvent(false, false); + return; + } + for (let r of this.resources) { + if (r.src === aFileName) { + debug("found integrity = " + r.integrity); + aCallback.fireVerifiedEvent(false, r.integrity === aHashValue); + return; + } + } + aCallback.fireVerifiedEvent(false, false); + }, +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([PackagedAppUtils]); diff --git a/netwerk/protocol/http/PackagedAppUtils.manifest b/netwerk/protocol/http/PackagedAppUtils.manifest new file mode 100644 index 00000000000..584b1a8cdc8 --- /dev/null +++ b/netwerk/protocol/http/PackagedAppUtils.manifest @@ -0,0 +1,3 @@ +# PackagedAppUtils.js +component {fe8f1c2e-3c13-11e5-9a3f-bbf47d1e6697} PackagedAppUtils.js +contract @mozilla.org/network/packaged-app-utils;1 {fe8f1c2e-3c13-11e5-9a3f-bbf47d1e6697} diff --git a/netwerk/protocol/http/moz.build b/netwerk/protocol/http/moz.build index 5a83a7f1f10..a208f495787 100644 --- a/netwerk/protocol/http/moz.build +++ b/netwerk/protocol/http/moz.build @@ -113,3 +113,8 @@ LOCAL_INCLUDES += [ '/dom/base', '/netwerk/base', ] + +EXTRA_COMPONENTS += [ + 'PackagedAppUtils.js', + 'PackagedAppUtils.manifest', +] diff --git a/netwerk/test/unit/test_packaged_app_utils.js b/netwerk/test/unit/test_packaged_app_utils.js new file mode 100644 index 00000000000..ed6f9bbcb7c --- /dev/null +++ b/netwerk/test/unit/test_packaged_app_utils.js @@ -0,0 +1,135 @@ +const header_missing_signature = "header1: content1"; +const header_invalid_signature = "header1: content1\r\nmanifest-signature: invalid-signature\r\n"; +const header = "manifest-signature: MIIF1AYJKoZIhvcNAQcCoIIFxTCCBcECAQExCzAJBgUrDgMCGgUAMAsGCSqGSIb3DQEHAaCCA54wggOaMIICgqADAgECAgECMA0GCSqGSIb3DQEBCwUAMHMxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEkMCIGA1UEChMbRXhhbXBsZSBUcnVzdGVkIENvcnBvcmF0aW9uMRkwFwYDVQQDExBUcnVzdGVkIFZhbGlkIENBMB4XDTE1MDkxMDA4MDQzNVoXDTM1MDkxMDA4MDQzNVowdDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MSQwIgYDVQQKExtFeGFtcGxlIFRydXN0ZWQgQ29ycG9yYXRpb24xGjAYBgNVBAMTEVRydXN0ZWQgQ29ycCBDZXJ0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAts8whjOzEbn/w1xkFJ67af7F/JPujBK91oyJekh2schIMzFau9pY8S1AiJQoJCulOJCJfUc8hBLKBZiGAkii+4Gpx6cVqMLe6C22MdD806Soxn8Dg4dQqbIvPuI4eeVKu5CEk80PW/BaFMmRvRHO62C7PILuH6yZeGHC4P7dTKpsk4CLxh/jRGXLC8jV2BCW0X+3BMbHBg53NoI9s1Gs7KGYnfOHbBP5wEFAa00RjHnubUaCdEBlC8Kl4X7p0S4RGb3rsB08wgFe9EmSZHIgcIm+SuVo7N4qqbI85qo2ulU6J8NN7ZtgMPHzrMhzgAgf/KnqPqwDIxnNmRNJmHTUYwIDAQABozgwNjAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMDMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAQEAukH6cJUUj5faa8CuPCqrEa0PoLY4SYNnff9NI+TTAHkB9l+kOcFl5eo2EQOcWmZKYi7QLlWC4jy/KQYattO9FMaxiOQL4FAc6ZIbNyfwWBzZWyr5syYJTTTnkLq8A9pCKarN49+FqhJseycU+8EhJEJyP5pv5hLvDNTTHOQ6SXhASsiX8cjo3AY4bxA5pWeXuTZ459qDxOnQd+GrOe4dIeqflk0hA2xYKe3SfF+QlK8EO370B8Dj8RX230OATM1E3OtYyALe34KW3wM9Qm9rb0eViDnVyDiCWkhhQnw5yPg/XQfloug2itRYuCnfUoRt8xfeHgwz2Ymz8cUADn3KpTGCAf4wggH6AgEBMHgwczELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MSQwIgYDVQQKExtFeGFtcGxlIFRydXN0ZWQgQ29ycG9yYXRpb24xGTAXBgNVBAMTEFRydXN0ZWQgVmFsaWQgQ0ECAQIwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTE1MDkxMDA4MDQ0M1owIwYJKoZIhvcNAQkEMRYEFNg6lGtV9bJbL2hA0c5DdOeuCQ6lMA0GCSqGSIb3DQEBAQUABIIBAKGziwzA5Q38rIvNUDHCjYVTR1FhALGZv677Tc2+pwd82W6O9q5GG9IfkF3ajb1fquUIpGPkf7r0oiO4udC8cSehA+lfhR94A8aCM9UhzvTtRI3tFB+TPSk1UcXlX8tB7dNkx4zC06ujlSaRKkmaZODVXQFEcsF6CKMApsBuUJrwzvbQqVi2KHXUO6oGlMEyt4tY+g2OY/vyxGajfAL49dAYOTtrV0arvJvoTYh+E0iSrsbuiuAxKAVjK/QnLJoV/dTaCqW4t3lzHrpE3+avqMXiewxu84VJSURxoryY89uAZS9+4MKrSOGlGCJy/8xDIAm9pi6lPJBP2pIRjaRt9r0=\r\n"; + +const manifest = "Content-Location: manifest.webapp\r\n" + + "Content-Type: application/x-web-app-manifest+json\r\n\r\n" + +`{ + "name": "My App", + "moz-resources": [ + { + "src": "page2.html", + "integrity": "JREF3JbXGvZ+I1KHtoz3f46ZkeIPrvXtG4VyFQrJ7II=" + }, + { + "src": "index.html", + "integrity": "B5Phw8L1tpyRBkI0gwg/evy1fgtMlMq3BIY3Q8X0rYU=" + }, + { + "src": "scripts/script.js", + "integrity": "6TqtNArQKrrsXEQWu3D9ZD8xvDRIkhyV6zVdTcmsT5Q=" + }, + { + "src": "scripts/library.js", + "integrity": "TN2ByXZiaBiBCvS4MeZ02UyNi44vED+KjdjLInUl4o8=" + } + ], + "moz-permissions": [ + { + "systemXHR": { + "description": "Needed to download stuff" + }, + "devicestorage:pictures": { + "description": "Need to load pictures" + } + } + ], + "moz-uuid": "some-uuid", + "moz-package-location": "https://example.com/myapp/app.pak", + "description": "A great app!" +}`; + +const manifest_missing_moz_resources = "Content-Location: manifest.webapp\r\n" + + "Content-Type: application/x-web-app-manifest+json\r\n\r\n" + +`{ + "name": "My App", + "moz-permissions": [ + { + "systemXHR": { + "description": "Needed to download stuff" + }, + "devicestorage:pictures": { + "description": "Need to load pictures" + } + } + ], + "moz-uuid": "some-uuid", + "moz-package-location": "https://example.com/myapp/app.pak", + "description": "A great app!" +}`; + +const manifest_malformed_json = "}"; + +let packagedAppUtils; + +function run_test() { + add_test(test_verify_manifest(header_missing_signature, manifest, false)); + add_test(test_verify_manifest(header_invalid_signature, manifest, false)); + add_test(test_verify_manifest(header, manifest_malformed_json, false)); + add_test(test_verify_manifest(header, manifest_missing_moz_resources, false)); + add_test(test_verify_manifest(header, manifest, true)); + + // The last verification must succeed, because check_integrity use that object; + add_test(test_check_integrity_success); + add_test(test_check_integrity_filename_not_matched); + add_test(test_check_integrity_hashvalue_not_matched); + + run_next_test(); +} + +function test_verify_manifest(aHeader, aManifest, aShouldSucceed) { + return function() { + do_test_pending(); + packagedAppUtils = Cc["@mozilla.org/network/packaged-app-utils;1"]. + createInstance(Ci.nsIPackagedAppUtils); + let fakeVerifier = { + fireVerifiedEvent: function(aForManifest, aSuccess) { + ok(aForManifest, "aForManifest should be true"); + equal(aSuccess, aShouldSucceed, "Expected verification result: " + aShouldSucceed); + do_test_finished(); + run_next_test(); + } + }; + packagedAppUtils.verifyManifest(aHeader, aManifest, fakeVerifier); + } +} + +function test_check_integrity_success() { + let manifestBody = manifest.substr(manifest.indexOf('\r\n\r\n') + 4); + fakeVerifier = { + fireVerifiedEvent: function(aForManifest, aSuccess) { + ok(!aForManifest && aSuccess, "checkIntegrity should succeed"); + do_test_finished(); + run_next_test(); + } + }; + for (let resource of JSON.parse(manifestBody)["moz-resources"]) { + do_test_pending(); + packagedAppUtils.checkIntegrity(resource.src, resource.integrity, fakeVerifier); + } +} + +function test_check_integrity_filename_not_matched() { + fakeVerifier = { + fireVerifiedEvent: function(aForManifest, aSuccess) { + ok(!aForManifest && !aSuccess, "checkIntegrity should fail"); + do_test_finished(); + run_next_test(); + } + }; + do_test_pending(); + packagedAppUtils.checkIntegrity("/nosuchfile.html", "sha256-kass...eoirW-e", fakeVerifier); + run_next_test(); +} + +function test_check_integrity_hashvalue_not_matched() { + fakeVerifier = { + fireVerifiedEvent: function(aForManifest, aSuccess) { + ok(!aForManifest && !aSuccess, "checkIntegrity should fail"); + do_test_finished(); + run_next_test(); + } + }; + do_test_pending(); + packagedAppUtils.checkIntegrity("/index.html", "kass...eoirW-e", fakeVerifier); +} diff --git a/netwerk/test/unit/xpcshell.ini b/netwerk/test/unit/xpcshell.ini index 824f9fab400..0226b443538 100644 --- a/netwerk/test/unit/xpcshell.ini +++ b/netwerk/test/unit/xpcshell.ini @@ -323,6 +323,7 @@ skip-if = os == "android" [test_safeoutputstream_append.js] [test_packaged_app_service.js] [test_packaged_app_verifier.js] +[test_packaged_app_utils.js] [test_suspend_channel_before_connect.js] [test_inhibit_caching.js] [test_dns_disable_ipv4.js] From d1649e6f2d75e7db9fc01d8b2f943b226d18101c Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Thu, 24 Sep 2015 10:07:01 +0800 Subject: [PATCH 60/78] Bug 1178518 - Verify signed packages. r=valentin --- netwerk/protocol/http/PackagedAppService.cpp | 13 +-- netwerk/protocol/http/PackagedAppVerifier.cpp | 95 +++++++++++++++---- netwerk/protocol/http/PackagedAppVerifier.h | 23 ++++- 3 files changed, 104 insertions(+), 27 deletions(-) diff --git a/netwerk/protocol/http/PackagedAppService.cpp b/netwerk/protocol/http/PackagedAppService.cpp index a9205370954..0e7be513d5c 100644 --- a/netwerk/protocol/http/PackagedAppService.cpp +++ b/netwerk/protocol/http/PackagedAppService.cpp @@ -225,15 +225,12 @@ CreateSharedStringStream(const char* aData, uint32_t aCount) static bool GetOriginalResponseHeader(nsIRequest* aRequest, nsACString& aHeader) { - // TODO: The flattened http header might be different from the original. - // See Bug 1198669 for further information. + nsCOMPtr multiPartChannel(do_QueryInterface(aRequest)); + if (!multiPartChannel) { + return false; + } - nsCOMPtr headerProvider(do_QueryInterface(aRequest)); - nsHttpResponseHead *responseHead = headerProvider->GetResponseHead(); - NS_ENSURE_TRUE(responseHead, false); - - responseHead->Flatten(aHeader, true); - aHeader.Append("\r\n"); + multiPartChannel->GetOriginalResponseHeader(aHeader); return true; } diff --git a/netwerk/protocol/http/PackagedAppVerifier.cpp b/netwerk/protocol/http/PackagedAppVerifier.cpp index ab94245272b..b30bb71ea89 100644 --- a/netwerk/protocol/http/PackagedAppVerifier.cpp +++ b/netwerk/protocol/http/PackagedAppVerifier.cpp @@ -14,6 +14,8 @@ #include "nsITimer.h" #include "nsIPackagedAppVerifier.h" #include "mozilla/Preferences.h" +#include "nsIPackagedAppUtils.h" +#include "nsIInputStream.h" static const short kResourceHashType = nsICryptoHash::SHA256; @@ -26,7 +28,7 @@ namespace net { /////////////////////////////////////////////////////////////////////////////// -NS_IMPL_ISUPPORTS(PackagedAppVerifier, nsIPackagedAppVerifier) +NS_IMPL_ISUPPORTS(PackagedAppVerifier, nsIPackagedAppVerifier, nsIVerificationCallback) NS_IMPL_ISUPPORTS(PackagedAppVerifier::ResourceCacheInfo, nsISupports) @@ -76,6 +78,14 @@ NS_IMETHODIMP PackagedAppVerifier::Init(nsIPackagedAppVerifierListener* aListene mSignature = aSignature; mIsPackageSigned = false; mPackageCacheEntry = aPackageCacheEntry; + mIsFirstResource = true; + + nsresult rv; + mPackagedAppUtils = do_CreateInstance(NS_PACKAGEDAPPUTILS_CONTRACTID, &rv); + if (NS_FAILED(rv)) { + LOG(("create packaged app utils failed")); + return rv; + } return NS_OK; } @@ -90,6 +100,11 @@ NS_IMETHODIMP PackagedAppVerifier::OnStartRequest(nsIRequest *aRequest, nsISupports *aContext) { + if (mIsFirstResource) { + // The first resource must be the manifest, hashes not needed + return NS_OK; + } + if (!mHasher) { mHasher = do_CreateInstance("@mozilla.org/security/hash;1"); } @@ -103,6 +118,22 @@ PackagedAppVerifier::OnStartRequest(nsIRequest *aRequest, return mHasher->Init(kResourceHashType); } +NS_METHOD +PackagedAppVerifier::WriteManifest(nsIInputStream* aStream, + void* aManifest, + const char* aFromRawSegment, + uint32_t aToOffset, + uint32_t aCount, + uint32_t* aWriteCount) +{ + LOG(("WriteManifest: length %u", aCount)); + LOG(("%s", aFromRawSegment)); + nsCString* manifest = static_cast(aManifest); + manifest->AppendASCII(aFromRawSegment, aCount); + *aWriteCount = aCount; + return NS_OK; +} + // @param aRequest nullptr. // @param aContext nullptr. // @param aInputStream as-is. @@ -115,6 +146,16 @@ PackagedAppVerifier::OnDataAvailable(nsIRequest *aRequest, uint64_t aOffset, uint32_t aCount) { + if (mIsFirstResource) { + // The first resource must be the manifest, hash value not needed. + // Instead, we read from the input stream and append to mManifest. + uint32_t count; + LOG(("ReadSegments: size = %u", aCount)); + nsresult rv = aInputStream->ReadSegments(WriteManifest, &mManifest, aCount, &count); + MOZ_ASSERT(count == aCount, "Bytes read by ReadSegments don't match"); + return rv; + } + MOZ_ASSERT(!mHashingResourceURI.IsEmpty(), "MUST call BeginResourceHash first."); NS_ENSURE_TRUE(mHasher, NS_ERROR_FAILURE); return mHasher->UpdateFromStream(aInputStream, aCount); @@ -130,17 +171,22 @@ PackagedAppVerifier::OnStopRequest(nsIRequest* aRequest, { MOZ_RELEASE_ASSERT(NS_IsMainThread(), "mHashingResourceURI is not thread safe."); - NS_ENSURE_TRUE(mHasher, NS_ERROR_FAILURE); + if (mIsFirstResource) { + // The first resource must be the manifest, hash value not needed + mIsFirstResource = false; + } else { + NS_ENSURE_TRUE(mHasher, NS_ERROR_FAILURE); - nsAutoCString hash; - nsresult rv = mHasher->Finish(true, hash); - NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString hash; + nsresult rv = mHasher->Finish(true, hash); + NS_ENSURE_SUCCESS(rv, rv); - LOG(("Hash of %s is %s", mHashingResourceURI.get(), hash.get())); + LOG(("Hash of %s is %s", mHashingResourceURI.get(), hash.get())); - // Store the computated hash associated with the resource URI. - mResourceHashStore.Put(mHashingResourceURI, new nsCString(hash)); - mHashingResourceURI = EmptyCString(); + // Store the computated hash associated with the resource URI. + mResourceHashStore.Put(mHashingResourceURI, new nsCString(hash)); + mHashingResourceURI = EmptyCString(); + } // Get a internal copy and take over the life cycle handling // since the linked list we use only supports pointer-based element. @@ -186,9 +232,10 @@ PackagedAppVerifier::ProcessResourceCache(const ResourceCacheInfo* aInfo) } } -void +NS_IMETHODIMP PackagedAppVerifier::FireVerifiedEvent(bool aForManifest, bool aSuccess) { + LOG(("FireVerifiedEvent aForManifest=%d aSuccess=%d", aForManifest, aSuccess)); nsCOMPtr r; if (aForManifest) { @@ -202,6 +249,8 @@ PackagedAppVerifier::FireVerifiedEvent(bool aForManifest, bool aSuccess) } NS_DispatchToMainThread(r); + + return NS_OK; } void @@ -231,9 +280,12 @@ PackagedAppVerifier::VerifyManifest(const ResourceCacheInfo* aInfo) return; } - // TODO: Implement manifest verification. - LOG(("Manifest verification not implemented yet. See Bug 1178518.")); - FireVerifiedEvent(true, false); + LOG(("Signature: length = %u\n%s", mSignature.Length(), mSignature.get())); + LOG(("Manifest: length = %u\n%s", mManifest.Length(), mManifest.get())); + nsresult rv = mPackagedAppUtils->VerifyManifest(mSignature, mManifest, this); + if (NS_FAILED(rv)) { + LOG(("VerifyManifest FAILED rv = %u", (unsigned)rv)); + } } void @@ -269,9 +321,20 @@ PackagedAppVerifier::VerifyResource(const ResourceCacheInfo* aInfo) return; } - // TODO: Implement resource integrity check. - LOG(("Resource integrity check not implemented yet. See Bug 1178518.")); - FireVerifiedEvent(false, false); + nsAutoCString path; + nsCOMPtr url(do_QueryInterface(aInfo->mURI)); + if (url) { + url->GetFilePath(path); + } + int32_t pos = path.Find("!//"); + if (pos == kNotFound) { + FireVerifiedEvent(false, false); + return; + } + // Only keep the part after "!//" + path.Cut(0, pos + 3); + + mPackagedAppUtils->CheckIntegrity(path, *resourceHash, this); } void diff --git a/netwerk/protocol/http/PackagedAppVerifier.h b/netwerk/protocol/http/PackagedAppVerifier.h index 7049dbcac35..4166097609e 100644 --- a/netwerk/protocol/http/PackagedAppVerifier.h +++ b/netwerk/protocol/http/PackagedAppVerifier.h @@ -14,18 +14,21 @@ #include "nsICryptoHash.h" #include "nsIPackagedAppVerifier.h" #include "mozilla/LinkedList.h" +#include "nsIPackagedAppUtils.h" namespace mozilla { namespace net { class PackagedAppVerifier final : public nsIPackagedAppVerifier + , public nsIVerificationCallback { public: NS_DECL_ISUPPORTS NS_DECL_NSIREQUESTOBSERVER NS_DECL_NSISTREAMLISTENER NS_DECL_NSIPACKAGEDAPPVERIFIER + NS_DECL_NSIVERIFICATIONCALLBACK public: enum EState { @@ -120,6 +123,14 @@ private: // void ProcessResourceCache(const ResourceCacheInfo* aInfo); + // Callback for nsIInputStream::ReadSegment() to read manifest + static NS_METHOD WriteManifest(nsIInputStream* aStream, + void* aManifest, + const char* aFromRawSegment, + uint32_t aToOffset, + uint32_t aCount, + uint32_t* aWriteCount); + // This two functions would call the actual verifier. void VerifyManifest(const ResourceCacheInfo* aInfo); void VerifyResource(const ResourceCacheInfo* aInfo); @@ -127,9 +138,6 @@ private: void OnManifestVerified(bool aSuccess); void OnResourceVerified(bool aSuccess); - // Fire a async event to notify the verification result. - void FireVerifiedEvent(bool aForManifest, bool aSuccess); - // To notify that either manifest or resource check is done. nsCOMPtr mListener; @@ -142,6 +150,12 @@ private: // The signature of the package. nsCString mSignature; + // The app manfiest of the package + nsCString mManifest; + + // Whether we're processing the first resource, which is the manfiest + bool mIsFirstResource; + // Whether this package app is signed. bool mIsPackageSigned; @@ -159,6 +173,9 @@ private: // |EndResourceHash| call. nsCString mLastComputedResourceHash; + // This will help to verify manifests and resource integrity + nsCOMPtr mPackagedAppUtils; + // A list of pending resource that is downloaded but not verified yet. mozilla::LinkedList mPendingResourceCacheInfoList; From 93ab59d33bdc1012e04283c5f482064c9bbe6c4a Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Mon, 7 Sep 2015 15:28:21 +0800 Subject: [PATCH 61/78] Bug 1178518 - Add an AppTrustedRoot for signed packaged app. r=keeler --- dom/apps/StoreTrustAnchor.jsm | 1 + security/apps/AppTrustDomain.cpp | 7 +++++++ security/apps/gen_cert_header.py | 1 + security/apps/moz.build | 1 + security/apps/privileged-package-root.der | Bin 0 -> 930 bytes security/manager/ssl/nsIX509CertDB.idl | 3 ++- 6 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 security/apps/privileged-package-root.der diff --git a/dom/apps/StoreTrustAnchor.jsm b/dom/apps/StoreTrustAnchor.jsm index 9321bcfdf23..199d7304e34 100644 --- a/dom/apps/StoreTrustAnchor.jsm +++ b/dom/apps/StoreTrustAnchor.jsm @@ -16,6 +16,7 @@ const APP_TRUSTED_ROOTS= ["AppMarketplaceProdPublicRoot", "AppMarketplaceDevPublicRoot", "AppMarketplaceDevReviewersRoot", "AppMarketplaceStageRoot", + "PrivilegedPackageRoot", "AppXPCShellRoot"]; this.TrustedRootCertificate = { diff --git a/security/apps/AppTrustDomain.cpp b/security/apps/AppTrustDomain.cpp index b2c3fe38027..a74b6da8eb0 100644 --- a/security/apps/AppTrustDomain.cpp +++ b/security/apps/AppTrustDomain.cpp @@ -26,6 +26,8 @@ // Add-on signing Certificates #include "addons-public.inc" #include "addons-stage.inc" +// Privileged Package Certificates +#include "privileged-package-root.inc" using namespace mozilla::pkix; @@ -94,6 +96,11 @@ AppTrustDomain::SetTrustedRoot(AppTrustedRoot trustedRoot) trustedDER.len = mozilla::ArrayLength(addonsStageRoot); break; + case nsIX509CertDB::PrivilegedPackageRoot: + trustedDER.data = const_cast(privilegedPackageRoot); + trustedDER.len = mozilla::ArrayLength(privilegedPackageRoot); + break; + default: PR_SetError(SEC_ERROR_INVALID_ARGS, 0); return SECFailure; diff --git a/security/apps/gen_cert_header.py b/security/apps/gen_cert_header.py index c82f974b3c6..0ffe25cf4e1 100644 --- a/security/apps/gen_cert_header.py +++ b/security/apps/gen_cert_header.py @@ -37,6 +37,7 @@ array_names = [ 'xpcshellRoot', 'addonsPublicRoot', 'addonsStageRoot', + 'privilegedPackageRoot', ] for n in array_names: diff --git a/security/apps/moz.build b/security/apps/moz.build index 921f6bc7c12..795cf641829 100644 --- a/security/apps/moz.build +++ b/security/apps/moz.build @@ -34,6 +34,7 @@ headers_arrays_certs = [ ('xpcshell.inc', 'xpcshellRoot', test_ssl_path + '/test_signed_apps/trusted_ca1.der'), ('addons-public.inc', 'addonsPublicRoot', 'addons-public.crt'), ('addons-stage.inc', 'addonsStageRoot', 'addons-stage.crt'), + ('privileged-package-root.inc', 'privilegedPackageRoot', 'privileged-package-root.der'), ] for header, array_name, cert in headers_arrays_certs: diff --git a/security/apps/privileged-package-root.der b/security/apps/privileged-package-root.der new file mode 100644 index 0000000000000000000000000000000000000000..9f15847bfccb5bfc22cbefd770b543b23ea01df3 GIT binary patch literal 930 zcmXqLVxDKv#MHKcnTe5!iILHOmyJ`a&7Gh=Wuy3k$$x!xD2cfodEL6vTNAO${s!4Gb&{OpHyUB>0UCOpyf! z##v2_O31;($jZRn#K_NJ(8S2a)WpchaPDe<-oBkv_O1+QuV0qLH1m$gxswgsIx4z* z*1amoy)*kir|yTH)hmmngP0Dx_OC1W`rGu=|BXjq=4^jzSh%l2>|XCFy_2;S4O&0^ zcd}lZu%VkJyz312%O4BZ-Prm4Wt{5iKZS*@OW66ZckEtb-hASmQ^r@;>oykm7f0JU zloq^}yK&)&sMCc)p9h9Mx7;j57C5H%FU&7Y??}CVYj(+&jht?NM;TIQvCeuvWxmqw z%eh9MCmNNcmTLR)xiyAPi!GbJ_m-l*{q#;wjVg;{lB%j2!V6YSF%9XvQ!YHCdg|4H z*C*s>=)Zayt@bW6W>2p3X|)wA)y2~m->v4Uj`ZIfQvZ>OnUR5Uv9*Dv0Y5NiWrZ0T z|Ff_fFas$AF_3^NNPx$Hi;Y8@jggg=otfEy4{_J;SK>fL8OKTHZ~D-Upx z?Q-GQN>1>v<@!+m?4p8CT#KQ>?5dy66(`qn2)xekJdv+9p{B7>!__6;-Sugn*`wfn zbJw*!aacb8!fdVo+>S~2>xxem8H1NYV)6|j;Tq@jA|!3d@hx%x1E&7 zFlkp#ve#qd`gd)?F$aa&H+vZW;(mL})T5fOxT;WVt|9-!^n3Y?73UJ3id9uT=Qx+Q zQCf21Ua!ZyT{o;S^*e8)aHBESbLyGhTm4gKzg`*tV8wRllcIM!4Jtn+E$?k&*ONJK IzBqFh0EZk^2><{9 literal 0 HcmV?d00001 diff --git a/security/manager/ssl/nsIX509CertDB.idl b/security/manager/ssl/nsIX509CertDB.idl index 7b05341e736..d1ba93edeb8 100644 --- a/security/manager/ssl/nsIX509CertDB.idl +++ b/security/manager/ssl/nsIX509CertDB.idl @@ -46,7 +46,7 @@ interface nsIVerifySignedManifestCallback : nsISupports * This represents a service to access and manipulate * X.509 certificates stored in a database. */ -[scriptable, uuid(3fe3702b-766b-47dd-8f77-c08c3a339a74)] +[scriptable, uuid(0a47571d-602c-4b21-9f52-c3d0e681d83a)] interface nsIX509CertDB : nsISupports { /** @@ -318,6 +318,7 @@ interface nsIX509CertDB : nsISupports { const AppTrustedRoot AppXPCShellRoot = 6; const AppTrustedRoot AddonsPublicRoot = 7; const AppTrustedRoot AddonsStageRoot = 8; + const AppTrustedRoot PrivilegedPackageRoot = 9; void openSignedAppFileAsync(in AppTrustedRoot trustedRoot, in nsIFile aJarFile, in nsIOpenSignedAppFileCallback callback); From fbf2b297d1e08b5b113843a661c1b0941aa64fbb Mon Sep 17 00:00:00 2001 From: Morris Tseng Date: Wed, 23 Sep 2015 18:53:00 +0200 Subject: [PATCH 62/78] Bug 1207887 - Fix AsyncPanZoomAnimation, WheelScrollAnimation and TextureHost doesn't compile in non-unified build. r=nical --- gfx/layers/apz/src/AsyncPanZoomAnimation.h | 1 + gfx/layers/apz/src/WheelScrollAnimation.cpp | 4 ++++ gfx/layers/apz/src/WheelScrollAnimation.h | 2 ++ gfx/layers/composite/TextureHost.cpp | 2 ++ 4 files changed, 9 insertions(+) diff --git a/gfx/layers/apz/src/AsyncPanZoomAnimation.h b/gfx/layers/apz/src/AsyncPanZoomAnimation.h index fd0574001a8..27caa20da91 100644 --- a/gfx/layers/apz/src/AsyncPanZoomAnimation.h +++ b/gfx/layers/apz/src/AsyncPanZoomAnimation.h @@ -10,6 +10,7 @@ #include "base/message_loop.h" #include "mozilla/RefPtr.h" #include "mozilla/TimeStamp.h" +#include "mozilla/Vector.h" #include "FrameMetrics.h" #include "nsISupportsImpl.h" diff --git a/gfx/layers/apz/src/WheelScrollAnimation.cpp b/gfx/layers/apz/src/WheelScrollAnimation.cpp index 486b814fbbc..f70682626de 100644 --- a/gfx/layers/apz/src/WheelScrollAnimation.cpp +++ b/gfx/layers/apz/src/WheelScrollAnimation.cpp @@ -6,6 +6,10 @@ #include "WheelScrollAnimation.h" +#include "AsyncPanZoomController.h" +#include "gfxPrefs.h" +#include "nsPoint.h" + namespace mozilla { namespace layers { diff --git a/gfx/layers/apz/src/WheelScrollAnimation.h b/gfx/layers/apz/src/WheelScrollAnimation.h index 64f0e330577..f743f5bf6c0 100644 --- a/gfx/layers/apz/src/WheelScrollAnimation.h +++ b/gfx/layers/apz/src/WheelScrollAnimation.h @@ -13,6 +13,8 @@ namespace mozilla { namespace layers { +class AsyncPanZoomController; + class WheelScrollAnimation : public AsyncPanZoomAnimation, public AsyncScrollBase diff --git a/gfx/layers/composite/TextureHost.cpp b/gfx/layers/composite/TextureHost.cpp index ae9f513b7c6..9f2458c8e14 100644 --- a/gfx/layers/composite/TextureHost.cpp +++ b/gfx/layers/composite/TextureHost.cpp @@ -6,6 +6,7 @@ #include "TextureHost.h" #include "CompositableHost.h" // for CompositableHost +#include "LayerScope.h" #include "LayersLogging.h" // for AppendToString #include "mozilla/gfx/2D.h" // for DataSourceSurface, Factory #include "mozilla/ipc/Shmem.h" // for Shmem @@ -23,6 +24,7 @@ #include "mozilla/unused.h" #include #include "../opengl/CompositorOGL.h" +#include "gfxPrefs.h" #include "gfxUtils.h" #ifdef MOZ_ENABLE_D3D10_LAYER From cf8f9e8b39be1fcba6ce4a20e1a8cf5938cfdafa Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 23 Sep 2015 15:40:14 -0400 Subject: [PATCH 63/78] Bug 1207677: remove unused file; r=wcosta --- testing/taskcluster/README.md | 3 +- .../tasks/branches/alder/job_flags.yml | 2 +- .../tasks/branches/b2g-inbound/job_flags.yml | 2 +- .../taskcluster/tasks/branches/base_jobs.yml | 2 +- .../tasks/branches/cedar/job_flags.yml | 2 +- .../tasks/branches/fx-team/job_flags.yml | 2 +- .../branches/mozilla-central/job_flags.yml | 2 +- .../branches/mozilla-inbound/job_flags.yml | 2 +- .../tasks/branches/try/job_flags.yml | 2 +- testing/taskcluster/tasks/job_flags.yml | 259 ------------------ 10 files changed, 10 insertions(+), 268 deletions(-) delete mode 100644 testing/taskcluster/tasks/job_flags.yml diff --git a/testing/taskcluster/README.md b/testing/taskcluster/README.md index ee47c6c3af8..32b4a0e73e1 100644 --- a/testing/taskcluster/README.md +++ b/testing/taskcluster/README.md @@ -57,7 +57,8 @@ templates variables: By convention build tasks are stored in `tasks/builds/` the location of each particular type of build is specified in `job_flags.yml` (and more -locations in the future) +locations in the future), which is located in the appropriate subdirectory +of `branches/`. #### Task format diff --git a/testing/taskcluster/tasks/branches/alder/job_flags.yml b/testing/taskcluster/tasks/branches/alder/job_flags.yml index 116c8d0f06d..89b77d6d89f 100644 --- a/testing/taskcluster/tasks/branches/alder/job_flags.yml +++ b/testing/taskcluster/tasks/branches/alder/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_job_flags.yml diff --git a/testing/taskcluster/tasks/branches/b2g-inbound/job_flags.yml b/testing/taskcluster/tasks/branches/b2g-inbound/job_flags.yml index daab36aad14..becf956e431 100644 --- a/testing/taskcluster/tasks/branches/b2g-inbound/job_flags.yml +++ b/testing/taskcluster/tasks/branches/b2g-inbound/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_jobs.yml diff --git a/testing/taskcluster/tasks/branches/base_jobs.yml b/testing/taskcluster/tasks/branches/base_jobs.yml index c2b7e8fb3e9..403f57bced5 100644 --- a/testing/taskcluster/tasks/branches/base_jobs.yml +++ b/testing/taskcluster/tasks/branches/base_jobs.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_job_flags.yml diff --git a/testing/taskcluster/tasks/branches/cedar/job_flags.yml b/testing/taskcluster/tasks/branches/cedar/job_flags.yml index b0c98d0d14e..101a1593b84 100644 --- a/testing/taskcluster/tasks/branches/cedar/job_flags.yml +++ b/testing/taskcluster/tasks/branches/cedar/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_job_flags.yml diff --git a/testing/taskcluster/tasks/branches/fx-team/job_flags.yml b/testing/taskcluster/tasks/branches/fx-team/job_flags.yml index daab36aad14..becf956e431 100644 --- a/testing/taskcluster/tasks/branches/fx-team/job_flags.yml +++ b/testing/taskcluster/tasks/branches/fx-team/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_jobs.yml diff --git a/testing/taskcluster/tasks/branches/mozilla-central/job_flags.yml b/testing/taskcluster/tasks/branches/mozilla-central/job_flags.yml index a95de44d247..396e4491073 100644 --- a/testing/taskcluster/tasks/branches/mozilla-central/job_flags.yml +++ b/testing/taskcluster/tasks/branches/mozilla-central/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_jobs.yml diff --git a/testing/taskcluster/tasks/branches/mozilla-inbound/job_flags.yml b/testing/taskcluster/tasks/branches/mozilla-inbound/job_flags.yml index daab36aad14..becf956e431 100644 --- a/testing/taskcluster/tasks/branches/mozilla-inbound/job_flags.yml +++ b/testing/taskcluster/tasks/branches/mozilla-inbound/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_jobs.yml diff --git a/testing/taskcluster/tasks/branches/try/job_flags.yml b/testing/taskcluster/tasks/branches/try/job_flags.yml index 2ebd3693047..7be4c18c006 100644 --- a/testing/taskcluster/tasks/branches/try/job_flags.yml +++ b/testing/taskcluster/tasks/branches/try/job_flags.yml @@ -1,6 +1,6 @@ --- # For complete sample of all build and test jobs, -# see /testing/taskcluster/tasks/job_flags.yml +# see /testing/taskcluster/tasks/branches/base_job_flags.yml $inherits: from: tasks/branches/base_job_flags.yml diff --git a/testing/taskcluster/tasks/job_flags.yml b/testing/taskcluster/tasks/job_flags.yml deleted file mode 100644 index 65d7f893cce..00000000000 --- a/testing/taskcluster/tasks/job_flags.yml +++ /dev/null @@ -1,259 +0,0 @@ -# This file contains the list of "job flags"/"try flags" for tests and builds. ---- - -# List of all possible flags for each category of tests used in the case where -# "all" is specified. -flags: - builds: - - emulator - - emulator-jb - - emulator-kk - - emulator-x86-kk - - linux32_gecko # b2g desktop linux 32 bit - - linux64_gecko # b2g desktop linux 64 bit - - linux64-mulet # Firefox desktop - b2g gecko linux 64 bit - - macosx64_gecko # b2g desktop osx 64 bit - - win32_gecko # b2g desktop win 32 bit - - sm-plain # spidermonkey plain - - sm-arm-sim # spidermonkey arm-sim - - sm-arm-sim-osx # spidermonkey arm-sim-osx - - sm-compacting # spidermonkey compacting - - sm-generational # spidermonkey generational - - sm-rootanalysis # spidermonkey rootanalysis - - sm-warnaserr # spidermonkey warnings-as-errors - - tests: - - cppunit - - crashtest - - crashtest-ipc - - gaia-build - - gaia-build-unit - - gaia-js-integration - - gaia-linter - - gaia-unit - - gaia-unit-oop - - gaia-ui-test-oop - - gaia-ui-test-accessibility - - gaia-ui-test-functional - - gaia-ui-test-unit - - jetpack - - jittests - - jsreftest - - marionette - - marionette-webapi - - mochitest - - mochitest-media - - mochitest-oop - - mozmill - - reftest - - reftest-ipc - - reftest-no-accel - - reftest-sanity-oop - - web-platform-tests - - xpcshell - -# Build section covers the -b[uild] and -p[latform] options that try provides. -builds: - # The format for registering a new build flag -> task looks like this: - # - # - # # Platforms are primarily used to restrict test runs to only X platform - # # but the information is stored on the build to indicate which platform(s) - # # the build belongs to. Note that `platforms` here is the term used by the - # # try chooser meaning "some group of tests" examples of platforms are - # # things like "b2g", "win32" - # platforms: - # - - # # note that for sanity o -> means opt and d -> means debug if additional - # # flags are passed we will attempt to match them up to an option here if - # # available - # types: - # opt: - # debug: - # - linux64_gecko: - platforms: - - b2g - types: - opt: - task: tasks/builds/b2g_desktop_opt.yml - debug: - task: tasks/builds/b2g_desktop_debug.yml - linux64-mulet: - platforms: - - Mulet Linux - types: - opt: - task: tasks/builds/mulet_linux.yml - emulator-kk: - platforms: - - b2g - types: - opt: - task: tasks/builds/b2g_emulator_kk_opt.yml - debug: - task: tasks/builds/b2g_emulator_kk_debug.yml - emulator-jb: - platforms: - - b2g - types: - opt: - task: tasks/builds/b2g_emulator_jb_opt.yml - debug: - task: tasks/builds/b2g_emulator_jb_debug.yml - emulator: - platforms: - - b2g - types: - opt: - task: tasks/builds/b2g_emulator_ics_opt.yml - debug: - task: tasks/builds/b2g_emulator_ics_debug.yml - -# Test section covers the -u options in the try flags -tests: - # The format for registering a new test flag -> task looks like this: - # - # - # task: - # # Note that total number of chunks effects more then just performance we - # # need to schedule specific chunks in some cases! - # chunks: - # # Not all tests can run on all builds and we may not want to run some - # # tests on all build variants so we use "allowed tasks" instead of - # # "allowed platforms" here. - # allowed_build_tasks: - # - builds/b2g_desktop_opt.yml - cppunit: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_cpp_unit.yml - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_cpp_unit.yml - crashtest: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_crashtest.yml - chunks: 3 - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_crashtest.yml - chunks: 3 - gaia-build: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_build_test.yml - gaia-build-unit: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_build_unit.yml - gaia-js-integration: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_js_integration_tests.yml - chunks: 10 - tasks/builds/b2g_desktop_debug.yml: - task: tasks/tests/b2g_gaia_js_integration_tests.yml - chunks: 10 - gaia-linter: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_linter.yml - gaia-ui-test-accessibility: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_ui_test_accessibility.yml - tasks/builds/b2g_emulator.yml: - task: tasks/tests/b2g_emulator_gaia_ui_test_accessibility.yml - gaia-ui-test-functional: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_ui_test_functional.yml - chunks: 3 - gaia-ui-test-oop: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gip_oop.yml - gaia-ui-test-unit: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_ui_test_unit.yml - gaia-unit: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_unit.yml - gaia-unit-oop: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_gaia_unit_oop.yml - jsreftest: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_js_reftest.yml - chunks: 3 - marionette: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_marionette.yml - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_marionette.yml - marionette-webapi: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_marionette_webapi.yml - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_marionette_webapi.yml - mochitest: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_mochitest.yml - chunks: 5 - tasks/builds/b2g_desktop_debug.yml: - task: tasks/tests/b2g_mochitest.yml - chunks: 5 - tasks/builds/mulet_linux.yml: - task: tasks/tests/mulet_mochitests.yml - chunks: 5 - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_mochitest.yml - chunks: 9 - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_mochitest.yml - chunks: 20 - tasks/builds/b2g_emulator_jb_opt.yml: - task: tasks/tests/b2g_emulator_mochitest.yml - chunks: 1 - mochitest-media: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_mochitest_media.yml - mochitest-oop: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_mochitest_oop.yml - chunks: 1 - reftest: - allowed_build_tasks: - tasks/builds/mulet_linux.yml: - task: tasks/tests/mulet_reftests.yml - chunks: 6 - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_reftests.yml - chunks: 10 - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_reftest.yml - chunks: 20 - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_reftest.yml - chunks: 10 - reftest-sanity-oop: - allowed_build_tasks: - tasks/builds/b2g_desktop_opt.yml: - task: tasks/tests/b2g_reftests_sanity_oop.yml - chunks: 1 - xpcshell: - allowed_build_tasks: - tasks/builds/b2g_emulator_ics_opt.yml: - task: tasks/tests/b2g_emulator_xpcshell.yml - tasks/builds/b2g_emulator_ics_debug.yml: - task: tasks/tests/b2g_emulator_xpcshell_chunked.yml - chunks: 2 From ba435f481ad400f65c93339e45b28ced1072667a Mon Sep 17 00:00:00 2001 From: Ehsan Akhgari Date: Wed, 23 Sep 2015 21:18:21 -0400 Subject: [PATCH 64/78] Bug 1207135 - Add linux64-st-an to the set of TaskCluster builds; r=dustin --- testing/taskcluster/tasks/branches/base_job_flags.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/taskcluster/tasks/branches/base_job_flags.yml b/testing/taskcluster/tasks/branches/base_job_flags.yml index 6591e591bbe..4846340bb73 100644 --- a/testing/taskcluster/tasks/branches/base_job_flags.yml +++ b/testing/taskcluster/tasks/branches/base_job_flags.yml @@ -34,6 +34,7 @@ flags: - aries-dogfood - android-api-11 - linux64 + - linux64-st-an - macosx64 tests: From d3188aaccf5ef5b3b9aaab76a9c2a9d90e20e48a Mon Sep 17 00:00:00 2001 From: Sotaro Ikeda Date: Thu, 24 Sep 2015 08:31:30 -0700 Subject: [PATCH 65/78] Bug 1187145 - Replace nsBaseHashtable::Enumerate() calls in gfx/ with iterators r=njn --- gfx/layers/ipc/CompositorChild.cpp | 21 +-- gfx/layers/ipc/CompositorChild.h | 4 - gfx/thebes/gfxFT2FontList.cpp | 202 ++++++++++------------- gfx/thebes/gfxFont.cpp | 6 +- gfx/thebes/gfxFont.h | 4 +- gfx/thebes/gfxFontEntry.cpp | 12 +- gfx/thebes/gfxGraphiteShaper.cpp | 3 +- gfx/thebes/gfxHarfBuzzShaper.cpp | 3 +- gfx/thebes/gfxPlatformFontList.cpp | 250 +++++++++-------------------- gfx/thebes/gfxPlatformFontList.h | 14 -- 10 files changed, 188 insertions(+), 331 deletions(-) diff --git a/gfx/layers/ipc/CompositorChild.cpp b/gfx/layers/ipc/CompositorChild.cpp index 24704334a2f..42b2a89f892 100644 --- a/gfx/layers/ipc/CompositorChild.cpp +++ b/gfx/layers/ipc/CompositorChild.cpp @@ -178,22 +178,17 @@ CompositorChild::AllocPLayerTransactionChild(const nsTArray& aBac return c; } -/*static*/ PLDHashOperator -CompositorChild::RemoveSharedMetricsForLayersId(const uint64_t& aKey, - nsAutoPtr& aData, - void* aLayerTransactionChild) -{ - uint64_t childId = static_cast(aLayerTransactionChild)->GetId(); - if (aData->GetLayersId() == childId) { - return PLDHashOperator::PL_DHASH_REMOVE; - } - return PLDHashOperator::PL_DHASH_NEXT; -} - bool CompositorChild::DeallocPLayerTransactionChild(PLayerTransactionChild* actor) { - mFrameMetricsTable.Enumerate(RemoveSharedMetricsForLayersId, actor); + uint64_t childId = static_cast(actor)->GetId(); + + for (auto iter = mFrameMetricsTable.Iter(); !iter.Done(); iter.Next()) { + nsAutoPtr& data = iter.Data(); + if (data->GetLayersId() == childId) { + iter.Remove(); + } + } static_cast(actor)->ReleaseIPDLReference(); return true; } diff --git a/gfx/layers/ipc/CompositorChild.h b/gfx/layers/ipc/CompositorChild.h index 09a7e0399d9..cb45de3787a 100644 --- a/gfx/layers/ipc/CompositorChild.h +++ b/gfx/layers/ipc/CompositorChild.h @@ -174,10 +174,6 @@ private: uint32_t mAPZCId; }; - static PLDHashOperator RemoveSharedMetricsForLayersId(const uint64_t& aKey, - nsAutoPtr& aData, - void* aLayerTransactionChild); - nsRefPtr mLayerManager; // When not multi-process, hold a reference to the CompositorParent to keep it // alive. This reference should be null in multi-process. diff --git a/gfx/thebes/gfxFT2FontList.cpp b/gfx/thebes/gfxFT2FontList.cpp index 9fc361df545..ebb2161c0ea 100644 --- a/gfx/thebes/gfxFT2FontList.cpp +++ b/gfx/thebes/gfxFT2FontList.cpp @@ -1115,22 +1115,19 @@ gfxFT2FontList::AppendFacesFromOmnijarEntry(nsZipArchive* aArchive, // Called on each family after all fonts are added to the list; // this will sort faces to give priority to "standard" font files // if aUserArg is non-null (i.e. we're using it as a boolean flag) -static PLDHashOperator +static void FinalizeFamilyMemberList(nsStringHashKey::KeyType aKey, nsRefPtr& aFamily, - void* aUserArg) + bool aSortFaces) { gfxFontFamily *family = aFamily.get(); - bool sortFaces = (aUserArg != nullptr); family->SetHasStyles(true); - if (sortFaces) { + if (aSortFaces) { family->SortAvailableFonts(); } family->CheckForSimpleFamily(); - - return PL_DHASH_NEXT; } void @@ -1157,8 +1154,17 @@ gfxFT2FontList::FindFonts() // Passing null for userdata tells Finalize that it does not need // to sort faces (because they were already sorted by chrome, // so we just maintain the existing order) - mFontFamilies.Enumerate(FinalizeFamilyMemberList, nullptr); - mHiddenFontFamilies.Enumerate(FinalizeFamilyMemberList, nullptr); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + FinalizeFamilyMemberList(key, family, /* aSortFaces */ false); + } + for (auto iter = mHiddenFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + FinalizeFamilyMemberList(key, family, /* aSortFaces */ false ); + } + LOG(("got font list from chrome process: %d faces in %d families " "and %d in hidden families", fonts.Length(), mFontFamilies.Count(), @@ -1237,8 +1243,16 @@ gfxFT2FontList::FindFonts() // Finalize the families by sorting faces into standard order // and marking "simple" families. // Passing non-null userData here says that we want faces to be sorted. - mFontFamilies.Enumerate(FinalizeFamilyMemberList, this); - mHiddenFontFamilies.Enumerate(FinalizeFamilyMemberList, this); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + FinalizeFamilyMemberList(key, family, /* aSortFaces */ true); + } + for (auto iter = mHiddenFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + FinalizeFamilyMemberList(key, family, /* aSortFaces */ true); + } } void @@ -1328,39 +1342,17 @@ gfxFT2FontList::AppendFaceFromFontListEntry(const FontListEntry& aFLE, } } -static PLDHashOperator -AddFamilyToFontList(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamily, - void* aUserArg) -{ - InfallibleTArray* fontlist = - reinterpret_cast*>(aUserArg); - - FT2FontFamily *family = static_cast(aFamily.get()); - family->AddFacesToFontList(fontlist, FT2FontFamily::kVisible); - - return PL_DHASH_NEXT; -} - -static PLDHashOperator -AddHiddenFamilyToFontList(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamily, - void* aUserArg) -{ - InfallibleTArray* fontlist = - reinterpret_cast*>(aUserArg); - - FT2FontFamily *family = static_cast(aFamily.get()); - family->AddFacesToFontList(fontlist, FT2FontFamily::kHidden); - - return PL_DHASH_NEXT; -} - void gfxFT2FontList::GetSystemFontList(InfallibleTArray* retValue) { - mFontFamilies.Enumerate(AddFamilyToFontList, retValue); - mHiddenFontFamilies.Enumerate(AddHiddenFamilyToFontList, retValue); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + auto family = static_cast(iter.Data().get()); + family->AddFacesToFontList(retValue, FT2FontFamily::kVisible); + } + for (auto iter = mHiddenFontFamilies.Iter(); !iter.Done(); iter.Next()) { + auto family = static_cast(iter.Data().get()); + family->AddFacesToFontList(retValue, FT2FontFamily::kHidden); + } } static void @@ -1377,10 +1369,9 @@ LoadSkipSpaceLookupCheck(nsTHashtable& aSkipSpaceLookupCheck) } } -static PLDHashOperator +void PreloadAsUserFontFaces(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamily, - void* aUserArg) + nsRefPtr& aFamily) { gfxFontFamily *family = aFamily.get(); @@ -1433,8 +1424,6 @@ PreloadAsUserFontFaces(nsStringHashKey::KeyType aKey, gfxUserFontSet::UserFontCache::CacheFont( fe, gfxUserFontSet::UserFontCache::kPersistent); } - - return PL_DHASH_NEXT; } nsresult @@ -1443,74 +1432,65 @@ gfxFT2FontList::InitFontList() // reset font lists gfxPlatformFontList::InitFontList(); mHiddenFontFamilies.Clear(); - + LoadSkipSpaceLookupCheck(mSkipSpaceLookupCheckFamilies); FindFonts(); - mHiddenFontFamilies.Enumerate(PreloadAsUserFontFaces, this); - + for (auto iter = mHiddenFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + PreloadAsUserFontFaces(key, family); + } return NS_OK; } -struct FullFontNameSearch { - FullFontNameSearch(const nsAString& aFullName) - : mFullName(aFullName), mFontEntry(nullptr) - { } - - nsString mFullName; - FT2FontEntry *mFontEntry; -}; - -// callback called for each family name, based on the assumption that the +// called for each family name, based on the assumption that the // first part of the full name is the family name -static PLDHashOperator -FindFullName(nsStringHashKey::KeyType aKey, - nsRefPtr& aFontFamily, - void* userArg) -{ - FullFontNameSearch *data = reinterpret_cast(userArg); - // does the family name match up to the length of the family name? - const nsString& family = aFontFamily->Name(); - - nsString fullNameFamily; - data->mFullName.Left(fullNameFamily, family.Length()); - - // if so, iterate over faces in this family to see if there is a match - if (family.Equals(fullNameFamily, nsCaseInsensitiveStringComparator())) { - nsTArray >& fontList = aFontFamily->GetFontList(); - int index, len = fontList.Length(); - for (index = 0; index < len; index++) { - gfxFontEntry* fe = fontList[index]; - if (!fe) { - continue; - } - if (fe->Name().Equals(data->mFullName, - nsCaseInsensitiveStringComparator())) { - data->mFontEntry = static_cast(fe); - return PL_DHASH_STOP; - } - } - } - - return PL_DHASH_NEXT; -} - -gfxFontEntry* +gfxFontEntry* gfxFT2FontList::LookupLocalFont(const nsAString& aFontName, uint16_t aWeight, int16_t aStretch, bool aItalic) { // walk over list of names - FullFontNameSearch data(aFontName); + FT2FontEntry* fontEntry = nullptr; + nsString fullName(aFontName); // Note that we only check mFontFamilies here, not mHiddenFontFamilies; // hence @font-face { src:local(...) } will not find hidden fonts. - mFontFamilies.Enumerate(FindFullName, &data); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + // Check family name, based on the assumption that the + // first part of the full name is the family name + nsRefPtr& fontFamily = iter.Data(); - if (!data.mFontEntry) { + // does the family name match up to the length of the family name? + const nsString& family = fontFamily->Name(); + nsString fullNameFamily; + + fullName.Left(fullNameFamily, family.Length()); + + // if so, iterate over faces in this family to see if there is a match + if (family.Equals(fullNameFamily, nsCaseInsensitiveStringComparator())) { + nsTArray >& fontList = fontFamily->GetFontList(); + int index, len = fontList.Length(); + for (index = 0; index < len; index++) { + gfxFontEntry* fe = fontList[index]; + if (!fe) { + continue; + } + if (fe->Name().Equals(fullName, + nsCaseInsensitiveStringComparator())) { + fontEntry = static_cast(fe); + goto searchDone; + } + } + } + } + +searchDone: + if (!fontEntry) { return nullptr; } @@ -1518,16 +1498,16 @@ gfxFT2FontList::LookupLocalFont(const nsAString& aFontName, // from the userfont entry rather than the actual font. // Ensure existence of mFTFace in the original entry - data.mFontEntry->CairoFontFace(); - if (!data.mFontEntry->mFTFace) { + fontEntry->CairoFontFace(); + if (!fontEntry->mFTFace) { return nullptr; } FT2FontEntry* fe = - FT2FontEntry::CreateFontEntry(data.mFontEntry->mFTFace, - data.mFontEntry->mFilename.get(), - data.mFontEntry->mFTFontIndex, - data.mFontEntry->Name(), nullptr); + FT2FontEntry::CreateFontEntry(fontEntry->mFTFace, + fontEntry->mFilename.get(), + fontEntry->mFTFontIndex, + fontEntry->Name(), nullptr); if (fe) { fe->mItalic = aItalic; fe->mWeight = aWeight; @@ -1569,21 +1549,15 @@ gfxFT2FontList::MakePlatformFont(const nsAString& aFontName, aItalic, aFontData, aLength); } -static PLDHashOperator -AppendFamily(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamily, - void* aUserArg) -{ - nsTArray > * familyArray = - reinterpret_cast>*>(aUserArg); - - familyArray->AppendElement(aFamily); - return PL_DHASH_NEXT; -} - void gfxFT2FontList::GetFontFamilyList(nsTArray >& aFamilyArray) { - mFontFamilies.Enumerate(AppendFamily, &aFamilyArray); - mHiddenFontFamilies.Enumerate(AppendFamily, &aFamilyArray); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + aFamilyArray.AppendElement(family); + } + for (auto iter = mHiddenFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + aFamilyArray.AppendElement(family); + } } diff --git a/gfx/thebes/gfxFont.cpp b/gfx/thebes/gfxFont.cpp index ae21f69f08e..dad88ce80f9 100644 --- a/gfx/thebes/gfxFont.cpp +++ b/gfx/thebes/gfxFont.cpp @@ -428,7 +428,7 @@ gfxFontShaper::MergeFontFeatures( bool aDisableLigatures, const nsAString& aFamilyName, bool aAddSmallCaps, - PLDHashOperator (*aHandleFeature)(const uint32_t&, uint32_t&, void*), + void (*aHandleFeature)(const uint32_t&, uint32_t&, void*), void* aHandleFeatureData) { uint32_t numAlts = aStyle->alternateValues.Length(); @@ -530,7 +530,9 @@ gfxFontShaper::MergeFontFeatures( } if (mergedFeatures.Count() != 0) { - mergedFeatures.Enumerate(aHandleFeature, aHandleFeatureData); + for (auto iter = mergedFeatures.Iter(); !iter.Done(); iter.Next()) { + aHandleFeature(iter.Key(), iter.Data(), aHandleFeatureData); + } } } diff --git a/gfx/thebes/gfxFont.h b/gfx/thebes/gfxFont.h index cb8f97dd2ae..547ae441786 100644 --- a/gfx/thebes/gfxFont.h +++ b/gfx/thebes/gfxFont.h @@ -636,8 +636,8 @@ public: bool aDisableLigatures, const nsAString& aFamilyName, bool aAddSmallCaps, - PLDHashOperator (*aHandleFeature)(const uint32_t&, - uint32_t&, void*), + void (*aHandleFeature)(const uint32_t&, + uint32_t&, void*), void* aHandleFeatureData); protected: diff --git a/gfx/thebes/gfxFontEntry.cpp b/gfx/thebes/gfxFontEntry.cpp index 7f734e9c34a..0596580d379 100644 --- a/gfx/thebes/gfxFontEntry.cpp +++ b/gfx/thebes/gfxFontEntry.cpp @@ -146,13 +146,6 @@ gfxFontEntry::gfxFontEntry(const nsAString& aName, bool aIsStandardFace) : memset(&mNonDefaultSubSpaceFeatures, 0, sizeof(mNonDefaultSubSpaceFeatures)); } -static PLDHashOperator -DestroyHBSet(const uint32_t& aTag, hb_set_t*& aSet, void *aUserArg) -{ - hb_set_destroy(aSet); - return PL_DHASH_NEXT; -} - gfxFontEntry::~gfxFontEntry() { if (mCOLR) { @@ -170,7 +163,10 @@ gfxFontEntry::~gfxFontEntry() } if (mFeatureInputs) { - mFeatureInputs->Enumerate(DestroyHBSet, nullptr); + for (auto iter = mFeatureInputs->Iter(); !iter.Done(); iter.Next()) { + hb_set_t*& set = iter.Data(); + hb_set_destroy(set); + } } // By the time the entry is destroyed, all font instances that were diff --git a/gfx/thebes/gfxGraphiteShaper.cpp b/gfx/thebes/gfxGraphiteShaper.cpp index ee7b4db3a5f..11e24c60155 100644 --- a/gfx/thebes/gfxGraphiteShaper.cpp +++ b/gfx/thebes/gfxGraphiteShaper.cpp @@ -72,7 +72,7 @@ struct GrFontFeatures { gr_feature_val *mFeatures; }; -static PLDHashOperator +static void AddFeature(const uint32_t& aTag, uint32_t& aValue, void *aUserArg) { GrFontFeatures *f = static_cast(aUserArg); @@ -81,7 +81,6 @@ AddFeature(const uint32_t& aTag, uint32_t& aValue, void *aUserArg) if (fref) { gr_fref_set_feature_value(fref, aValue, f->mFeatures); } - return PL_DHASH_NEXT; } bool diff --git a/gfx/thebes/gfxHarfBuzzShaper.cpp b/gfx/thebes/gfxHarfBuzzShaper.cpp index 4a683c197b4..92d7c5f96ad 100644 --- a/gfx/thebes/gfxHarfBuzzShaper.cpp +++ b/gfx/thebes/gfxHarfBuzzShaper.cpp @@ -1224,7 +1224,7 @@ HBUnicodeDecompose(hb_unicode_funcs_t *ufuncs, #endif } -static PLDHashOperator +static void AddOpenTypeFeature(const uint32_t& aTag, uint32_t& aValue, void *aUserArg) { nsTArray* features = static_cast*> (aUserArg); @@ -1233,7 +1233,6 @@ AddOpenTypeFeature(const uint32_t& aTag, uint32_t& aValue, void *aUserArg) feat.tag = aTag; feat.value = aValue; features->AppendElement(feat); - return PL_DHASH_NEXT; } /* diff --git a/gfx/thebes/gfxPlatformFontList.cpp b/gfx/thebes/gfxPlatformFontList.cpp index 4e5f37be836..84ddaddef28 100644 --- a/gfx/thebes/gfxPlatformFontList.cpp +++ b/gfx/thebes/gfxPlatformFontList.cpp @@ -237,18 +237,9 @@ gfxPlatformFontList::GenerateFontListKey(const nsAString& aKeyName, nsAString& a ToLowerCase(aResult); } -struct InitOtherNamesData { - InitOtherNamesData(gfxPlatformFontList *aFontList, - TimeStamp aStartTime) - : mFontList(aFontList), mStartTime(aStartTime), mTimedOut(false) - {} +#define OTHERNAMES_TIMEOUT 200 - gfxPlatformFontList *mFontList; - TimeStamp mStartTime; - bool mTimedOut; -}; - -void +void gfxPlatformFontList::InitOtherFamilyNames() { if (mOtherFamilyNamesInitialized) { @@ -256,14 +247,19 @@ gfxPlatformFontList::InitOtherFamilyNames() } TimeStamp start = TimeStamp::Now(); + bool timedOut = false; - // iterate over all font families and read in other family names - InitOtherNamesData otherNamesData(this, start); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + family->ReadOtherFamilyNames(this); + TimeDuration elapsed = TimeStamp::Now() - start; + if (elapsed.ToMilliseconds() > OTHERNAMES_TIMEOUT) { + timedOut = true; + break; + } + } - mFontFamilies.Enumerate(gfxPlatformFontList::InitOtherFamilyNamesProc, - &otherNamesData); - - if (!otherNamesData.mTimedOut) { + if (!timedOut) { mOtherFamilyNamesInitialized = true; } TimeStamp end = TimeStamp::Now(); @@ -274,55 +270,44 @@ gfxPlatformFontList::InitOtherFamilyNames() TimeDuration elapsed = end - start; LOG_FONTINIT(("(fontinit) InitOtherFamilyNames took %8.2f ms %s", elapsed.ToMilliseconds(), - (otherNamesData.mTimedOut ? "timeout" : ""))); + (timedOut ? "timeout" : ""))); } } -#define OTHERNAMES_TIMEOUT 200 - -PLDHashOperator -gfxPlatformFontList::InitOtherFamilyNamesProc(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void* userArg) -{ - InitOtherNamesData *data = static_cast(userArg); - - aFamilyEntry->ReadOtherFamilyNames(data->mFontList); - TimeDuration elapsed = TimeStamp::Now() - data->mStartTime; - if (elapsed.ToMilliseconds() > OTHERNAMES_TIMEOUT) { - data->mTimedOut = true; - return PL_DHASH_STOP; - } - return PL_DHASH_NEXT; -} - -struct ReadFaceNamesData { - ReadFaceNamesData(gfxPlatformFontList *aFontList, TimeStamp aStartTime) - : mFontList(aFontList), mStartTime(aStartTime), mTimedOut(false), - mFirstChar(0) - {} - - gfxPlatformFontList *mFontList; - TimeStamp mStartTime; - bool mTimedOut; - - // if mFirstChar is not 0, only load facenames for families - // that start with this character - char16_t mFirstChar; -}; +// time limit for loading facename lists (ms) +#define NAMELIST_TIMEOUT 200 gfxFontEntry* gfxPlatformFontList::SearchFamiliesForFaceName(const nsAString& aFaceName) { TimeStamp start = TimeStamp::Now(); + bool timedOut = false; + // if mFirstChar is not 0, only load facenames for families + // that start with this character + char16_t firstChar = 0; gfxFontEntry *lookup = nullptr; - ReadFaceNamesData faceNameListsData(this, start); - // iterate over familes starting with the same letter - faceNameListsData.mFirstChar = ToLowerCase(aFaceName.CharAt(0)); - mFontFamilies.Enumerate(gfxPlatformFontList::ReadFaceNamesProc, - &faceNameListsData); + firstChar = ToLowerCase(aFaceName.CharAt(0)); + + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsStringHashKey::KeyType key = iter.Key(); + nsRefPtr& family = iter.Data(); + + // when filtering, skip names that don't start with the filter character + if (firstChar && ToLowerCase(key.CharAt(0)) != firstChar) { + continue; + } + + family->ReadFaceNames(this, NeedFullnamePostscriptNames()); + + TimeDuration elapsed = TimeStamp::Now() - start; + if (elapsed.ToMilliseconds() > NAMELIST_TIMEOUT) { + timedOut = true; + break; + } + } + lookup = FindFaceName(aFaceName); TimeStamp end = TimeStamp::Now(); @@ -333,38 +318,12 @@ gfxPlatformFontList::SearchFamiliesForFaceName(const nsAString& aFaceName) LOG_FONTINIT(("(fontinit) SearchFamiliesForFaceName took %8.2f ms %s %s", elapsed.ToMilliseconds(), (lookup ? "found name" : ""), - (faceNameListsData.mTimedOut ? "timeout" : ""))); + (timedOut ? "timeout" : ""))); } return lookup; } -// time limit for loading facename lists (ms) -#define NAMELIST_TIMEOUT 200 - -PLDHashOperator -gfxPlatformFontList::ReadFaceNamesProc(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void* userArg) -{ - ReadFaceNamesData *data = static_cast(userArg); - gfxPlatformFontList *fc = data->mFontList; - - // when filtering, skip names that don't start with the filter character - if (data->mFirstChar && ToLowerCase(aKey.CharAt(0)) != data->mFirstChar) { - return PL_DHASH_NEXT; - } - - aFamilyEntry->ReadFaceNames(fc, fc->NeedFullnamePostscriptNames()); - - TimeDuration elapsed = TimeStamp::Now() - data->mStartTime; - if (elapsed.ToMilliseconds() > NAMELIST_TIMEOUT) { - data->mTimedOut = true; - return PL_DHASH_STOP; - } - return PL_DHASH_NEXT; -} - gfxFontEntry* gfxPlatformFontList::FindFaceName(const nsAString& aFaceName) { @@ -449,84 +408,49 @@ gfxPlatformFontList::UpdateFontList() RebuildLocalFonts(); } -struct FontListData { - FontListData(nsIAtom *aLangGroup, - const nsACString& aGenericFamily, - nsTArray& aListOfFonts) : - mLangGroup(aLangGroup), mGenericFamily(aGenericFamily), - mListOfFonts(aListOfFonts) {} - nsIAtom *mLangGroup; - const nsACString& mGenericFamily; - nsTArray& mListOfFonts; -}; - -PLDHashOperator -gfxPlatformFontList::HashEnumFuncForFamilies(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void *aUserArg) -{ - FontListData *data = static_cast(aUserArg); - - // use the first variation for now. This data should be the same - // for all the variations and should probably be moved up to - // the Family - gfxFontStyle style; - style.language = data->mLangGroup; - bool needsBold; - nsRefPtr aFontEntry = aFamilyEntry->FindFontForStyle(style, needsBold); - NS_ASSERTION(aFontEntry, "couldn't find any font entry in family"); - if (!aFontEntry) - return PL_DHASH_NEXT; - - /* skip symbol fonts */ - if (aFontEntry->IsSymbolFont()) - return PL_DHASH_NEXT; - - if (aFontEntry->SupportsLangGroup(data->mLangGroup) && - aFontEntry->MatchesGenericFamily(data->mGenericFamily)) { - nsAutoString localizedFamilyName; - aFamilyEntry->LocalizedName(localizedFamilyName); - data->mListOfFonts.AppendElement(localizedFamilyName); - } - - return PL_DHASH_NEXT; -} - void gfxPlatformFontList::GetFontList(nsIAtom *aLangGroup, const nsACString& aGenericFamily, nsTArray& aListOfFonts) { - FontListData data(aLangGroup, aGenericFamily, aListOfFonts); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + // use the first variation for now. This data should be the same + // for all the variations and should probably be moved up to + // the Family + gfxFontStyle style; + style.language = aLangGroup; + bool needsBold; + nsRefPtr fontEntry = family->FindFontForStyle(style, needsBold); + NS_ASSERTION(fontEntry, "couldn't find any font entry in family"); + if (!fontEntry) { + continue; + } - mFontFamilies.Enumerate(gfxPlatformFontList::HashEnumFuncForFamilies, &data); + /* skip symbol fonts */ + if (fontEntry->IsSymbolFont()) { + continue; + } + + if (fontEntry->SupportsLangGroup(aLangGroup) && + fontEntry->MatchesGenericFamily(aGenericFamily)) { + nsAutoString localizedFamilyName; + family->LocalizedName(localizedFamilyName); + aListOfFonts.AppendElement(localizedFamilyName); + } + } aListOfFonts.Sort(); aListOfFonts.Compact(); } -struct FontFamilyListData { - explicit FontFamilyListData(nsTArray >& aFamilyArray) - : mFamilyArray(aFamilyArray) - {} - - static PLDHashOperator AppendFamily(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void *aUserArg) - { - FontFamilyListData *data = static_cast(aUserArg); - data->mFamilyArray.AppendElement(aFamilyEntry); - return PL_DHASH_NEXT; - } - - nsTArray >& mFamilyArray; -}; - void gfxPlatformFontList::GetFontFamilyList(nsTArray >& aFamilyArray) { - FontFamilyListData data(aFamilyArray); - mFontFamilies.Enumerate(FontFamilyListData::AppendFamily, &data); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + aFamilyArray.AppendElement(family); + } } gfxFontEntry* @@ -616,18 +540,6 @@ gfxPlatformFontList::SystemFindFontForChar(uint32_t aCh, uint32_t aNextCh, return fontEntry; } -PLDHashOperator -gfxPlatformFontList::FindFontForCharProc(nsStringHashKey::KeyType aKey, nsRefPtr& aFamilyEntry, - void *userArg) -{ - GlobalFontMatch *data = static_cast(userArg); - - // evaluate all fonts in this family for a match - aFamilyEntry->FindFontForChar(data); - - return PL_DHASH_NEXT; -} - #define NUM_FALLBACK_FONTS 8 gfxFontEntry* @@ -677,7 +589,11 @@ gfxPlatformFontList::GlobalFontFallback(const uint32_t aCh, GlobalFontMatch data(aCh, aRunScript, aMatchStyle); // iterate over all font families to find a font that support the character - mFontFamilies.Enumerate(gfxPlatformFontList::FindFontForCharProc, &data); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + // evaluate all fonts in this family for a match + family->FindFontForChar(&data); + } aCmapCount = data.mCmapsTested; *aMatchedFamily = data.mMatchedFamily; @@ -885,22 +801,16 @@ gfxPlatformFontList::RemoveCmap(const gfxCharacterMap* aCharMap) } } -static PLDHashOperator AppendFamilyToList(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void *aUserArg) -{ - nsTArray *familyNames = static_cast *>(aUserArg); - familyNames->AppendElement(aFamilyEntry->Name()); - return PL_DHASH_NEXT; -} - void gfxPlatformFontList::GetFontFamilyNames(nsTArray& aFontFamilyNames) { - mFontFamilies.Enumerate(AppendFamilyToList, &aFontFamilyNames); + for (auto iter = mFontFamilies.Iter(); !iter.Done(); iter.Next()) { + nsRefPtr& family = iter.Data(); + aFontFamilyNames.AppendElement(family->Name()); + } } -void +void gfxPlatformFontList::InitLoader() { GetFontFamilyNames(mFontInfo->mFontFamiliesToLoad); diff --git a/gfx/thebes/gfxPlatformFontList.h b/gfx/thebes/gfxPlatformFontList.h index da173496858..97e3526041b 100644 --- a/gfx/thebes/gfxPlatformFontList.h +++ b/gfx/thebes/gfxPlatformFontList.h @@ -212,10 +212,6 @@ protected: static gfxPlatformFontList *sPlatformFontList; - static PLDHashOperator FindFontForCharProc(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void* userArg); - // Lookup family name in global family list without substitutions or // localized family name lookup. Used for common font fallback families. gfxFontFamily* FindFamilyByCanonicalName(const nsAString& aFamily) { @@ -251,21 +247,11 @@ protected: // initialize localized family names void InitOtherFamilyNames(); - static PLDHashOperator - InitOtherFamilyNamesProc(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void* userArg); - // search through font families, looking for a given name, initializing // facename lists along the way. first checks all families with names // close to face name, then searchs all families if not found. gfxFontEntry* SearchFamiliesForFaceName(const nsAString& aFaceName); - static PLDHashOperator - ReadFaceNamesProc(nsStringHashKey::KeyType aKey, - nsRefPtr& aFamilyEntry, - void* userArg); - // helper method for finding fullname/postscript names in facename lists gfxFontEntry* FindFaceName(const nsAString& aFaceName); From 7f03885fb033839f5b2b0fa65f54c88e4ce4b1ac Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:34:43 +0200 Subject: [PATCH 66/78] Bug 1083101 - Add a memory arena to Moz2D. r=jrmuizel --- gfx/2d/IterableArena.h | 193 ++++++++++++++++++++++++++++++++++ gfx/2d/moz.build | 1 + gfx/tests/gtest/TestArena.cpp | 188 +++++++++++++++++++++++++++++++++ gfx/tests/gtest/moz.build | 1 + 4 files changed, 383 insertions(+) create mode 100644 gfx/2d/IterableArena.h create mode 100644 gfx/tests/gtest/TestArena.cpp diff --git a/gfx/2d/IterableArena.h b/gfx/2d/IterableArena.h new file mode 100644 index 00000000000..0f993c9c110 --- /dev/null +++ b/gfx/2d/IterableArena.h @@ -0,0 +1,193 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_GFX_ITERABLEARENA_H_ +#define MOZILLA_GFX_ITERABLEARENA_H_ + +#include "mozilla/Move.h" +#include "mozilla/Assertions.h" +#include "mozilla/gfx/Logging.h" + +#include +#include +#include +#include + +namespace mozilla { +namespace gfx { + +/// A simple pool allocator for plain data structures. +/// +/// Beware that the pool will not attempt to run the destructors. It is the +/// responsibility of the user of this class to either use objects with no +/// destructor or to manually call the allocated objects destructors. +/// If the pool is growable, its allocated objects must be safely moveable in +/// in memory (through memcpy). +class IterableArena { +protected: + struct Header + { + size_t mBlocSize; + }; +public: + enum ArenaType { + FIXED_SIZE, + GROWABLE + }; + + IterableArena(ArenaType aType, size_t aStorageSize) + : mSize(aStorageSize) + , mCursor(0) + , mIsGrowable(aType == GROWABLE) + { + if (mSize == 0) { + mSize = 128; + } + + mStorage = (uint8_t*)malloc(mSize); + if (mStorage == nullptr) { + gfxCriticalError() << "Not enough Memory allocate a memory pool of size " << aStorageSize; + MOZ_CRASH(); + } + } + + ~IterableArena() + { + free(mStorage); + } + + /// Constructs a new item in the pool and returns a positive offset in case of + /// success. + /// + /// The offset never changes even if the storage is reallocated, so users + /// of this class should prefer storing offsets rather than direct pointers + /// to the allocated objects. + /// Alloc can cause the storage to be reallocated if the pool was initialized + /// with IterableArena::GROWABLE. + /// If for any reason the pool fails to allocate enough space for the new item + /// Alloc returns a negative offset and the object's constructor is not called. + template + ptrdiff_t + Alloc(Args&&... aArgs) + { + void* storage = nullptr; + auto offset = AllocRaw(sizeof(T), &storage); + if (offset < 0) { + return offset; + } + new (storage) T(Forward(aArgs)...); + return offset; + } + + ptrdiff_t AllocRaw(size_t aSize, void** aOutPtr = nullptr) + { + const size_t blocSize = AlignedSize(sizeof(Header) + aSize); + + if (AlignedSize(mCursor + blocSize) > mSize) { + if (!mIsGrowable) { + return -1; + } + + size_t newSize = mSize * 2; + while (AlignedSize(mCursor + blocSize) > newSize) { + newSize *= 2; + } + + uint8_t* newStorage = (uint8_t*)realloc(mStorage, newSize); + if (!newStorage) { + gfxCriticalError() << "Not enough Memory to grow the memory pool, size: " << newSize; + return -1; + } + + mStorage = newStorage; + mSize = newSize; + } + ptrdiff_t offset = mCursor; + GetHeader(offset)->mBlocSize = blocSize; + mCursor += blocSize; + if (aOutPtr) { + *aOutPtr = GetStorage(offset); + } + return offset; + } + + /// Get access to an allocated item at a given offset (only use offsets returned + /// by Alloc or AllocRaw). + /// + /// If the pool is growable, the returned pointer is only valid temporarily. The + /// underlying storage can be reallocated in Alloc or AllocRaw, so do not keep + /// these pointers around and store the offset instead. + void* GetStorage(ptrdiff_t offset = 0) + { + MOZ_ASSERT(offset >= 0); + MOZ_ASSERT(offset < mCursor); + return offset >= 0 ? mStorage + offset + sizeof(Header) : nullptr; + } + + /// Clears the storage without running any destructor and without deallocating it. + void Clear() + { + mCursor = 0; + } + + /// Iterate over the elements allocated in this pool. + /// + /// Takes a lambda or function object accepting a void* as parameter. + template + void ForEach(Func cb) + { + Iterator it; + while (void* ptr = it.Next(this)) { + cb(ptr); + } + } + + /// A simple iterator over an arena. + class Iterator { + public: + Iterator() + : mCursor(0) + {} + + void* Next(IterableArena* aArena) + { + if (mCursor >= aArena->mCursor) { + return nullptr; + } + void* result = aArena->GetStorage(mCursor); + const size_t blocSize = aArena->GetHeader(mCursor)->mBlocSize; + MOZ_ASSERT(blocSize != 0); + mCursor += blocSize; + return result; + } + + private: + ptrdiff_t mCursor; + }; + +protected: + Header* GetHeader(ptrdiff_t offset) + { + return (Header*) (mStorage + offset); + } + + size_t AlignedSize(size_t aSize) const + { + const size_t alignment = sizeof(uintptr_t); + return aSize + (alignment - (aSize % alignment)) % alignment; + } + + uint8_t* mStorage; + uint32_t mSize; + ptrdiff_t mCursor; + bool mIsGrowable; + + friend class Iterator; +}; + +} // namespace +} // namespace + +#endif diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index b7981946bba..be08de2b57b 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -25,6 +25,7 @@ EXPORTS.mozilla.gfx += [ 'Filters.h', 'Helpers.h', 'HelpersCairo.h', + 'IterableArena.h', 'Logging.h', 'Matrix.h', 'NumericTools.h', diff --git a/gfx/tests/gtest/TestArena.cpp b/gfx/tests/gtest/TestArena.cpp new file mode 100644 index 00000000000..4822b32f954 --- /dev/null +++ b/gfx/tests/gtest/TestArena.cpp @@ -0,0 +1,188 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "mozilla/gfx/IterableArena.h" +#include + +using namespace mozilla; +using namespace mozilla::gfx; + +#ifdef A +#undef A +#endif + +#ifdef B +#undef B +#endif + +// to avoid having symbols that collide easily like A and B in the global namespace +namespace test_arena { + +class A; +class B; + +class Base { +public: + virtual ~Base() {} + virtual A* AsA() { return nullptr; } + virtual B* AsB() { return nullptr; } +}; + +static int sDtorItemA = 0; +static int sDtorItemB = 0; + +class A : public Base { +public: + virtual A* AsA() override { return this; } + + explicit A(uint64_t val) : mVal(val) {} + ~A() { ++sDtorItemA; } + + uint64_t mVal; +}; + +class B : public Base { +public: + virtual B* AsB() override { return this; } + + explicit B(const string& str) : mVal(str) {} + ~B() { ++sDtorItemB; } + + std::string mVal; +}; + +struct BigStruct { + uint64_t mVal; + uint8_t data[120]; + + explicit BigStruct(uint64_t val) : mVal(val) {} +}; + +void TestArenaAlloc(IterableArena::ArenaType aType) +{ + sDtorItemA = 0; + sDtorItemB = 0; + IterableArena arena(aType, 256); + + // An empty arena has no items to iterate over. + { + int iterations = 0; + arena.ForEach([&](void* item){ + iterations++; + }); + ASSERT_EQ(iterations, 0); + } + + auto a1 = arena.Alloc(42); + auto b1 = arena.Alloc("Obladi oblada"); + auto a2 = arena.Alloc(1337); + auto b2 = arena.Alloc("Yellow submarine"); + auto b3 = arena.Alloc("She's got a ticket to ride"); + + // Alloc returns a non-negative offset if the allocation succeeded. + ASSERT_TRUE(a1 >= 0); + ASSERT_TRUE(a2 >= 0); + ASSERT_TRUE(b1 >= 0); + ASSERT_TRUE(b2 >= 0); + ASSERT_TRUE(b3 >= 0); + + ASSERT_TRUE(arena.GetStorage(a1) != nullptr); + ASSERT_TRUE(arena.GetStorage(a2) != nullptr); + ASSERT_TRUE(arena.GetStorage(b1) != nullptr); + ASSERT_TRUE(arena.GetStorage(b2) != nullptr); + ASSERT_TRUE(arena.GetStorage(b3) != nullptr); + + ASSERT_TRUE(((Base*)arena.GetStorage(a1))->AsA() != nullptr); + ASSERT_TRUE(((Base*)arena.GetStorage(a2))->AsA() != nullptr); + + ASSERT_TRUE(((Base*)arena.GetStorage(b1))->AsB() != nullptr); + ASSERT_TRUE(((Base*)arena.GetStorage(b2))->AsB() != nullptr); + ASSERT_TRUE(((Base*)arena.GetStorage(b3))->AsB() != nullptr); + + ASSERT_EQ(((Base*)arena.GetStorage(a1))->AsA()->mVal, (uint64_t)42); + ASSERT_EQ(((Base*)arena.GetStorage(a2))->AsA()->mVal, (uint64_t)1337); + + ASSERT_EQ(((Base*)arena.GetStorage(b1))->AsB()->mVal, std::string("Obladi oblada")); + ASSERT_EQ(((Base*)arena.GetStorage(b2))->AsB()->mVal, std::string("Yellow submarine")); + ASSERT_EQ(((Base*)arena.GetStorage(b3))->AsB()->mVal, std::string("She's got a ticket to ride")); + + { + int iterations = 0; + arena.ForEach([&](void* item){ + iterations++; + }); + ASSERT_EQ(iterations, 5); + } + + // Typically, running the destructors of the elements in the arena will is done + // manually like this: + arena.ForEach([](void* item){ + ((Base*)item)->~Base(); + }); + arena.Clear(); + ASSERT_EQ(sDtorItemA, 2); + ASSERT_EQ(sDtorItemB, 3); + + // An empty arena has no items to iterate over (we just cleared it). + { + int iterations = 0; + arena.ForEach([&](void* item){ + iterations++; + }); + ASSERT_EQ(iterations, 0); + } + +} + +void TestArenaLimit(IterableArena::ArenaType aType, bool aShouldReachLimit) +{ + IterableArena arena(aType, 128); + + // A non-growable arena should return a negative offset when running out + // of space, without crashing. + // We should not run out of space with a growable arena (unless the os is + // running out of memory but this isn't expected for this test). + bool reachedLimit = false; + for (int i = 0; i < 100; ++i) { + auto offset = arena.Alloc(42); + if (offset < 0) { + reachedLimit = true; + break; + } + } + ASSERT_EQ(reachedLimit, aShouldReachLimit); +} + +} // namespace test_arena + +using namespace test_arena; + +TEST(Moz2D, FixedArena) { + TestArenaAlloc(IterableArena::FIXED_SIZE); + TestArenaLimit(IterableArena::FIXED_SIZE, true); +} + +TEST(Moz2D, GrowableArena) { + TestArenaAlloc(IterableArena::GROWABLE); + TestArenaLimit(IterableArena::GROWABLE, false); + + IterableArena arena(IterableArena::GROWABLE, 16); + // sizeof(BigStruct) is more than twice the initial capacity, make sure that + // this doesn't blow everything up, since the arena doubles its storage size each + // time it grows (until it finds a size that fits). + auto a = arena.Alloc(1); + auto b = arena.Alloc(2); + auto c = arena.Alloc(3); + + // Offsets should also still point to the appropriate values after reallocation. + ASSERT_EQ(((BigStruct*)arena.GetStorage(a))->mVal, (uint64_t)1); + ASSERT_EQ(((BigStruct*)arena.GetStorage(b))->mVal, (uint64_t)2); + ASSERT_EQ(((BigStruct*)arena.GetStorage(c))->mVal, (uint64_t)3); + + arena.Clear(); +} diff --git a/gfx/tests/gtest/moz.build b/gfx/tests/gtest/moz.build index cda7439695d..c70cadbb097 100644 --- a/gfx/tests/gtest/moz.build +++ b/gfx/tests/gtest/moz.build @@ -8,6 +8,7 @@ UNIFIED_SOURCES += [ 'gfxSurfaceRefCountTest.cpp', # Disabled on suspicion of causing bug 904227 #'gfxWordCacheTest.cpp', + 'TestArena.cpp', 'TestBufferRotation.cpp', 'TestColorNames.cpp', 'TestCompositor.cpp', From 7bb2fc5f80054148ea633e06991855d4b30eaf29 Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:10 +0200 Subject: [PATCH 67/78] Bug 1083101 - Add a task scheduler to Moz2D. r=jrmuizel --- gfx/2d/JobScheduler.cpp | 236 +++++++++++++++++++++++++ gfx/2d/JobScheduler.h | 231 +++++++++++++++++++++++++ gfx/2d/JobScheduler_posix.cpp | 202 ++++++++++++++++++++++ gfx/2d/JobScheduler_posix.h | 187 ++++++++++++++++++++ gfx/2d/JobScheduler_win32.h | 76 +++++++++ gfx/2d/Types.h | 7 + gfx/2d/moz.build | 9 + gfx/tests/gtest/TestJobScheduler.cpp | 246 +++++++++++++++++++++++++++ gfx/tests/gtest/moz.build | 1 + 9 files changed, 1195 insertions(+) create mode 100644 gfx/2d/JobScheduler.cpp create mode 100644 gfx/2d/JobScheduler.h create mode 100644 gfx/2d/JobScheduler_posix.cpp create mode 100644 gfx/2d/JobScheduler_posix.h create mode 100644 gfx/2d/JobScheduler_win32.h create mode 100644 gfx/tests/gtest/TestJobScheduler.cpp diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp new file mode 100644 index 00000000000..8d7ab727e90 --- /dev/null +++ b/gfx/2d/JobScheduler.cpp @@ -0,0 +1,236 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "JobScheduler.h" + +namespace mozilla { +namespace gfx { + +JobScheduler* JobScheduler::sSingleton = nullptr; + +bool JobScheduler::Init(uint32_t aNumThreads, uint32_t aNumQueues) +{ + MOZ_ASSERT(!sSingleton); + MOZ_ASSERT(aNumThreads >= aNumQueues); + + sSingleton = new JobScheduler(); + sSingleton->mNextQueue = 0; + + for (uint32_t i = 0; i < aNumQueues; ++i) { + sSingleton->mDrawingQueues.push_back(new MultiThreadedJobQueue()); + } + + for (uint32_t i = 0; i < aNumThreads; ++i) { + sSingleton->mWorkerThreads.push_back(new WorkerThread(sSingleton->mDrawingQueues[i%aNumQueues])); + } + return true; +} + +void JobScheduler::ShutDown() +{ + MOZ_ASSERT(IsEnabled()); + if (!IsEnabled()) { + return; + } + + for (auto queue : sSingleton->mDrawingQueues) { + queue->ShutDown(); + delete queue; + } + + for (WorkerThread* thread : sSingleton->mWorkerThreads) { + // this will block until the thread is joined. + delete thread; + } + + sSingleton->mWorkerThreads.clear(); + delete sSingleton; + sSingleton = nullptr; +} + +JobStatus +JobScheduler::ProcessJob(Job* aJob) +{ + MOZ_ASSERT(aJob); + auto status = aJob->Run(); + if (status == JobStatus::Error || status == JobStatus::Complete) { + delete aJob; + } + return status; +} + +void +JobScheduler::SubmitJob(Job* aJob) +{ + MOZ_ASSERT(aJob); + RefPtr start = aJob->GetStartSync(); + if (start && start->Register(aJob)) { + // The Job buffer starts with a non-signaled sync object, it + // is now registered in the list of task buffers waiting on the + // sync object, so we should not place it in the queue. + return; + } + + GetQueueForJob(aJob)->SubmitJob(aJob); +} + +MultiThreadedJobQueue* +JobScheduler::GetQueueForJob(Job* aJob) +{ + return aJob->IsPinnedToAThread() ? aJob->GetWorkerThread()->GetJobQueue() + : GetDrawingQueue(); +} + +Job::Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread) +: mStartSync(aStart) +, mCompletionSync(aCompletion) +, mPinToThread(aThread) +{ + if (mStartSync) { + mStartSync->AddSubsequent(this); + } + if (mCompletionSync) { + mCompletionSync->AddPrerequisite(this); + } +} + +Job::~Job() +{ + if (mCompletionSync) { + //printf(" -- Job %p dtor completion %p\n", this, mCompletionSync); + mCompletionSync->Signal(); + mCompletionSync = nullptr; + } +} + +JobStatus +SetEventJob::Run() +{ + mEvent->Set(); + return JobStatus::Complete; +} + +SetEventJob::SetEventJob(EventObject* aEvent, + SyncObject* aStart, SyncObject* aCompletion, + WorkerThread* aWorker) +: Job(aStart, aCompletion, aWorker) +, mEvent(aEvent) +{} + +SetEventJob::~SetEventJob() +{} + +SyncObject::SyncObject(uint32_t aNumPrerequisites) +: mSignals(aNumPrerequisites) +#ifdef DEBUG +, mNumPrerequisites(aNumPrerequisites) +, mAddedPrerequisites(0) +#endif +{} + +SyncObject::~SyncObject() +{ + MOZ_ASSERT(mWaitingJobs.size() == 0); +} + +bool +SyncObject::Register(Job* aJob) +{ + MOZ_ASSERT(aJob); + + // For now, ensure that when we schedule the first subsequent, we have already + // created all of the prerequisites. This is an arbitrary restriction because + // we specify the number of prerequisites in the constructor, but in the typical + // scenario, if the assertion FreezePrerequisite blows up here it probably means + // we got the initial nmber of prerequisites wrong. We can decide to remove + // this restriction if needed. + FreezePrerequisites(); + + int32_t signals = mSignals; + + if (signals > 0) { + AddWaitingJob(aJob); + // Since Register and Signal can be called concurrently, it can happen that + // reading mSignals in Register happens before decrementing mSignals in Signal, + // but SubmitWaitingJobs happens before AddWaitingJob. This ordering means + // the SyncObject ends up in the signaled state with a task sitting in the + // waiting list. To prevent that we check mSignals a second time and submit + // again if signals reached zero in the mean time. + // We do this instead of holding a mutex around mSignals+mJobs to reduce + // lock contention. + int32_t signals2 = mSignals; + if (signals2 == 0) { + SubmitWaitingJobs(); + } + return true; + } + + return false; +} + +void +SyncObject::Signal() +{ + int32_t signals = --mSignals; + MOZ_ASSERT(signals >= 0); + + if (signals == 0) { + SubmitWaitingJobs(); + } +} + +void +SyncObject::AddWaitingJob(Job* aJob) +{ + MutexAutoLock lock(&mMutex); + mWaitingJobs.push_back(aJob); +} + +void SyncObject::SubmitWaitingJobs() +{ + std::vector tasksToSubmit; + { + // Scheduling the tasks can cause code that modifies 's reference + // count to run concurrently, and cause the caller of this function to + // be owned by another thread. We need to make sure the reference count + // does not reach 0 on another thread before mWaitingJobs.clear(), so + // hold a strong ref to prevent that! + RefPtr kungFuDeathGrip(this); + + MutexAutoLock lock(&mMutex); + tasksToSubmit = Move(mWaitingJobs); + mWaitingJobs.clear(); + } + + for (Job* task : tasksToSubmit) { + JobScheduler::GetQueueForJob(task)->SubmitJob(task); + } +} + +bool +SyncObject::IsSignaled() +{ + return mSignals == 0; +} + +void +SyncObject::FreezePrerequisites() +{ + MOZ_ASSERT(mAddedPrerequisites == mNumPrerequisites); +} + +void +SyncObject::AddPrerequisite(Job* aJob) +{ + MOZ_ASSERT(++mAddedPrerequisites <= mNumPrerequisites); +} + +void +SyncObject::AddSubsequent(Job* aJob) +{ +} + +} //namespace +} //namespace diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h new file mode 100644 index 00000000000..e58cee721a1 --- /dev/null +++ b/gfx/2d/JobScheduler.h @@ -0,0 +1,231 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_GFX_TASKSCHEDULER_H_ +#define MOZILLA_GFX_TASKSCHEDULER_H_ + +#include "mozilla/RefPtr.h" +#include "mozilla/gfx/Types.h" + +#ifdef WIN32 +#include "mozilla/gfx/JobScheduler_win32.h" +#else +#include "mozilla/gfx/JobScheduler_posix.h" +#endif + +namespace mozilla { +namespace gfx { + +class MultiThreadedJobQueue; +class SyncObject; + +class JobScheduler { +public: + /// Return one of the queues that the drawing worker threads pull from, chosen + /// pseudo-randomly. + static MultiThreadedJobQueue* GetDrawingQueue() + { + return sSingleton->mDrawingQueues[ + sSingleton->mNextQueue++ % sSingleton->mDrawingQueues.size() + ]; + } + + /// Return one of the queues that the drawing worker threads pull from with a + /// hash to choose the queue. + /// + /// Calling this function several times with the same hash will yield the same queue. + static MultiThreadedJobQueue* GetDrawingQueue(uint32_t aHash) + { + return sSingleton->mDrawingQueues[ + aHash % sSingleton->mDrawingQueues.size() + ]; + } + + /// Return the task queue associated to the worker the task is pinned to if + /// the task is pinned to a worker, or a random queue. + static MultiThreadedJobQueue* GetQueueForJob(Job* aJob); + + /// Initialize the task scheduler with aNumThreads worker threads for drawing + /// and aNumQueues task queues. + /// + /// The number of threads must be superior or equal to the number of queues + /// (since for now a worker thread only pulls from one queue). + static bool Init(uint32_t aNumThreads, uint32_t aNumQueues); + + /// Shut the scheduler down. + /// + /// This will block until worker threads are joined and deleted. + static void ShutDown(); + + /// Returns true if there is a successfully initialized JobScheduler singleton. + static bool IsEnabled() { return !!sSingleton; } + + /// Submit a task buffer to its associated queue. + /// + /// The caller looses ownership of the task buffer. + static void SubmitJob(Job* aJobs); + + /// Process commands until the command buffer needs to block on a sync object, + /// completes, yields, or encounters an error. + /// + /// Can be used on any thread. Worker threads basically loop over this, but the + /// main thread can also dequeue pending task buffers and process them alongside + /// the worker threads if it is about to block until completion anyway. + /// + /// The caller looses ownership of the task buffer. + static JobStatus ProcessJob(Job* aJobs); + +protected: + static JobScheduler* sSingleton; + + // queues of Job that are ready to be processed + std::vector mDrawingQueues; + std::vector mWorkerThreads; + Atomic mNextQueue; +}; + +/// Jobs are not reference-counted because they don't have shared ownership. +/// The ownership of tasks can change when they are passed to certain methods +/// of JobScheduler and SyncObject. See the docuumentaion of these classes. +class Job { +public: + Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread = nullptr); + + virtual ~Job(); + + virtual JobStatus Run() = 0; + + /// For use in JobScheduler::SubmitJob. Don't use it anywhere else. + //already_AddRefed GetAndResetStartSync(); + SyncObject* GetStartSync() { return mStartSync; } + + bool IsPinnedToAThread() const { return !!mPinToThread; } + + WorkerThread* GetWorkerThread() { return mPinToThread; } + +protected: + RefPtr mStartSync; + RefPtr mCompletionSync; + WorkerThread* mPinToThread; +}; + +class EventObject; + +/// This task will set an EventObject. +/// +/// Typically used as the final task, so that the main thread can block on the +/// corresponfing EventObject until all of the tasks are processed. +class SetEventJob : public Job +{ +public: + explicit SetEventJob(EventObject* aEvent, + SyncObject* aStart, SyncObject* aCompletion = nullptr, + WorkerThread* aPinToWorker = nullptr); + + ~SetEventJob(); + + JobStatus Run() override; + + EventObject* GetEvent() { return mEvent; } + +protected: + RefPtr mEvent; +}; + +/// A synchronization object that can be used to express dependencies and ordering between +/// tasks. +/// +/// Jobs can register to SyncObjects in order to asynchronously wait for a signal. +/// In practice, Job objects usually start with a sync object (startSyc) and end +/// with another one (completionSync). +/// a Job never gets processed before its startSync is in the signaled state, and +/// signals its completionSync as soon as it finishes. This is how dependencies +/// between tasks is expressed. +class SyncObject final : public external::AtomicRefCounted { +public: + MOZ_DECLARE_REFCOUNTED_TYPENAME(SyncObject) + + /// Create a synchronization object. + /// + /// aNumPrerequisites represents the number of times the object must be signaled + /// before actually entering the signaled state (in other words, it means the + /// number of dependencies of this sync object). + /// + /// Explicitly specifying the number of prerequisites when creating sync objects + /// makes it easy to start scheduling some of the prerequisite tasks while + /// creating the others, which is how we typically use the task scheduler. + /// Automatically determining the number of prerequisites using Job's constructor + /// brings the risk that the sync object enters the signaled state while we + /// are still adding prerequisites which is hard to fix without using muteces. + explicit SyncObject(uint32_t aNumPrerequisites = 1); + + ~SyncObject(); + + /// Attempt to register a task. + /// + /// If the sync object is already in the signaled state, the buffer is *not* + /// registered and the sync object does not take ownership of the task. + /// If the object is not yet in the signaled state, it takes ownership of + /// the task and places it in a list of pending tasks. + /// Pending tasks will not be processed by the worker thread. + /// When the SyncObject reaches the signaled state, it places the pending + /// tasks back in the available buffer queue, so that they can be + /// scheduled again. + /// + /// Returns true if the SyncOject is not already in the signaled state. + /// This means that if this method returns true, the SyncObject has taken + /// ownership of the Job. + bool Register(Job* aJob); + + /// Signal the SyncObject. + /// + /// This decrements an internal counter. The sync object reaches the signaled + /// state when the counter gets to zero. + void Signal(); + + /// Returns true if mSignals is equal to zero. In other words, returns true + /// if all prerequisite tasks have already signaled the sync object. + bool IsSignaled(); + + /// Asserts that the number of added prerequisites is equal to the number + /// specified in the constructor (does nothin in release builds). + void FreezePrerequisites(); + +private: + // Called by Job's constructor + void AddSubsequent(Job* aJob); + void AddPrerequisite(Job* aJob); + + void AddWaitingJob(Job* aJob); + + void SubmitWaitingJobs(); + + std::vector mWaitingJobs; + Mutex mMutex; // for concurrent access to mWaintingJobs + Atomic mSignals; + +#ifdef DEBUG + uint32_t mNumPrerequisites; + Atomic mAddedPrerequisites; +#endif + + friend class Job; + friend class JobScheduler; +}; + + +/// RAII helper. +struct MutexAutoLock { + MutexAutoLock(Mutex* aMutex) : mMutex(aMutex) { mMutex->Lock(); } + ~MutexAutoLock() { mMutex->Unlock(); } +protected: + Mutex* mMutex; +}; + + +} // namespace +} // namespace + +#endif \ No newline at end of file diff --git a/gfx/2d/JobScheduler_posix.cpp b/gfx/2d/JobScheduler_posix.cpp new file mode 100644 index 00000000000..0b7df32c7f6 --- /dev/null +++ b/gfx/2d/JobScheduler_posix.cpp @@ -0,0 +1,202 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "JobScheduler.h" +#include "mozilla/gfx/Logging.h" + +using namespace std; + +namespace mozilla { +namespace gfx { + +MultiThreadedJobQueue::MultiThreadedJobQueue() +: mThreadsCount(0) +, mShuttingDown(false) +{} + +MultiThreadedJobQueue::~MultiThreadedJobQueue() +{ + MOZ_ASSERT(mJobs.empty()); +} + +bool +MultiThreadedJobQueue::WaitForJob(Job*& aOutJob) +{ + return PopJob(aOutJob, BLOCKING); +} + +bool +MultiThreadedJobQueue::PopJob(Job*& aOutJobs, AccessType aAccess) +{ + for (;;) { + MutexAutoLock lock(&mMutex); + + while (aAccess == BLOCKING && !mShuttingDown && mJobs.empty()) { + mAvailableCondvar.Wait(&mMutex); + } + + if (mShuttingDown) { + return false; + } + + if (mJobs.empty()) { + if (aAccess == NON_BLOCKING) { + return false; + } + continue; + } + + Job* task = mJobs.front(); + MOZ_ASSERT(task); + + mJobs.pop_front(); + + aOutJobs = task; + return true; + } +} + +void +MultiThreadedJobQueue::SubmitJob(Job* aJobs) +{ + MOZ_ASSERT(aJobs); + MutexAutoLock lock(&mMutex); + mJobs.push_back(aJobs); + mAvailableCondvar.Broadcast(); +} + +size_t +MultiThreadedJobQueue::NumJobs() +{ + MutexAutoLock lock(&mMutex); + return mJobs.size(); +} + +bool +MultiThreadedJobQueue::IsEmpty() +{ + MutexAutoLock lock(&mMutex); + return mJobs.empty(); +} + +void +MultiThreadedJobQueue::ShutDown() +{ + MutexAutoLock lock(&mMutex); + mShuttingDown = true; + while (mThreadsCount) { + mAvailableCondvar.Broadcast(); + mShutdownCondvar.Wait(&mMutex); + } +} + +void +MultiThreadedJobQueue::RegisterThread() +{ + mThreadsCount += 1; +} + +void +MultiThreadedJobQueue::UnregisterThread() +{ + MutexAutoLock lock(&mMutex); + mThreadsCount -= 1; + if (mThreadsCount == 0) { + mShutdownCondvar.Broadcast(); + } +} + +void* ThreadCallback(void* threadData) +{ + WorkerThread* thread = (WorkerThread*)threadData; + thread->Run(); + return nullptr; +} + +WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) +: mQueue(aJobQueue) +{ + aJobQueue->RegisterThread(); + pthread_create(&mThread, nullptr, ThreadCallback, this); +} + +WorkerThread::~WorkerThread() +{ + pthread_join(mThread, nullptr); +} + +void +WorkerThread::SetName(const char* aName) +{ + // Call this from the thread itself because of Mac. +#ifdef XP_MACOSX + pthread_setname_np(aName); +#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) + pthread_set_name_np(mThread, aName); +#elif defined(__NetBSD__) + pthread_setname_np(mThread, "%s", (void*)aName); +#else + pthread_setname_np(mThread, aName); +#endif +} + +void +WorkerThread::Run() +{ + SetName("gfx worker"); + + for (;;) { + Job* commands = nullptr; + if (!mQueue->WaitForJob(commands)) { + mQueue->UnregisterThread(); + return; + } + + JobStatus status = JobScheduler::ProcessJob(commands); + + if (status == JobStatus::Error) { + // Don't try to handle errors for now, but that's open to discussions. + // I expect errors to be mostly OOM issues. + MOZ_CRASH(); + } + } +} + +EventObject::EventObject() +: mIsSet(false) +{} + +EventObject::~EventObject() +{} + +bool +EventObject::Peak() +{ + MutexAutoLock lock(&mMutex); + return mIsSet; +} + +void +EventObject::Set() +{ + MutexAutoLock lock(&mMutex); + if (!mIsSet) { + mIsSet = true; + mCond.Broadcast(); + } +} + +void +EventObject::Wait() +{ + MutexAutoLock lock(&mMutex); + if (mIsSet) { + return; + } + mCond.Wait(&mMutex); +} + +} // namespce +} // namespce diff --git a/gfx/2d/JobScheduler_posix.h b/gfx/2d/JobScheduler_posix.h new file mode 100644 index 00000000000..f4071c34e47 --- /dev/null +++ b/gfx/2d/JobScheduler_posix.h @@ -0,0 +1,187 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef WIN32 +#ifndef MOZILLA_GFX_TASKSCHEDULER_POSIX_H_ +#define MOZILLA_GFX_TASKSCHEDULER_POSIX_H_ + +#include +#include +#include +#include +#include +#include + +#include "mozilla/RefPtr.h" +#include "mozilla/DebugOnly.h" + +namespace mozilla { +namespace gfx { + +class Job; +class PosixCondVar; + +class Mutex { +public: + Mutex() { + DebugOnly err = pthread_mutex_init(&mMutex, nullptr); + MOZ_ASSERT(!err); + } + + ~Mutex() { + DebugOnly err = pthread_mutex_destroy(&mMutex); + MOZ_ASSERT(!err); + } + + void Lock() { + DebugOnly err = pthread_mutex_lock(&mMutex); + MOZ_ASSERT(!err); + } + + void Unlock() { + DebugOnly err = pthread_mutex_unlock(&mMutex); + MOZ_ASSERT(!err); + } + +protected: + pthread_mutex_t mMutex; + friend class PosixCondVar; +}; + +// posix platforms only! +class PosixCondVar { +public: + PosixCondVar() { + DebugOnly err = pthread_cond_init(&mCond, nullptr); + MOZ_ASSERT(!err); + } + + ~PosixCondVar() { + DebugOnly err = pthread_cond_destroy(&mCond); + MOZ_ASSERT(!err); + } + + void Wait(Mutex* aMutex) { + DebugOnly err = pthread_cond_wait(&mCond, &aMutex->mMutex); + MOZ_ASSERT(!err); + } + + void Broadcast() { + DebugOnly err = pthread_cond_broadcast(&mCond); + MOZ_ASSERT(!err); + } + +protected: + pthread_cond_t mCond; +}; + + +/// A simple and naive multithreaded task queue +/// +/// The public interface of this class must remain identical to its equivalent +/// in JobScheduler_win32.h +class MultiThreadedJobQueue { +public: + enum AccessType { + BLOCKING, + NON_BLOCKING + }; + + // Producer thread + MultiThreadedJobQueue(); + + // Producer thread + ~MultiThreadedJobQueue(); + + // Worker threads + bool WaitForJob(Job*& aOutJob); + + // Any thread + bool PopJob(Job*& aOutJob, AccessType aAccess); + + // Any threads + void SubmitJob(Job* aJob); + + // Producer thread + void ShutDown(); + + // Any thread + size_t NumJobs(); + + // Any thread + bool IsEmpty(); + + // Producer thread + void RegisterThread(); + + // Worker threads + void UnregisterThread(); + +protected: + + std::list mJobs; + Mutex mMutex; + PosixCondVar mAvailableCondvar; + PosixCondVar mShutdownCondvar; + int32_t mThreadsCount; + bool mShuttingDown; + + friend class WorkerThread; +}; + +/// Worker thread that continuously dequeues Jobs from a MultiThreadedJobQueue +/// and process them. +/// +/// The public interface of this class must remain identical to its equivalent +/// in JobScheduler_win32.h +class WorkerThread { +public: + explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); + + ~WorkerThread(); + + void Run(); + + MultiThreadedJobQueue* GetJobQueue() { return mQueue; } +protected: + void SetName(const char* name); + + MultiThreadedJobQueue* mQueue; + pthread_t mThread; +}; + +/// An object that a thread can synchronously wait on. +/// Usually set by a SetEventJob. +class EventObject : public external::AtomicRefCounted +{ +public: + MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) + + EventObject(); + + ~EventObject(); + + /// Synchronously wait until the event is set. + void Wait(); + + /// Return true if the event is set, without blocking. + bool Peak(); + + /// Set the event. + void Set(); + +protected: + Mutex mMutex; + PosixCondVar mCond; + bool mIsSet; +}; + +} // namespace +} // namespace + +#include "JobScheduler.h" + +#endif +#endif diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h new file mode 100644 index 00000000000..ca2db97dad0 --- /dev/null +++ b/gfx/2d/JobScheduler_win32.h @@ -0,0 +1,76 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef WIN32 +#ifndef MOZILLA_GFX_TASKSCHEDULER_WIN32_H_ +#define MOZILLA_GFX_TASKSCHEDULER_WIN32_H_ + +#define NOT_IMPLEMENTED MOZ_CRASH("Not implemented") + +#include "mozilla/RefPtr.h" + +namespace mozilla { +namespace gfx { + +class WorkerThread; +class Job; + +class Mutex { +public: + Mutex() { NOT_IMPLEMENTED; } + ~Mutex() { NOT_IMPLEMENTED; } + void Lock() { NOT_IMPLEMENTED; } + void Unlock() { NOT_IMPLEMENTED; } +}; + +// The public interface of this class must remain identical to its equivalent +// in JobScheduler_posix.h +class MultiThreadedJobQueue { +public: + enum AccessType { + BLOCKING, + NON_BLOCKING + }; + + bool WaitForJob(Job*& aOutCommands) { NOT_IMPLEMENTED; } + bool PopJob(Job*& aOutCommands, AccessType aAccess) { NOT_IMPLEMENTED; } + void SubmitJob(Job* aCommands) { NOT_IMPLEMENTED; } + void ShutDown() { NOT_IMPLEMENTED; } + size_t NumJobs() { NOT_IMPLEMENTED; } + bool IsEmpty() { NOT_IMPLEMENTED; } + void RegisterThread() { NOT_IMPLEMENTED; } + void UnregisterThread() { NOT_IMPLEMENTED; } + + friend class WorkerThread; +}; + + +// The public interface of this class must remain identical to its equivalent +// in JobScheduler_posix.h +class EventObject : public external::AtomicRefCounted +{ +public: + MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) + + EventObject() { NOT_IMPLEMENTED; } + ~EventObject() { NOT_IMPLEMENTED; } + void Wait() { NOT_IMPLEMENTED; } + bool Peak() { NOT_IMPLEMENTED; } + void Set() { NOT_IMPLEMENTED; } +}; + +// The public interface of this class must remain identical to its equivalent +// in JobScheduler_posix.h +class WorkerThread { +public: + explicit WorkerThread(MultiThreadedJobQueue* aJobQueue) { NOT_IMPLEMENTED; } + void Run(); +}; + +} // namespace +} // namespace + +#endif +#endif diff --git a/gfx/2d/Types.h b/gfx/2d/Types.h index 99143b6978e..145e8142969 100644 --- a/gfx/2d/Types.h +++ b/gfx/2d/Types.h @@ -289,6 +289,13 @@ struct GradientStop Color color; }; +enum class JobStatus { + Complete, + Wait, + Yield, + Error +}; + } // namespace gfx } // namespace mozilla diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index be08de2b57b..331fe3a55ab 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -26,6 +26,9 @@ EXPORTS.mozilla.gfx += [ 'Helpers.h', 'HelpersCairo.h', 'IterableArena.h', + 'JobScheduler.h', + 'JobScheduler_posix.h', + 'JobScheduler_win32.h', 'Logging.h', 'Matrix.h', 'NumericTools.h', @@ -71,6 +74,11 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': ] DEFINES['WIN32'] = True +if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'windows': + SOURCES += [ + 'JobScheduler_posix.cpp', + ] + if CONFIG['MOZ_ENABLE_SKIA']: UNIFIED_SOURCES += [ 'convolver.cpp', @@ -130,6 +138,7 @@ UNIFIED_SOURCES += [ 'FilterProcessing.cpp', 'FilterProcessingScalar.cpp', 'ImageScaling.cpp', + 'JobScheduler.cpp', 'Matrix.cpp', 'Path.cpp', 'PathCairo.cpp', diff --git a/gfx/tests/gtest/TestJobScheduler.cpp b/gfx/tests/gtest/TestJobScheduler.cpp new file mode 100644 index 00000000000..d1eed148c35 --- /dev/null +++ b/gfx/tests/gtest/TestJobScheduler.cpp @@ -0,0 +1,246 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef WIN32 + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "mozilla/gfx/JobScheduler.h" + +#include +#include +#include +#include + +namespace test_scheduler { + +using namespace mozilla::gfx; +using namespace mozilla; + +// Artificially cause threads to yield randomly in an attempt to make racy +// things more apparent (if any). +void MaybeYieldThread() +{ + if (rand() % 5 == 0) { + sched_yield(); + } +} + +/// Used by the TestCommand to check that tasks are processed in the right order. +struct SanityChecker { + std::vector mAdvancements; + mozilla::gfx::Mutex mMutex; + + explicit SanityChecker(uint64_t aNumCmdBuffers) + { + for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { + mAdvancements.push_back(0); + } + } + + virtual void Check(uint64_t aJobId, uint64_t aCmdId) + { + MaybeYieldThread(); + MutexAutoLock lock(&mMutex); + ASSERT_EQ(mAdvancements[aJobId], aCmdId-1); + mAdvancements[aJobId] = aCmdId; + } +}; + +/// Run checks that are specific to TestSchulerJoin. +struct JoinTestSanityCheck : public SanityChecker { + bool mSpecialJobHasRun; + + explicit JoinTestSanityCheck(uint64_t aNumCmdBuffers) + : SanityChecker(aNumCmdBuffers) + , mSpecialJobHasRun(false) + {} + + virtual void Check(uint64_t aJobId, uint64_t aCmdId) override + { + // Job 0 is the special task executed when everything is joined after task 1 + if (aCmdId == 0) { + ASSERT_FALSE(mSpecialJobHasRun); + mSpecialJobHasRun = true; + for (auto advancement : mAdvancements) { + // Because of the synchronization point (beforeFilter), all + // task buffers should have run task 1 when task 0 is run. + ASSERT_EQ(advancement, (uint32_t)1); + } + } else { + // This check does not apply to task 0. + SanityChecker::Check(aJobId, aCmdId); + } + + if (aCmdId == 2) { + ASSERT_TRUE(mSpecialJobHasRun); + } + } +}; + +class TestJob : public Job +{ +public: + TestJob(uint64_t aCmdId, uint64_t aJobId, SanityChecker* aChecker, + SyncObject* aStart, SyncObject* aCompletion) + : Job(aStart, aCompletion, nullptr) + , mCmdId(aCmdId) + , mCmdBufferId(aJobId) + , mSanityChecker(aChecker) + {} + + JobStatus Run() + { + MaybeYieldThread(); + mSanityChecker->Check(mCmdBufferId, mCmdId); + MaybeYieldThread(); + return JobStatus::Complete; + } + + uint64_t mCmdId; + uint64_t mCmdBufferId; + SanityChecker* mSanityChecker; +}; + +/// This test creates aNumCmdBuffers task buffers with sync objects set up +/// so that all tasks will join after command 5 before a task buffer runs +/// a special task (task 0) after which all task buffers fork again. +/// This simulates the kind of scenario where all tiles must join at +/// a certain point to execute, say, a filter, and fork again after the filter +/// has been processed. +/// The main thread is only blocked when waiting for the completion of the entire +/// task stream (it doesn't have to wait at the filter's sync points to orchestrate it). +void TestSchedulerJoin(uint32_t aNumThreads, uint32_t aNumCmdBuffers) +{ + JoinTestSanityCheck check(aNumCmdBuffers); + + RefPtr beforeFilter = new SyncObject(aNumCmdBuffers); + RefPtr afterFilter = new SyncObject(); + RefPtr completion = new SyncObject(aNumCmdBuffers); + + + for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { + Job* t1 = new TestJob(1, i, &check, nullptr, beforeFilter); + JobScheduler::SubmitJob(t1); + MaybeYieldThread(); + } + beforeFilter->FreezePrerequisites(); + + // This task buffer is executed when all other tasks have joined after task 1 + JobScheduler::SubmitJob( + new TestJob(0, 0, &check, beforeFilter, afterFilter) + ); + afterFilter->FreezePrerequisites(); + + for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { + Job* t2 = new TestJob(2, i, &check, afterFilter, completion); + JobScheduler::SubmitJob(t2); + MaybeYieldThread(); + } + completion->FreezePrerequisites(); + + RefPtr waitForCompletion = new EventObject(); + auto evtJob = new SetEventJob(waitForCompletion, completion); + JobScheduler::SubmitJob(evtJob); + + MaybeYieldThread(); + + waitForCompletion->Wait(); + + MaybeYieldThread(); + + for (auto advancement : check.mAdvancements) { + ASSERT_TRUE(advancement == 2); + } +} + +/// This test creates several chains of 10 task, tasks of a given chain are executed +/// sequentially, and chains are exectuted in parallel. +/// This simulates the typical scenario where we want to process sequences of drawing +/// commands for several tiles in parallel. +void TestSchedulerChain(uint32_t aNumThreads, uint32_t aNumCmdBuffers) +{ + SanityChecker check(aNumCmdBuffers); + + RefPtr completion = new SyncObject(aNumCmdBuffers); + + uint32_t numJobs = 10; + + for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { + + std::vector> syncs; + std::vector tasks; + syncs.reserve(numJobs); + tasks.reserve(numJobs); + + for (uint32_t t = 0; t < numJobs-1; ++t) { + syncs.push_back(new SyncObject()); + tasks.push_back(new TestJob(t+1, i, &check, t == 0 ? nullptr + : syncs[t-1].get(), + syncs[t])); + syncs.back()->FreezePrerequisites(); + } + + tasks.push_back(new TestJob(numJobs, i, &check, syncs.back(), completion)); + + if (i % 2 == 0) { + // submit half of the tasks in order + for (Job* task : tasks) { + JobScheduler::SubmitJob(task); + MaybeYieldThread(); + } + } else { + // ... and submit the other half in reverse order + for (int32_t reverse = numJobs-1; reverse >= 0; --reverse) { + JobScheduler::SubmitJob(tasks[reverse]); + MaybeYieldThread(); + } + } + } + completion->FreezePrerequisites(); + + RefPtr waitForCompletion = new EventObject(); + auto evtJob = new SetEventJob(waitForCompletion, completion); + JobScheduler::SubmitJob(evtJob); + + MaybeYieldThread(); + + waitForCompletion->Wait(); + + for (auto advancement : check.mAdvancements) { + ASSERT_TRUE(advancement == numJobs); + } +} + +} // namespace test_scheduler + +TEST(Moz2D, JobScheduler_Join) { + srand(time(nullptr)); + for (uint32_t threads = 1; threads < 8; ++threads) { + for (uint32_t queues = 1; queues < threads; ++queues) { + for (uint32_t buffers = 1; buffers < 100; buffers += 3) { + mozilla::gfx::JobScheduler::Init(threads, queues); + test_scheduler::TestSchedulerJoin(threads, buffers); + mozilla::gfx::JobScheduler::ShutDown(); + } + } + } +} + +TEST(Moz2D, JobScheduler_Chain) { + srand(time(nullptr)); + for (uint32_t threads = 1; threads < 8; ++threads) { + for (uint32_t queues = 1; queues < threads; ++queues) { + for (uint32_t buffers = 1; buffers < 100; buffers += 3) { + mozilla::gfx::JobScheduler::Init(threads, queues); + test_scheduler::TestSchedulerChain(threads, buffers); + mozilla::gfx::JobScheduler::ShutDown(); + } + } + } +} + +#endif diff --git a/gfx/tests/gtest/moz.build b/gfx/tests/gtest/moz.build index c70cadbb097..fedbbc6b06b 100644 --- a/gfx/tests/gtest/moz.build +++ b/gfx/tests/gtest/moz.build @@ -14,6 +14,7 @@ UNIFIED_SOURCES += [ 'TestCompositor.cpp', 'TestGfxPrefs.cpp', 'TestGfxWidgets.cpp', + 'TestJobScheduler.cpp', 'TestLayers.cpp', 'TestMoz2D.cpp', 'TestQcms.cpp', From e4fe7226bb3a6b167a8230959a12f82beb72a642 Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:15 +0200 Subject: [PATCH 68/78] Bug 1083101 - Implement gfx::DrawingJob. r=jrmuizel --- gfx/2d/DrawCommand.h | 48 ++++++----- gfx/2d/DrawTargetCapture.cpp | 2 +- gfx/2d/DrawingJob.cpp | 120 ++++++++++++++++++++++++++ gfx/2d/DrawingJob.h | 157 +++++++++++++++++++++++++++++++++++ gfx/2d/moz.build | 1 + 5 files changed, 305 insertions(+), 23 deletions(-) create mode 100644 gfx/2d/DrawingJob.cpp create mode 100644 gfx/2d/DrawingJob.h diff --git a/gfx/2d/DrawCommand.h b/gfx/2d/DrawCommand.h index 228d627c088..bad569c8b0c 100644 --- a/gfx/2d/DrawCommand.h +++ b/gfx/2d/DrawCommand.h @@ -39,7 +39,7 @@ class DrawingCommand public: virtual ~DrawingCommand() {} - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aTransform) = 0; + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform = nullptr) = 0; protected: explicit DrawingCommand(CommandType aType) @@ -130,7 +130,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->DrawSurface(mSurface, mDest, mSource, mSurfOptions, mOptions); } @@ -154,7 +154,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->DrawFilter(mFilter, mSourceRect, mDestPoint, mOptions); } @@ -175,7 +175,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->ClearRect(mRect); } @@ -197,11 +197,13 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aTransform) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform) { - MOZ_ASSERT(!aTransform.HasNonIntegerTranslation()); + MOZ_ASSERT(!aTransform || !aTransform->HasNonIntegerTranslation()); Point dest(Float(mDestination.x), Float(mDestination.y)); - dest = aTransform * dest; + if (aTransform) { + dest = (*aTransform) * dest; + } aDT->CopySurface(mSurface, mSourceRect, IntPoint(uint32_t(dest.x), uint32_t(dest.y))); } @@ -224,7 +226,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->FillRect(mRect, mPattern, mOptions); } @@ -250,7 +252,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->StrokeRect(mRect, mPattern, mStrokeOptions, mOptions); } @@ -279,7 +281,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->StrokeLine(mStart, mEnd, mPattern, mStrokeOptions, mOptions); } @@ -305,7 +307,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->Fill(mPath, mPattern, mOptions); } @@ -331,7 +333,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->Stroke(mPath, mPattern, mStrokeOptions, mOptions); } @@ -361,7 +363,7 @@ public: memcpy(&mGlyphs.front(), aBuffer.mGlyphs, sizeof(Glyph) * aBuffer.mNumGlyphs); } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { GlyphBuffer buf; buf.mNumGlyphs = mGlyphs.size(); @@ -390,7 +392,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->Mask(mSource, mMask, mOptions); } @@ -416,7 +418,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->MaskSurface(mSource, mMask, mOffset, mOptions); } @@ -437,7 +439,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->PushClip(mPath); } @@ -455,7 +457,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->PushClipRect(mRect); } @@ -472,7 +474,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) { aDT->PopClip(); } @@ -487,11 +489,13 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aMatrix) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aMatrix) { - Matrix transform = mTransform; - transform *= aMatrix; - aDT->SetTransform(transform); + if (aMatrix) { + aDT->SetTransform(mTransform * (*aMatrix)); + } else { + aDT->SetTransform(mTransform); + } } private: diff --git a/gfx/2d/DrawTargetCapture.cpp b/gfx/2d/DrawTargetCapture.cpp index 84d08d7d28c..f3dd4d0923e 100644 --- a/gfx/2d/DrawTargetCapture.cpp +++ b/gfx/2d/DrawTargetCapture.cpp @@ -188,7 +188,7 @@ DrawTargetCaptureImpl::ReplayToDrawTarget(DrawTarget* aDT, const Matrix& aTransf uint8_t* current = start; while (current < start + mDrawCommandStorage.size()) { - reinterpret_cast(current + sizeof(uint32_t))->ExecuteOnDT(aDT, aTransform); + reinterpret_cast(current + sizeof(uint32_t))->ExecuteOnDT(aDT, &aTransform); current += *(uint32_t*)current; } } diff --git a/gfx/2d/DrawingJob.cpp b/gfx/2d/DrawingJob.cpp new file mode 100644 index 00000000000..ce5ca20a282 --- /dev/null +++ b/gfx/2d/DrawingJob.cpp @@ -0,0 +1,120 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DrawingJob.h" +#include "JobScheduler.h" +#include "mozilla/gfx/2D.h" + +namespace mozilla { +namespace gfx { + +DrawingJobBuilder::DrawingJobBuilder() +{} + +DrawingJobBuilder::~DrawingJobBuilder() +{ + MOZ_ASSERT(!mDrawTarget); +} + +void +DrawingJob::Clear() +{ + mCommandBuffer = nullptr; + mCursor = 0; +} + +void +DrawingJobBuilder::BeginDrawingJob(DrawTarget* aTarget, IntPoint aOffset, + SyncObject* aStart) +{ + MOZ_ASSERT(mCommandOffsets.empty()); + MOZ_ASSERT(aTarget); + mDrawTarget = aTarget; + mOffset = aOffset; + mStart = aStart; +} + +DrawingJob* +DrawingJobBuilder::EndDrawingJob(CommandBuffer* aCmdBuffer, + SyncObject* aCompletion, + WorkerThread* aPinToWorker) +{ + MOZ_ASSERT(mDrawTarget); + DrawingJob* task = new DrawingJob(mDrawTarget, mOffset, mStart, aCompletion, aPinToWorker); + task->mCommandBuffer = aCmdBuffer; + task->mCommandOffsets = Move(mCommandOffsets); + + mDrawTarget = nullptr; + mOffset = IntPoint(); + mStart = nullptr; + + return task; +} + +DrawingJob::DrawingJob(DrawTarget* aTarget, IntPoint aOffset, + SyncObject* aStart, SyncObject* aCompletion, + WorkerThread* aPinToWorker) +: Job(aStart, aCompletion, aPinToWorker) +, mCommandBuffer(nullptr) +, mCursor(0) +, mDrawTarget(aTarget) +, mOffset(aOffset) +{ + mCommandOffsets.reserve(64); +} + +JobStatus +DrawingJob::Run() +{ + while (mCursor < mCommandOffsets.size()) { + + DrawingCommand* cmd = mCommandBuffer->GetDrawingCommand(mCommandOffsets[mCursor]); + + if (!cmd) { + return JobStatus::Error; + } + + cmd->ExecuteOnDT(mDrawTarget); + + ++mCursor; + } + + return JobStatus::Complete; +} + +DrawingJob::~DrawingJob() +{ + Clear(); +} + +DrawingCommand* +CommandBuffer::GetDrawingCommand(ptrdiff_t aId) +{ + return static_cast(mStorage.GetStorage(aId)); +} + +CommandBuffer::~CommandBuffer() +{ + mStorage.ForEach([](void* item){ + static_cast(item)->~DrawingCommand(); + }); + mStorage.Clear(); +} + +void +CommandBufferBuilder::BeginCommandBuffer(size_t aBufferSize) +{ + MOZ_ASSERT(!mCommands); + mCommands = new CommandBuffer(aBufferSize); +} + +already_AddRefed +CommandBufferBuilder::EndCommandBuffer() +{ + return mCommands.forget(); +} + +} // namespace +} // namespace diff --git a/gfx/2d/DrawingJob.h b/gfx/2d/DrawingJob.h new file mode 100644 index 00000000000..560d4f97c3b --- /dev/null +++ b/gfx/2d/DrawingJob.h @@ -0,0 +1,157 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_GFX_COMMANDBUFFER_H_ +#define MOZILLA_GFX_COMMANDBUFFER_H_ + +#include + +#include "mozilla/RefPtr.h" +#include "mozilla/Assertions.h" +#include "mozilla/gfx/Matrix.h" +#include "mozilla/gfx/JobScheduler.h" +#include "mozilla/gfx/IterableArena.h" +#include "DrawCommand.h" + +namespace mozilla { +namespace gfx { + +class DrawingCommand; +class PrintCommand; +class SignalCommand; +class DrawingJob; +class WaitCommand; + +class SyncObject; +class MultiThreadedJobQueue; + +class DrawTarget; + +class DrawingJobBuilder; +class CommandBufferBuilder; + +/// Contains a sequence of immutable drawing commands that are typically used by +/// several DrawingJobs. +/// +/// CommandBuffer objects are built using CommandBufferBuilder. +class CommandBuffer : public external::AtomicRefCounted +{ +public: + MOZ_DECLARE_REFCOUNTED_TYPENAME(CommandBuffer) + + ~CommandBuffer(); + + DrawingCommand* GetDrawingCommand(ptrdiff_t aId); + +protected: + explicit CommandBuffer(size_t aSize = 256) + : mStorage(IterableArena::GROWABLE, aSize) + {} + + IterableArena mStorage; + friend class CommandBufferBuilder; +}; + +/// Generates CommandBuffer objects. +/// +/// The builder is a separate object to ensure that commands are not added to a +/// submitted CommandBuffer. +class CommandBufferBuilder +{ +public: + void BeginCommandBuffer(size_t aBufferSize = 256); + + already_AddRefed EndCommandBuffer(); + + /// Build the CommandBuffer, command after command. + /// This must be used between BeginCommandBuffer and EndCommandBuffer. + template + ptrdiff_t AddCommand(Args&&... aArgs) + { + static_assert(IsBaseOf::value, + "T must derive from DrawingCommand"); + return mCommands->mStorage.Alloc(Forward(aArgs)...); + } + + bool HasCommands() const { return !!mCommands; } + +protected: + RefPtr mCommands; +}; + +/// Stores multiple commands to be executed sequencially. +class DrawingJob : public Job { +public: + ~DrawingJob(); + + virtual JobStatus Run() override; + +protected: + DrawingJob(DrawTarget* aTarget, + IntPoint aOffset, + SyncObject* aStart, + SyncObject* aCompletion, + WorkerThread* aPinToWorker = nullptr); + + /// Runs the tasks's destructors and resets the buffer. + void Clear(); + + std::vector mCommandOffsets; + RefPtr mCommandBuffer; + uint32_t mCursor; + + RefPtr mDrawTarget; + IntPoint mOffset; + + friend class DrawingJobBuilder; +}; + +/// Generates DrawingJob objects. +/// +/// The builder is a separate object to ensure that commands are not added to a +/// submitted DrawingJob. +class DrawingJobBuilder { +public: + DrawingJobBuilder(); + + ~DrawingJobBuilder(); + + /// Allocates a DrawingJob. + /// + /// call this method before starting to add commands. + void BeginDrawingJob(DrawTarget* aTarget, IntPoint aOffset, + SyncObject* aStart = nullptr); + + /// Build the DrawingJob, command after command. + /// This must be used between BeginDrawingJob and EndDrawingJob. + void AddCommand(ptrdiff_t offset) + { + mCommandOffsets.push_back(offset); + } + + /// Finalizes and returns the drawing task. + /// + /// If aCompletion is not null, the sync object will be signaled after the + /// task buffer is destroyed (and after the destructor of the tasks have run). + /// In most cases this means after the completion of all tasks in the task buffer, + /// but also when the task buffer is destroyed due to an error. + DrawingJob* EndDrawingJob(CommandBuffer* aCmdBuffer, + SyncObject* aCompletion = nullptr, + WorkerThread* aPinToWorker = nullptr); + + /// Returns true between BeginDrawingJob and EndDrawingJob, false otherwise. + bool HasDrawingJob() const { return !!mDrawTarget; } + +protected: + std::vector mCommandOffsets; + RefPtr mDrawTarget; + IntPoint mOffset; + RefPtr mStart; +}; + +} // namespace +} // namespace + +#endif diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index 331fe3a55ab..5480302b99d 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -127,6 +127,7 @@ UNIFIED_SOURCES += [ 'DataSourceSurface.cpp', 'DataSurfaceHelpers.cpp', 'DrawEventRecorder.cpp', + 'DrawingJob.cpp', 'DrawTarget.cpp', 'DrawTargetCairo.cpp', 'DrawTargetCapture.cpp', From 5101aabb74b485a1d812e8f6de1fb9259182f49e Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Wed, 10 Jun 2015 19:57:08 +0200 Subject: [PATCH 69/78] Bug 1083101 - Extend DrawCommand storage functionality and fix a bug with dashed stroke patterns. r=jrmuizel --- gfx/2d/DrawCommand.h | 88 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/gfx/2d/DrawCommand.h b/gfx/2d/DrawCommand.h index bad569c8b0c..76ae32f8dfc 100644 --- a/gfx/2d/DrawCommand.h +++ b/gfx/2d/DrawCommand.h @@ -6,6 +6,10 @@ #ifndef MOZILLA_GFX_DRAWCOMMAND_H_ #define MOZILLA_GFX_DRAWCOMMAND_H_ + +#define _USE_MATH_DEFINES +#include + #include "2D.h" #include "Filters.h" #include @@ -31,7 +35,8 @@ enum class CommandType : int8_t { PUSHCLIP, PUSHCLIPRECT, POPCLIP, - SETTRANSFORM + SETTRANSFORM, + FLUSH }; class DrawingCommand @@ -41,6 +46,8 @@ public: virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform = nullptr) = 0; + virtual bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) { return false; } + protected: explicit DrawingCommand(CommandType aType) : mType(aType) @@ -231,6 +238,12 @@ public: aDT->FillRect(mRect, mPattern, mOptions); } + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + { + aDeviceRect = aTransform.TransformBounds(mRect); + return true; + } + private: Rect mRect; StoredPattern mPattern; @@ -250,6 +263,11 @@ public: , mStrokeOptions(aStrokeOptions) , mOptions(aOptions) { + if (aStrokeOptions.mDashLength) { + mDashes.resize(aStrokeOptions.mDashLength); + mStrokeOptions.mDashPattern = &mDashes.front(); + memcpy(&mDashes.front(), aStrokeOptions.mDashPattern, mStrokeOptions.mDashLength * sizeof(Float)); + } } virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) @@ -262,6 +280,7 @@ private: StoredPattern mPattern; StrokeOptions mStrokeOptions; DrawOptions mOptions; + std::vector mDashes; }; class StrokeLineCommand : public DrawingCommand @@ -312,12 +331,53 @@ public: aDT->Fill(mPath, mPattern, mOptions); } + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + { + aDeviceRect = mPath->GetBounds(aTransform); + return true; + } + private: RefPtr mPath; StoredPattern mPattern; DrawOptions mOptions; }; +#ifndef M_SQRT2 +#define M_SQRT2 1.41421356237309504880 +#endif + +#ifndef M_SQRT1_2 +#define M_SQRT1_2 0.707106781186547524400844362104849039 +#endif + +// The logic for this comes from _cairo_stroke_style_max_distance_from_path +static Rect +PathExtentsToMaxStrokeExtents(const StrokeOptions &aStrokeOptions, + const Rect &aRect, + const Matrix &aTransform) +{ + double styleExpansionFactor = 0.5f; + + if (aStrokeOptions.mLineCap == CapStyle::SQUARE) { + styleExpansionFactor = M_SQRT1_2; + } + + if (aStrokeOptions.mLineJoin == JoinStyle::MITER && + styleExpansionFactor < M_SQRT2 * aStrokeOptions.mMiterLimit) { + styleExpansionFactor = M_SQRT2 * aStrokeOptions.mMiterLimit; + } + + styleExpansionFactor *= aStrokeOptions.mLineWidth; + + double dx = styleExpansionFactor * hypot(aTransform._11, aTransform._21); + double dy = styleExpansionFactor * hypot(aTransform._22, aTransform._12); + + Rect result = aRect; + result.Inflate(dx, dy); + return result; +} + class StrokeCommand : public DrawingCommand { public: @@ -331,6 +391,11 @@ public: , mStrokeOptions(aStrokeOptions) , mOptions(aOptions) { + if (aStrokeOptions.mDashLength) { + mDashes.resize(aStrokeOptions.mDashLength); + mStrokeOptions.mDashPattern = &mDashes.front(); + memcpy(&mDashes.front(), aStrokeOptions.mDashPattern, mStrokeOptions.mDashLength * sizeof(Float)); + } } virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) @@ -338,11 +403,18 @@ public: aDT->Stroke(mPath, mPattern, mStrokeOptions, mOptions); } + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + { + aDeviceRect = PathExtentsToMaxStrokeExtents(mStrokeOptions, mPath->GetBounds(aTransform), aTransform); + return true; + } + private: RefPtr mPath; StoredPattern mPattern; StrokeOptions mStrokeOptions; DrawOptions mOptions; + std::vector mDashes; }; class FillGlyphsCommand : public DrawingCommand @@ -502,6 +574,20 @@ private: Matrix mTransform; }; +class FlushCommand : public DrawingCommand +{ +public: + explicit FlushCommand() + : DrawingCommand(CommandType::FLUSH) + { + } + + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + { + aDT->Flush(); + } +}; + } // namespace gfx } // namespace mozilla From 3858146effe23eb10b9d1715c215e4fe07a3ef0b Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:27 +0200 Subject: [PATCH 70/78] Bug 1083101 - Make gfx::DrawingCommand methods const. r=jrmuizel --- gfx/2d/DrawCommand.h | 44 +++++++++++++++++++++---------------------- gfx/2d/DrawingJob.cpp | 4 ++-- gfx/2d/DrawingJob.h | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/gfx/2d/DrawCommand.h b/gfx/2d/DrawCommand.h index 76ae32f8dfc..135548b9c6d 100644 --- a/gfx/2d/DrawCommand.h +++ b/gfx/2d/DrawCommand.h @@ -44,9 +44,9 @@ class DrawingCommand public: virtual ~DrawingCommand() {} - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform = nullptr) = 0; + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform = nullptr) const = 0; - virtual bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) { return false; } + virtual bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const { return false; } protected: explicit DrawingCommand(CommandType aType) @@ -137,7 +137,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->DrawSurface(mSurface, mDest, mSource, mSurfOptions, mOptions); } @@ -161,7 +161,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->DrawFilter(mFilter, mSourceRect, mDestPoint, mOptions); } @@ -182,7 +182,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->ClearRect(mRect); } @@ -204,7 +204,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform) const { MOZ_ASSERT(!aTransform || !aTransform->HasNonIntegerTranslation()); Point dest(Float(mDestination.x), Float(mDestination.y)); @@ -233,12 +233,12 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->FillRect(mRect, mPattern, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const { aDeviceRect = aTransform.TransformBounds(mRect); return true; @@ -270,7 +270,7 @@ public: } } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->StrokeRect(mRect, mPattern, mStrokeOptions, mOptions); } @@ -300,7 +300,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->StrokeLine(mStart, mEnd, mPattern, mStrokeOptions, mOptions); } @@ -326,12 +326,12 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->Fill(mPath, mPattern, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const { aDeviceRect = mPath->GetBounds(aTransform); return true; @@ -398,12 +398,12 @@ public: } } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->Stroke(mPath, mPattern, mStrokeOptions, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) + bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const { aDeviceRect = PathExtentsToMaxStrokeExtents(mStrokeOptions, mPath->GetBounds(aTransform), aTransform); return true; @@ -435,7 +435,7 @@ public: memcpy(&mGlyphs.front(), aBuffer.mGlyphs, sizeof(Glyph) * aBuffer.mNumGlyphs); } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { GlyphBuffer buf; buf.mNumGlyphs = mGlyphs.size(); @@ -464,7 +464,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->Mask(mSource, mMask, mOptions); } @@ -490,7 +490,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->MaskSurface(mSource, mMask, mOffset, mOptions); } @@ -511,7 +511,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->PushClip(mPath); } @@ -529,7 +529,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->PushClipRect(mRect); } @@ -546,7 +546,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->PopClip(); } @@ -561,7 +561,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aMatrix) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aMatrix) const { if (aMatrix) { aDT->SetTransform(mTransform * (*aMatrix)); @@ -582,7 +582,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const { aDT->Flush(); } diff --git a/gfx/2d/DrawingJob.cpp b/gfx/2d/DrawingJob.cpp index ce5ca20a282..728e330f421 100644 --- a/gfx/2d/DrawingJob.cpp +++ b/gfx/2d/DrawingJob.cpp @@ -70,7 +70,7 @@ DrawingJob::Run() { while (mCursor < mCommandOffsets.size()) { - DrawingCommand* cmd = mCommandBuffer->GetDrawingCommand(mCommandOffsets[mCursor]); + const DrawingCommand* cmd = mCommandBuffer->GetDrawingCommand(mCommandOffsets[mCursor]); if (!cmd) { return JobStatus::Error; @@ -89,7 +89,7 @@ DrawingJob::~DrawingJob() Clear(); } -DrawingCommand* +const DrawingCommand* CommandBuffer::GetDrawingCommand(ptrdiff_t aId) { return static_cast(mStorage.GetStorage(aId)); diff --git a/gfx/2d/DrawingJob.h b/gfx/2d/DrawingJob.h index 560d4f97c3b..1f279bc3d86 100644 --- a/gfx/2d/DrawingJob.h +++ b/gfx/2d/DrawingJob.h @@ -43,7 +43,7 @@ public: ~CommandBuffer(); - DrawingCommand* GetDrawingCommand(ptrdiff_t aId); + const DrawingCommand* GetDrawingCommand(ptrdiff_t aId); protected: explicit CommandBuffer(size_t aSize = 256) From eece822466bcaa1bbf225c635c4b15aa78b383bb Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:30 +0200 Subject: [PATCH 71/78] Bug 1083101 - Win32 implementation of the JobScheduler. r=jrmuizel --- gfx/2d/JobScheduler.cpp | 30 +++++- gfx/2d/JobScheduler.h | 22 +++++ gfx/2d/JobScheduler_posix.cpp | 102 +++++++++---------- gfx/2d/JobScheduler_posix.h | 22 +---- gfx/2d/JobScheduler_win32.cpp | 143 +++++++++++++++++++++++++++ gfx/2d/JobScheduler_win32.h | 105 +++++++++++++++----- gfx/2d/moz.build | 1 + gfx/tests/gtest/TestJobScheduler.cpp | 9 +- 8 files changed, 328 insertions(+), 106 deletions(-) create mode 100644 gfx/2d/JobScheduler_win32.cpp diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp index 8d7ab727e90..e486f3b4c74 100644 --- a/gfx/2d/JobScheduler.cpp +++ b/gfx/2d/JobScheduler.cpp @@ -23,7 +23,7 @@ bool JobScheduler::Init(uint32_t aNumThreads, uint32_t aNumQueues) } for (uint32_t i = 0; i < aNumThreads; ++i) { - sSingleton->mWorkerThreads.push_back(new WorkerThread(sSingleton->mDrawingQueues[i%aNumQueues])); + sSingleton->mWorkerThreads.push_back(WorkerThread::Create(sSingleton->mDrawingQueues[i%aNumQueues])); } return true; } @@ -232,5 +232,33 @@ SyncObject::AddSubsequent(Job* aJob) { } +WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) +: mQueue(aJobQueue) +{ + aJobQueue->RegisterThread(); +} + +void +WorkerThread::Run() +{ + SetName("gfx worker"); + + for (;;) { + Job* commands = nullptr; + if (!mQueue->WaitForJob(commands)) { + mQueue->UnregisterThread(); + return; + } + + JobStatus status = JobScheduler::ProcessJob(commands); + + if (status == JobStatus::Error) { + // Don't try to handle errors for now, but that's open to discussions. + // I expect errors to be mostly OOM issues. + MOZ_CRASH(); + } + } +} + } //namespace } //namespace diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h index e58cee721a1..50da4874d5e 100644 --- a/gfx/2d/JobScheduler.h +++ b/gfx/2d/JobScheduler.h @@ -15,11 +15,14 @@ #include "mozilla/gfx/JobScheduler_posix.h" #endif +#include + namespace mozilla { namespace gfx { class MultiThreadedJobQueue; class SyncObject; +class WorkerThread; class JobScheduler { public: @@ -224,6 +227,25 @@ protected: Mutex* mMutex; }; +/// Base class for worker threads. +class WorkerThread +{ +public: + static WorkerThread* Create(MultiThreadedJobQueue* aJobQueue); + + virtual ~WorkerThread() {} + + void Run(); + + MultiThreadedJobQueue* GetJobQueue() { return mQueue; } + +protected: + explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); + + virtual void SetName(const char* aName) {} + + MultiThreadedJobQueue* mQueue; +}; } // namespace } // namespace diff --git a/gfx/2d/JobScheduler_posix.cpp b/gfx/2d/JobScheduler_posix.cpp index 0b7df32c7f6..c19103e28d8 100644 --- a/gfx/2d/JobScheduler_posix.cpp +++ b/gfx/2d/JobScheduler_posix.cpp @@ -11,6 +11,52 @@ using namespace std; namespace mozilla { namespace gfx { +void* ThreadCallback(void* threadData); + +class WorkerThreadPosix : public WorkerThread { +public: + explicit WorkerThreadPosix(MultiThreadedJobQueue* aJobQueue) + : WorkerThread(aJobQueue) + { + pthread_create(&mThread, nullptr, ThreadCallback, static_cast(this)); + } + + ~WorkerThreadPosix() + { + pthread_join(mThread, nullptr); + } + + virtual void SetName(const char* aName) override + { + // Call this from the thread itself because of Mac. +#ifdef XP_MACOSX + pthread_setname_np(aName); +#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) + pthread_set_name_np(mThread, aName); +#elif defined(__NetBSD__) + pthread_setname_np(mThread, "%s", (void*)aName); +#else + pthread_setname_np(mThread, aName); +#endif + } + +protected: + pthread_t mThread; +}; + +void* ThreadCallback(void* threadData) +{ + WorkerThread* thread = static_cast(threadData); + thread->Run(); + return nullptr; +} + +WorkerThread* +WorkerThread::Create(MultiThreadedJobQueue* aJobQueue) +{ + return new WorkerThreadPosix(aJobQueue); +} + MultiThreadedJobQueue::MultiThreadedJobQueue() : mThreadsCount(0) , mShuttingDown(false) @@ -108,62 +154,6 @@ MultiThreadedJobQueue::UnregisterThread() } } -void* ThreadCallback(void* threadData) -{ - WorkerThread* thread = (WorkerThread*)threadData; - thread->Run(); - return nullptr; -} - -WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) -: mQueue(aJobQueue) -{ - aJobQueue->RegisterThread(); - pthread_create(&mThread, nullptr, ThreadCallback, this); -} - -WorkerThread::~WorkerThread() -{ - pthread_join(mThread, nullptr); -} - -void -WorkerThread::SetName(const char* aName) -{ - // Call this from the thread itself because of Mac. -#ifdef XP_MACOSX - pthread_setname_np(aName); -#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) - pthread_set_name_np(mThread, aName); -#elif defined(__NetBSD__) - pthread_setname_np(mThread, "%s", (void*)aName); -#else - pthread_setname_np(mThread, aName); -#endif -} - -void -WorkerThread::Run() -{ - SetName("gfx worker"); - - for (;;) { - Job* commands = nullptr; - if (!mQueue->WaitForJob(commands)) { - mQueue->UnregisterThread(); - return; - } - - JobStatus status = JobScheduler::ProcessJob(commands); - - if (status == JobStatus::Error) { - // Don't try to handle errors for now, but that's open to discussions. - // I expect errors to be mostly OOM issues. - MOZ_CRASH(); - } - } -} - EventObject::EventObject() : mIsSet(false) {} diff --git a/gfx/2d/JobScheduler_posix.h b/gfx/2d/JobScheduler_posix.h index f4071c34e47..74cd975df22 100644 --- a/gfx/2d/JobScheduler_posix.h +++ b/gfx/2d/JobScheduler_posix.h @@ -22,6 +22,7 @@ namespace gfx { class Job; class PosixCondVar; +class WorkerThread; class Mutex { public: @@ -131,27 +132,6 @@ protected: friend class WorkerThread; }; -/// Worker thread that continuously dequeues Jobs from a MultiThreadedJobQueue -/// and process them. -/// -/// The public interface of this class must remain identical to its equivalent -/// in JobScheduler_win32.h -class WorkerThread { -public: - explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); - - ~WorkerThread(); - - void Run(); - - MultiThreadedJobQueue* GetJobQueue() { return mQueue; } -protected: - void SetName(const char* name); - - MultiThreadedJobQueue* mQueue; - pthread_t mThread; -}; - /// An object that a thread can synchronously wait on. /// Usually set by a SetEventJob. class EventObject : public external::AtomicRefCounted diff --git a/gfx/2d/JobScheduler_win32.cpp b/gfx/2d/JobScheduler_win32.cpp new file mode 100644 index 00000000000..d5aedf012af --- /dev/null +++ b/gfx/2d/JobScheduler_win32.cpp @@ -0,0 +1,143 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "JobScheduler.h" +#include "mozilla/gfx/Logging.h" + +using namespace std; + +namespace mozilla { +namespace gfx { + +DWORD __stdcall ThreadCallback(void* threadData); + +class WorkerThreadWin32 : public WorkerThread { +public: + explicit WorkerThreadWin32(MultiThreadedJobQueue* aJobQueue) + : WorkerThread(aJobQueue) + { + mThread = ::CreateThread(nullptr, 0, ThreadCallback, static_cast(this), 0, nullptr); + } + + ~WorkerThreadWin32() + { + ::WaitForSingleObject(mThread, INFINITE); + ::CloseHandle(mThread); + } + +protected: + HANDLE mThread; +}; + +DWORD __stdcall ThreadCallback(void* threadData) +{ + WorkerThread* thread = static_cast(threadData); + thread->Run(); + return 0; +} + +WorkerThread* +WorkerThread::Create(MultiThreadedJobQueue* aJobQueue) +{ + return new WorkerThreadWin32(aJobQueue); +} + +bool +MultiThreadedJobQueue::PopJob(Job*& aOutJob, AccessType aAccess) +{ + for (;;) { + while (aAccess == BLOCKING && mJobs.empty()) { + { + MutexAutoLock lock(&mMutex); + if (mShuttingDown) { + return false; + } + } + + HANDLE handles[] = { mAvailableEvent, mShutdownEvent }; + ::WaitForMultipleObjects(2, handles, FALSE, INFINITE); + } + + MutexAutoLock lock(&mMutex); + + if (mShuttingDown) { + return false; + } + + if (mJobs.empty()) { + if (aAccess == NON_BLOCKING) { + return false; + } + continue; + } + + Job* task = mJobs.front(); + MOZ_ASSERT(task); + + mJobs.pop_front(); + + if (mJobs.empty()) { + ::ResetEvent(mAvailableEvent); + } + + aOutJob = task; + return true; + } +} + +void +MultiThreadedJobQueue::SubmitJob(Job* aJob) +{ + MOZ_ASSERT(aJob); + MutexAutoLock lock(&mMutex); + mJobs.push_back(aJob); + ::SetEvent(mAvailableEvent); +} + +void +MultiThreadedJobQueue::ShutDown() +{ + { + MutexAutoLock lock(&mMutex); + mShuttingDown = true; + } + while (mThreadsCount) { + ::SetEvent(mAvailableEvent); + ::WaitForSingleObject(mShutdownEvent, INFINITE); + } +} + +size_t +MultiThreadedJobQueue::NumJobs() +{ + MutexAutoLock lock(&mMutex); + return mJobs.size(); +} + +bool +MultiThreadedJobQueue::IsEmpty() +{ + MutexAutoLock lock(&mMutex); + return mJobs.empty(); +} + +void +MultiThreadedJobQueue::RegisterThread() +{ + mThreadsCount += 1; +} + +void +MultiThreadedJobQueue::UnregisterThread() +{ + MutexAutoLock lock(&mMutex); + mThreadsCount -= 1; + if (mThreadsCount == 0) { + ::SetEvent(mShutdownEvent); + } +} + +} // namespace +} // namespace diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h index ca2db97dad0..2ff0a476115 100644 --- a/gfx/2d/JobScheduler_win32.h +++ b/gfx/2d/JobScheduler_win32.h @@ -10,6 +10,8 @@ #define NOT_IMPLEMENTED MOZ_CRASH("Not implemented") #include "mozilla/RefPtr.h" +#include +#include namespace mozilla { namespace gfx { @@ -19,10 +21,37 @@ class Job; class Mutex { public: - Mutex() { NOT_IMPLEMENTED; } - ~Mutex() { NOT_IMPLEMENTED; } - void Lock() { NOT_IMPLEMENTED; } - void Unlock() { NOT_IMPLEMENTED; } + Mutex() { + ::InitializeCriticalSection(&mMutex); +#ifdef DEBUG + mOwner = 0; +#endif + } + + ~Mutex() { ::DeleteCriticalSection(&mMutex); } + + void Lock() { + ::EnterCriticalSection(&mMutex); +#ifdef DEBUG + MOZ_ASSERT(mOwner != GetCurrentThreadId(), "recursive locking"); + mOwner = GetCurrentThreadId(); +#endif + } + + void Unlock() { +#ifdef DEBUG + // GetCurrentThreadId cannot return 0: it is not a valid thread id + MOZ_ASSERT(mOwner == GetCurrentThreadId(), "mismatched lock/unlock"); + mOwner = 0; +#endif + ::LeaveCriticalSection(&mMutex); + } + +protected: + CRITICAL_SECTION mMutex; +#ifdef DEBUG + DWORD mOwner; +#endif }; // The public interface of this class must remain identical to its equivalent @@ -34,14 +63,43 @@ public: NON_BLOCKING }; - bool WaitForJob(Job*& aOutCommands) { NOT_IMPLEMENTED; } - bool PopJob(Job*& aOutCommands, AccessType aAccess) { NOT_IMPLEMENTED; } - void SubmitJob(Job* aCommands) { NOT_IMPLEMENTED; } - void ShutDown() { NOT_IMPLEMENTED; } - size_t NumJobs() { NOT_IMPLEMENTED; } - bool IsEmpty() { NOT_IMPLEMENTED; } - void RegisterThread() { NOT_IMPLEMENTED; } - void UnregisterThread() { NOT_IMPLEMENTED; } + MultiThreadedJobQueue() + : mThreadsCount(0) + , mShuttingDown(false) + { + mAvailableEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); + mShutdownEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); + } + + ~MultiThreadedJobQueue() + { + ::CloseHandle(mAvailableEvent); + ::CloseHandle(mShutdownEvent); + } + + bool WaitForJob(Job*& aOutJob) { return PopJob(aOutJob, BLOCKING); } + + bool PopJob(Job*& aOutJob, AccessType aAccess); + + void SubmitJob(Job* aJob); + + void ShutDown(); + + size_t NumJobs(); + + bool IsEmpty(); + + void RegisterThread(); + + void UnregisterThread(); + +protected: + std::list mJobs; + Mutex mMutex; + HANDLE mAvailableEvent; + HANDLE mShutdownEvent; + int32_t mThreadsCount; + bool mShuttingDown; friend class WorkerThread; }; @@ -54,19 +112,18 @@ class EventObject : public external::AtomicRefCounted public: MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) - EventObject() { NOT_IMPLEMENTED; } - ~EventObject() { NOT_IMPLEMENTED; } - void Wait() { NOT_IMPLEMENTED; } - bool Peak() { NOT_IMPLEMENTED; } - void Set() { NOT_IMPLEMENTED; } -}; + EventObject() { mEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); } -// The public interface of this class must remain identical to its equivalent -// in JobScheduler_posix.h -class WorkerThread { -public: - explicit WorkerThread(MultiThreadedJobQueue* aJobQueue) { NOT_IMPLEMENTED; } - void Run(); + ~EventObject() { ::CloseHandle(mEvent); } + + void Wait() { ::WaitForSingleObject(mEvent, INFINITE); } + + bool Peak() { return ::WaitForSingleObject(mEvent, 0) == WAIT_OBJECT_0; } + + void Set() { ::SetEvent(mEvent); } +protected: + // TODO: it's expensive to create events so we should try to reuse them + HANDLE mEvent; }; } // namespace diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index 5480302b99d..b49c90bfdec 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -71,6 +71,7 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': 'SourceSurfaceD2D.cpp', 'SourceSurfaceD2D1.cpp', 'SourceSurfaceD2DTarget.cpp', + 'JobScheduler_win32.cpp', ] DEFINES['WIN32'] = True diff --git a/gfx/tests/gtest/TestJobScheduler.cpp b/gfx/tests/gtest/TestJobScheduler.cpp index d1eed148c35..b6a155fb35d 100644 --- a/gfx/tests/gtest/TestJobScheduler.cpp +++ b/gfx/tests/gtest/TestJobScheduler.cpp @@ -3,15 +3,16 @@ * http://creativecommons.org/publicdomain/zero/1.0/ */ -#ifndef WIN32 - #include "gtest/gtest.h" #include "gmock/gmock.h" #include "mozilla/gfx/JobScheduler.h" +#ifndef WIN32 #include #include +#endif + #include #include @@ -24,9 +25,11 @@ using namespace mozilla; // things more apparent (if any). void MaybeYieldThread() { +#ifndef WIN32 if (rand() % 5 == 0) { sched_yield(); } +#endif } /// Used by the TestCommand to check that tasks are processed in the right order. @@ -242,5 +245,3 @@ TEST(Moz2D, JobScheduler_Chain) { } } } - -#endif From 74021d279aa6dda61e4f896d4701f105ac8e3bea Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:37 +0200 Subject: [PATCH 72/78] Bug 1083101 - rename gfx::Mutex into gfx::CriticalSection and move it to its own file. r=jrmuizel --- gfx/2d/CriticalSection.h | 80 ++++++++++++++++++++++++++++ gfx/2d/JobScheduler.cpp | 4 +- gfx/2d/JobScheduler.h | 11 +--- gfx/2d/JobScheduler_posix.h | 29 ++-------- gfx/2d/JobScheduler_win32.cpp | 14 ++--- gfx/2d/JobScheduler_win32.h | 38 +------------ gfx/2d/moz.build | 1 + gfx/tests/gtest/TestJobScheduler.cpp | 4 +- 8 files changed, 98 insertions(+), 83 deletions(-) create mode 100644 gfx/2d/CriticalSection.h diff --git a/gfx/2d/CriticalSection.h b/gfx/2d/CriticalSection.h new file mode 100644 index 00000000000..d1eb69abcf6 --- /dev/null +++ b/gfx/2d/CriticalSection.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_GFX_CRITICALSECTION_H_ +#define MOZILLA_GFX_CRITICALSECTION_H_ + +#ifdef WIN32 +#include +#else +#include +#include "mozilla/DebugOnly.h" +#endif + +namespace mozilla { +namespace gfx { + +#ifdef WIN32 + +class CriticalSection { +public: + CriticalSection() { ::InitializeCriticalSection(&mCriticalSection); } + + ~CriticalSection() { ::DeleteCriticalSection(&mCriticalSection); } + + void Enter() { ::EnterCriticalSection(&mCriticalSection); } + + void Leave() { ::LeaveCriticalSection(&mCriticalSection); } + +protected: + CRITICAL_SECTION mCriticalSection; +}; + +#else +// posix + +class PosixCondvar; +class CriticalSection { +public: + CriticalSection() { + DebugOnly err = pthread_mutex_init(&mMutex, nullptr); + MOZ_ASSERT(!err); + } + + ~CriticalSection() { + DebugOnly err = pthread_mutex_destroy(&mMutex); + MOZ_ASSERT(!err); + } + + void Enter() { + DebugOnly err = pthread_mutex_lock(&mMutex); + MOZ_ASSERT(!err); + } + + void Leave() { + DebugOnly err = pthread_mutex_unlock(&mMutex); + MOZ_ASSERT(!err); + } + +protected: + pthread_mutex_t mMutex; + friend class PosixCondVar; +}; + +#endif + +/// RAII helper. +struct CriticalSectionAutoEnter { + explicit CriticalSectionAutoEnter(CriticalSection* aSection) : mSection(aSection) { mSection->Enter(); } + ~CriticalSectionAutoEnter() { mSection->Leave(); } +protected: + CriticalSection* mSection; +}; + + +} // namespace +} // namespace + +#endif diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp index e486f3b4c74..3845c46db44 100644 --- a/gfx/2d/JobScheduler.cpp +++ b/gfx/2d/JobScheduler.cpp @@ -184,7 +184,7 @@ SyncObject::Signal() void SyncObject::AddWaitingJob(Job* aJob) { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mWaitingJobsSection); mWaitingJobs.push_back(aJob); } @@ -199,7 +199,7 @@ void SyncObject::SubmitWaitingJobs() // hold a strong ref to prevent that! RefPtr kungFuDeathGrip(this); - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mWaitingJobsSection); tasksToSubmit = Move(mWaitingJobs); mWaitingJobs.clear(); } diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h index 50da4874d5e..afe717596fe 100644 --- a/gfx/2d/JobScheduler.h +++ b/gfx/2d/JobScheduler.h @@ -206,7 +206,7 @@ private: void SubmitWaitingJobs(); std::vector mWaitingJobs; - Mutex mMutex; // for concurrent access to mWaintingJobs + CriticalSection mWaitingJobsSection; // for concurrent access to mWaintingJobs Atomic mSignals; #ifdef DEBUG @@ -218,15 +218,6 @@ private: friend class JobScheduler; }; - -/// RAII helper. -struct MutexAutoLock { - MutexAutoLock(Mutex* aMutex) : mMutex(aMutex) { mMutex->Lock(); } - ~MutexAutoLock() { mMutex->Unlock(); } -protected: - Mutex* mMutex; -}; - /// Base class for worker threads. class WorkerThread { diff --git a/gfx/2d/JobScheduler_posix.h b/gfx/2d/JobScheduler_posix.h index 74cd975df22..e106f62ea28 100644 --- a/gfx/2d/JobScheduler_posix.h +++ b/gfx/2d/JobScheduler_posix.h @@ -16,6 +16,7 @@ #include "mozilla/RefPtr.h" #include "mozilla/DebugOnly.h" +#include "mozilla/gfx/CriticalSection.h" namespace mozilla { namespace gfx { @@ -24,32 +25,8 @@ class Job; class PosixCondVar; class WorkerThread; -class Mutex { -public: - Mutex() { - DebugOnly err = pthread_mutex_init(&mMutex, nullptr); - MOZ_ASSERT(!err); - } - - ~Mutex() { - DebugOnly err = pthread_mutex_destroy(&mMutex); - MOZ_ASSERT(!err); - } - - void Lock() { - DebugOnly err = pthread_mutex_lock(&mMutex); - MOZ_ASSERT(!err); - } - - void Unlock() { - DebugOnly err = pthread_mutex_unlock(&mMutex); - MOZ_ASSERT(!err); - } - -protected: - pthread_mutex_t mMutex; - friend class PosixCondVar; -}; +typedef mozilla::gfx::CriticalSection Mutex; +typedef mozilla::gfx::CriticalSectionAutoEnter MutexAutoLock; // posix platforms only! class PosixCondVar { diff --git a/gfx/2d/JobScheduler_win32.cpp b/gfx/2d/JobScheduler_win32.cpp index d5aedf012af..9c8e925fd35 100644 --- a/gfx/2d/JobScheduler_win32.cpp +++ b/gfx/2d/JobScheduler_win32.cpp @@ -50,7 +50,7 @@ MultiThreadedJobQueue::PopJob(Job*& aOutJob, AccessType aAccess) for (;;) { while (aAccess == BLOCKING && mJobs.empty()) { { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); if (mShuttingDown) { return false; } @@ -60,7 +60,7 @@ MultiThreadedJobQueue::PopJob(Job*& aOutJob, AccessType aAccess) ::WaitForMultipleObjects(2, handles, FALSE, INFINITE); } - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); if (mShuttingDown) { return false; @@ -91,7 +91,7 @@ void MultiThreadedJobQueue::SubmitJob(Job* aJob) { MOZ_ASSERT(aJob); - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); mJobs.push_back(aJob); ::SetEvent(mAvailableEvent); } @@ -100,7 +100,7 @@ void MultiThreadedJobQueue::ShutDown() { { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); mShuttingDown = true; } while (mThreadsCount) { @@ -112,14 +112,14 @@ MultiThreadedJobQueue::ShutDown() size_t MultiThreadedJobQueue::NumJobs() { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); return mJobs.size(); } bool MultiThreadedJobQueue::IsEmpty() { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); return mJobs.empty(); } @@ -132,7 +132,7 @@ MultiThreadedJobQueue::RegisterThread() void MultiThreadedJobQueue::UnregisterThread() { - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); mThreadsCount -= 1; if (mThreadsCount == 0) { ::SetEvent(mShutdownEvent); diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h index 2ff0a476115..4a9d7e68276 100644 --- a/gfx/2d/JobScheduler_win32.h +++ b/gfx/2d/JobScheduler_win32.h @@ -10,6 +10,7 @@ #define NOT_IMPLEMENTED MOZ_CRASH("Not implemented") #include "mozilla/RefPtr.h" +#include "mozilla/gfx/CriticalSection.h" #include #include @@ -19,41 +20,6 @@ namespace gfx { class WorkerThread; class Job; -class Mutex { -public: - Mutex() { - ::InitializeCriticalSection(&mMutex); -#ifdef DEBUG - mOwner = 0; -#endif - } - - ~Mutex() { ::DeleteCriticalSection(&mMutex); } - - void Lock() { - ::EnterCriticalSection(&mMutex); -#ifdef DEBUG - MOZ_ASSERT(mOwner != GetCurrentThreadId(), "recursive locking"); - mOwner = GetCurrentThreadId(); -#endif - } - - void Unlock() { -#ifdef DEBUG - // GetCurrentThreadId cannot return 0: it is not a valid thread id - MOZ_ASSERT(mOwner == GetCurrentThreadId(), "mismatched lock/unlock"); - mOwner = 0; -#endif - ::LeaveCriticalSection(&mMutex); - } - -protected: - CRITICAL_SECTION mMutex; -#ifdef DEBUG - DWORD mOwner; -#endif -}; - // The public interface of this class must remain identical to its equivalent // in JobScheduler_posix.h class MultiThreadedJobQueue { @@ -95,7 +61,7 @@ public: protected: std::list mJobs; - Mutex mMutex; + CriticalSection mSection; HANDLE mAvailableEvent; HANDLE mShutdownEvent; int32_t mThreadsCount; diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index b49c90bfdec..d0780231fcc 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -20,6 +20,7 @@ EXPORTS.mozilla.gfx += [ 'Blur.h', 'BorrowedContext.h', 'Coord.h', + 'CriticalSection.h', 'DataSurfaceHelpers.h', 'DrawTargetTiled.h', 'Filters.h', diff --git a/gfx/tests/gtest/TestJobScheduler.cpp b/gfx/tests/gtest/TestJobScheduler.cpp index b6a155fb35d..8c18ffcaa08 100644 --- a/gfx/tests/gtest/TestJobScheduler.cpp +++ b/gfx/tests/gtest/TestJobScheduler.cpp @@ -35,7 +35,7 @@ void MaybeYieldThread() /// Used by the TestCommand to check that tasks are processed in the right order. struct SanityChecker { std::vector mAdvancements; - mozilla::gfx::Mutex mMutex; + mozilla::gfx::CriticalSection mSection; explicit SanityChecker(uint64_t aNumCmdBuffers) { @@ -47,7 +47,7 @@ struct SanityChecker { virtual void Check(uint64_t aJobId, uint64_t aCmdId) { MaybeYieldThread(); - MutexAutoLock lock(&mMutex); + CriticalSectionAutoEnter lock(&mSection); ASSERT_EQ(mAdvancements[aJobId], aCmdId-1); mAdvancements[aJobId] = aCmdId; } From d5d8b4e3fd6bd24ec6f2a7a907ef6c5f8ffc45fe Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 17:35:47 +0200 Subject: [PATCH 73/78] Bug 1083101 - Make SyncObject's list of waiting jobs lock-free. r=jrmuizel --- gfx/2d/JobScheduler.cpp | 49 +++++++++++++++++++++++++++-------------- gfx/2d/JobScheduler.h | 10 +++++++-- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp index 3845c46db44..2294380d0b2 100644 --- a/gfx/2d/JobScheduler.cpp +++ b/gfx/2d/JobScheduler.cpp @@ -84,7 +84,8 @@ JobScheduler::GetQueueForJob(Job* aJob) } Job::Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread) -: mStartSync(aStart) +: mNextWaitingJob(nullptr) +, mStartSync(aStart) , mCompletionSync(aCompletion) , mPinToThread(aThread) { @@ -124,6 +125,7 @@ SetEventJob::~SetEventJob() SyncObject::SyncObject(uint32_t aNumPrerequisites) : mSignals(aNumPrerequisites) +, mFirstWaitingJob(nullptr) #ifdef DEBUG , mNumPrerequisites(aNumPrerequisites) , mAddedPrerequisites(0) @@ -132,7 +134,7 @@ SyncObject::SyncObject(uint32_t aNumPrerequisites) SyncObject::~SyncObject() { - MOZ_ASSERT(mWaitingJobs.size() == 0); + MOZ_ASSERT(mFirstWaitingJob == nullptr); } bool @@ -184,28 +186,41 @@ SyncObject::Signal() void SyncObject::AddWaitingJob(Job* aJob) { - CriticalSectionAutoEnter lock(&mWaitingJobsSection); - mWaitingJobs.push_back(aJob); + // Push (using atomics) the task into the list of waiting tasks. + for (;;) { + Job* first = mFirstWaitingJob; + aJob->mNextWaitingJob = first; + if (mFirstWaitingJob.compareExchange(first, aJob)) { + break; + } + } } void SyncObject::SubmitWaitingJobs() { - std::vector tasksToSubmit; - { - // Scheduling the tasks can cause code that modifies 's reference - // count to run concurrently, and cause the caller of this function to - // be owned by another thread. We need to make sure the reference count - // does not reach 0 on another thread before mWaitingJobs.clear(), so - // hold a strong ref to prevent that! - RefPtr kungFuDeathGrip(this); + // Scheduling the tasks can cause code that modifies 's reference + // count to run concurrently, and cause the caller of this function to + // be owned by another thread. We need to make sure the reference count + // does not reach 0 on another thread before the end of this method, so + // hold a strong ref to prevent that! + RefPtr kungFuDeathGrip(this); - CriticalSectionAutoEnter lock(&mWaitingJobsSection); - tasksToSubmit = Move(mWaitingJobs); - mWaitingJobs.clear(); + // First atomically swap mFirstWaitingJob and waitingJobs... + Job* waitingJobs = nullptr; + for (;;) { + waitingJobs = mFirstWaitingJob; + if (mFirstWaitingJob.compareExchange(waitingJobs, nullptr)) { + break; + } } - for (Job* task : tasksToSubmit) { - JobScheduler::GetQueueForJob(task)->SubmitJob(task); + // ... and submit all of the waiting tasks in waitingJob now that they belong + // to this thread. + while (waitingJobs) { + Job* next = waitingJobs->mNextWaitingJob; + waitingJobs->mNextWaitingJob = nullptr; + JobScheduler::GetQueueForJob(waitingJobs)->SubmitJob(waitingJobs); + waitingJobs = next; } } diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h index afe717596fe..9c72bc8faf5 100644 --- a/gfx/2d/JobScheduler.h +++ b/gfx/2d/JobScheduler.h @@ -109,9 +109,16 @@ public: WorkerThread* GetWorkerThread() { return mPinToThread; } protected: + // An intrusive linked list of tasks waiting for a sync object to enter the + // signaled state. When the task is not waiting for a sync object, mNextWaitingJob + // should be null. This is only accessed from the thread that owns the task. + Job* mNextWaitingJob; + RefPtr mStartSync; RefPtr mCompletionSync; WorkerThread* mPinToThread; + + friend class SyncObject; }; class EventObject; @@ -205,9 +212,8 @@ private: void SubmitWaitingJobs(); - std::vector mWaitingJobs; - CriticalSection mWaitingJobsSection; // for concurrent access to mWaintingJobs Atomic mSignals; + Atomic mFirstWaitingJob; #ifdef DEBUG uint32_t mNumPrerequisites; From 2468f65e6fe82559bb080ec8bacd37e34e0ba024 Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 18:08:10 +0200 Subject: [PATCH 74/78] Bug 1083101 - build fix on CLOSED TREE --- gfx/2d/moz.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index d0780231fcc..4d64a716a1f 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -65,6 +65,7 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': 'DrawTargetD2D1.cpp', 'ExtendInputEffectD2D1.cpp', 'FilterNodeD2D1.cpp', + 'JobScheduler_win32.cpp', 'PathD2D.cpp', 'RadialGradientEffectD2D1.cpp', 'ScaledFontDWrite.cpp', @@ -72,7 +73,6 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': 'SourceSurfaceD2D.cpp', 'SourceSurfaceD2D1.cpp', 'SourceSurfaceD2DTarget.cpp', - 'JobScheduler_win32.cpp', ] DEFINES['WIN32'] = True From 53c17214ad99b2f2a75ecf2afbe830f83e8175a4 Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 19:11:26 +0200 Subject: [PATCH 75/78] Bug 1083101 - build fix on CLOSED TREE --- gfx/2d/JobScheduler_win32.h | 1 + 1 file changed, 1 insertion(+) diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h index 4a9d7e68276..58c3a96e601 100644 --- a/gfx/2d/JobScheduler_win32.h +++ b/gfx/2d/JobScheduler_win32.h @@ -12,6 +12,7 @@ #include "mozilla/RefPtr.h" #include "mozilla/gfx/CriticalSection.h" #include +#include #include namespace mozilla { From e4b953afb894e276b41e333761371d9d0fdc2786 Mon Sep 17 00:00:00 2001 From: Nicolas Silva Date: Thu, 24 Sep 2015 19:47:20 +0200 Subject: [PATCH 76/78] Bug 1083101 - Back 5 patches out due to windows build issues, CLOSED TREE. --- gfx/2d/CriticalSection.h | 80 --------------- gfx/2d/JobScheduler.cpp | 79 ++++----------- gfx/2d/JobScheduler.h | 35 ++----- gfx/2d/JobScheduler_posix.cpp | 102 ++++++++++--------- gfx/2d/JobScheduler_posix.h | 51 +++++++++- gfx/2d/JobScheduler_win32.cpp | 143 --------------------------- gfx/2d/JobScheduler_win32.h | 80 ++++++--------- gfx/2d/moz.build | 2 - gfx/tests/gtest/TestJobScheduler.cpp | 13 ++- 9 files changed, 163 insertions(+), 422 deletions(-) delete mode 100644 gfx/2d/CriticalSection.h delete mode 100644 gfx/2d/JobScheduler_win32.cpp diff --git a/gfx/2d/CriticalSection.h b/gfx/2d/CriticalSection.h deleted file mode 100644 index d1eb69abcf6..00000000000 --- a/gfx/2d/CriticalSection.h +++ /dev/null @@ -1,80 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MOZILLA_GFX_CRITICALSECTION_H_ -#define MOZILLA_GFX_CRITICALSECTION_H_ - -#ifdef WIN32 -#include -#else -#include -#include "mozilla/DebugOnly.h" -#endif - -namespace mozilla { -namespace gfx { - -#ifdef WIN32 - -class CriticalSection { -public: - CriticalSection() { ::InitializeCriticalSection(&mCriticalSection); } - - ~CriticalSection() { ::DeleteCriticalSection(&mCriticalSection); } - - void Enter() { ::EnterCriticalSection(&mCriticalSection); } - - void Leave() { ::LeaveCriticalSection(&mCriticalSection); } - -protected: - CRITICAL_SECTION mCriticalSection; -}; - -#else -// posix - -class PosixCondvar; -class CriticalSection { -public: - CriticalSection() { - DebugOnly err = pthread_mutex_init(&mMutex, nullptr); - MOZ_ASSERT(!err); - } - - ~CriticalSection() { - DebugOnly err = pthread_mutex_destroy(&mMutex); - MOZ_ASSERT(!err); - } - - void Enter() { - DebugOnly err = pthread_mutex_lock(&mMutex); - MOZ_ASSERT(!err); - } - - void Leave() { - DebugOnly err = pthread_mutex_unlock(&mMutex); - MOZ_ASSERT(!err); - } - -protected: - pthread_mutex_t mMutex; - friend class PosixCondVar; -}; - -#endif - -/// RAII helper. -struct CriticalSectionAutoEnter { - explicit CriticalSectionAutoEnter(CriticalSection* aSection) : mSection(aSection) { mSection->Enter(); } - ~CriticalSectionAutoEnter() { mSection->Leave(); } -protected: - CriticalSection* mSection; -}; - - -} // namespace -} // namespace - -#endif diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp index 2294380d0b2..8d7ab727e90 100644 --- a/gfx/2d/JobScheduler.cpp +++ b/gfx/2d/JobScheduler.cpp @@ -23,7 +23,7 @@ bool JobScheduler::Init(uint32_t aNumThreads, uint32_t aNumQueues) } for (uint32_t i = 0; i < aNumThreads; ++i) { - sSingleton->mWorkerThreads.push_back(WorkerThread::Create(sSingleton->mDrawingQueues[i%aNumQueues])); + sSingleton->mWorkerThreads.push_back(new WorkerThread(sSingleton->mDrawingQueues[i%aNumQueues])); } return true; } @@ -84,8 +84,7 @@ JobScheduler::GetQueueForJob(Job* aJob) } Job::Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread) -: mNextWaitingJob(nullptr) -, mStartSync(aStart) +: mStartSync(aStart) , mCompletionSync(aCompletion) , mPinToThread(aThread) { @@ -125,7 +124,6 @@ SetEventJob::~SetEventJob() SyncObject::SyncObject(uint32_t aNumPrerequisites) : mSignals(aNumPrerequisites) -, mFirstWaitingJob(nullptr) #ifdef DEBUG , mNumPrerequisites(aNumPrerequisites) , mAddedPrerequisites(0) @@ -134,7 +132,7 @@ SyncObject::SyncObject(uint32_t aNumPrerequisites) SyncObject::~SyncObject() { - MOZ_ASSERT(mFirstWaitingJob == nullptr); + MOZ_ASSERT(mWaitingJobs.size() == 0); } bool @@ -186,41 +184,28 @@ SyncObject::Signal() void SyncObject::AddWaitingJob(Job* aJob) { - // Push (using atomics) the task into the list of waiting tasks. - for (;;) { - Job* first = mFirstWaitingJob; - aJob->mNextWaitingJob = first; - if (mFirstWaitingJob.compareExchange(first, aJob)) { - break; - } - } + MutexAutoLock lock(&mMutex); + mWaitingJobs.push_back(aJob); } void SyncObject::SubmitWaitingJobs() { - // Scheduling the tasks can cause code that modifies 's reference - // count to run concurrently, and cause the caller of this function to - // be owned by another thread. We need to make sure the reference count - // does not reach 0 on another thread before the end of this method, so - // hold a strong ref to prevent that! - RefPtr kungFuDeathGrip(this); + std::vector tasksToSubmit; + { + // Scheduling the tasks can cause code that modifies 's reference + // count to run concurrently, and cause the caller of this function to + // be owned by another thread. We need to make sure the reference count + // does not reach 0 on another thread before mWaitingJobs.clear(), so + // hold a strong ref to prevent that! + RefPtr kungFuDeathGrip(this); - // First atomically swap mFirstWaitingJob and waitingJobs... - Job* waitingJobs = nullptr; - for (;;) { - waitingJobs = mFirstWaitingJob; - if (mFirstWaitingJob.compareExchange(waitingJobs, nullptr)) { - break; - } + MutexAutoLock lock(&mMutex); + tasksToSubmit = Move(mWaitingJobs); + mWaitingJobs.clear(); } - // ... and submit all of the waiting tasks in waitingJob now that they belong - // to this thread. - while (waitingJobs) { - Job* next = waitingJobs->mNextWaitingJob; - waitingJobs->mNextWaitingJob = nullptr; - JobScheduler::GetQueueForJob(waitingJobs)->SubmitJob(waitingJobs); - waitingJobs = next; + for (Job* task : tasksToSubmit) { + JobScheduler::GetQueueForJob(task)->SubmitJob(task); } } @@ -247,33 +232,5 @@ SyncObject::AddSubsequent(Job* aJob) { } -WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) -: mQueue(aJobQueue) -{ - aJobQueue->RegisterThread(); -} - -void -WorkerThread::Run() -{ - SetName("gfx worker"); - - for (;;) { - Job* commands = nullptr; - if (!mQueue->WaitForJob(commands)) { - mQueue->UnregisterThread(); - return; - } - - JobStatus status = JobScheduler::ProcessJob(commands); - - if (status == JobStatus::Error) { - // Don't try to handle errors for now, but that's open to discussions. - // I expect errors to be mostly OOM issues. - MOZ_CRASH(); - } - } -} - } //namespace } //namespace diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h index 9c72bc8faf5..e58cee721a1 100644 --- a/gfx/2d/JobScheduler.h +++ b/gfx/2d/JobScheduler.h @@ -15,14 +15,11 @@ #include "mozilla/gfx/JobScheduler_posix.h" #endif -#include - namespace mozilla { namespace gfx { class MultiThreadedJobQueue; class SyncObject; -class WorkerThread; class JobScheduler { public: @@ -109,16 +106,9 @@ public: WorkerThread* GetWorkerThread() { return mPinToThread; } protected: - // An intrusive linked list of tasks waiting for a sync object to enter the - // signaled state. When the task is not waiting for a sync object, mNextWaitingJob - // should be null. This is only accessed from the thread that owns the task. - Job* mNextWaitingJob; - RefPtr mStartSync; RefPtr mCompletionSync; WorkerThread* mPinToThread; - - friend class SyncObject; }; class EventObject; @@ -212,8 +202,9 @@ private: void SubmitWaitingJobs(); + std::vector mWaitingJobs; + Mutex mMutex; // for concurrent access to mWaintingJobs Atomic mSignals; - Atomic mFirstWaitingJob; #ifdef DEBUG uint32_t mNumPrerequisites; @@ -224,26 +215,16 @@ private: friend class JobScheduler; }; -/// Base class for worker threads. -class WorkerThread -{ -public: - static WorkerThread* Create(MultiThreadedJobQueue* aJobQueue); - - virtual ~WorkerThread() {} - - void Run(); - - MultiThreadedJobQueue* GetJobQueue() { return mQueue; } +/// RAII helper. +struct MutexAutoLock { + MutexAutoLock(Mutex* aMutex) : mMutex(aMutex) { mMutex->Lock(); } + ~MutexAutoLock() { mMutex->Unlock(); } protected: - explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); - - virtual void SetName(const char* aName) {} - - MultiThreadedJobQueue* mQueue; + Mutex* mMutex; }; + } // namespace } // namespace diff --git a/gfx/2d/JobScheduler_posix.cpp b/gfx/2d/JobScheduler_posix.cpp index c19103e28d8..0b7df32c7f6 100644 --- a/gfx/2d/JobScheduler_posix.cpp +++ b/gfx/2d/JobScheduler_posix.cpp @@ -11,52 +11,6 @@ using namespace std; namespace mozilla { namespace gfx { -void* ThreadCallback(void* threadData); - -class WorkerThreadPosix : public WorkerThread { -public: - explicit WorkerThreadPosix(MultiThreadedJobQueue* aJobQueue) - : WorkerThread(aJobQueue) - { - pthread_create(&mThread, nullptr, ThreadCallback, static_cast(this)); - } - - ~WorkerThreadPosix() - { - pthread_join(mThread, nullptr); - } - - virtual void SetName(const char* aName) override - { - // Call this from the thread itself because of Mac. -#ifdef XP_MACOSX - pthread_setname_np(aName); -#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) - pthread_set_name_np(mThread, aName); -#elif defined(__NetBSD__) - pthread_setname_np(mThread, "%s", (void*)aName); -#else - pthread_setname_np(mThread, aName); -#endif - } - -protected: - pthread_t mThread; -}; - -void* ThreadCallback(void* threadData) -{ - WorkerThread* thread = static_cast(threadData); - thread->Run(); - return nullptr; -} - -WorkerThread* -WorkerThread::Create(MultiThreadedJobQueue* aJobQueue) -{ - return new WorkerThreadPosix(aJobQueue); -} - MultiThreadedJobQueue::MultiThreadedJobQueue() : mThreadsCount(0) , mShuttingDown(false) @@ -154,6 +108,62 @@ MultiThreadedJobQueue::UnregisterThread() } } +void* ThreadCallback(void* threadData) +{ + WorkerThread* thread = (WorkerThread*)threadData; + thread->Run(); + return nullptr; +} + +WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) +: mQueue(aJobQueue) +{ + aJobQueue->RegisterThread(); + pthread_create(&mThread, nullptr, ThreadCallback, this); +} + +WorkerThread::~WorkerThread() +{ + pthread_join(mThread, nullptr); +} + +void +WorkerThread::SetName(const char* aName) +{ + // Call this from the thread itself because of Mac. +#ifdef XP_MACOSX + pthread_setname_np(aName); +#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) + pthread_set_name_np(mThread, aName); +#elif defined(__NetBSD__) + pthread_setname_np(mThread, "%s", (void*)aName); +#else + pthread_setname_np(mThread, aName); +#endif +} + +void +WorkerThread::Run() +{ + SetName("gfx worker"); + + for (;;) { + Job* commands = nullptr; + if (!mQueue->WaitForJob(commands)) { + mQueue->UnregisterThread(); + return; + } + + JobStatus status = JobScheduler::ProcessJob(commands); + + if (status == JobStatus::Error) { + // Don't try to handle errors for now, but that's open to discussions. + // I expect errors to be mostly OOM issues. + MOZ_CRASH(); + } + } +} + EventObject::EventObject() : mIsSet(false) {} diff --git a/gfx/2d/JobScheduler_posix.h b/gfx/2d/JobScheduler_posix.h index e106f62ea28..f4071c34e47 100644 --- a/gfx/2d/JobScheduler_posix.h +++ b/gfx/2d/JobScheduler_posix.h @@ -16,17 +16,39 @@ #include "mozilla/RefPtr.h" #include "mozilla/DebugOnly.h" -#include "mozilla/gfx/CriticalSection.h" namespace mozilla { namespace gfx { class Job; class PosixCondVar; -class WorkerThread; -typedef mozilla::gfx::CriticalSection Mutex; -typedef mozilla::gfx::CriticalSectionAutoEnter MutexAutoLock; +class Mutex { +public: + Mutex() { + DebugOnly err = pthread_mutex_init(&mMutex, nullptr); + MOZ_ASSERT(!err); + } + + ~Mutex() { + DebugOnly err = pthread_mutex_destroy(&mMutex); + MOZ_ASSERT(!err); + } + + void Lock() { + DebugOnly err = pthread_mutex_lock(&mMutex); + MOZ_ASSERT(!err); + } + + void Unlock() { + DebugOnly err = pthread_mutex_unlock(&mMutex); + MOZ_ASSERT(!err); + } + +protected: + pthread_mutex_t mMutex; + friend class PosixCondVar; +}; // posix platforms only! class PosixCondVar { @@ -109,6 +131,27 @@ protected: friend class WorkerThread; }; +/// Worker thread that continuously dequeues Jobs from a MultiThreadedJobQueue +/// and process them. +/// +/// The public interface of this class must remain identical to its equivalent +/// in JobScheduler_win32.h +class WorkerThread { +public: + explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); + + ~WorkerThread(); + + void Run(); + + MultiThreadedJobQueue* GetJobQueue() { return mQueue; } +protected: + void SetName(const char* name); + + MultiThreadedJobQueue* mQueue; + pthread_t mThread; +}; + /// An object that a thread can synchronously wait on. /// Usually set by a SetEventJob. class EventObject : public external::AtomicRefCounted diff --git a/gfx/2d/JobScheduler_win32.cpp b/gfx/2d/JobScheduler_win32.cpp deleted file mode 100644 index 9c8e925fd35..00000000000 --- a/gfx/2d/JobScheduler_win32.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "JobScheduler.h" -#include "mozilla/gfx/Logging.h" - -using namespace std; - -namespace mozilla { -namespace gfx { - -DWORD __stdcall ThreadCallback(void* threadData); - -class WorkerThreadWin32 : public WorkerThread { -public: - explicit WorkerThreadWin32(MultiThreadedJobQueue* aJobQueue) - : WorkerThread(aJobQueue) - { - mThread = ::CreateThread(nullptr, 0, ThreadCallback, static_cast(this), 0, nullptr); - } - - ~WorkerThreadWin32() - { - ::WaitForSingleObject(mThread, INFINITE); - ::CloseHandle(mThread); - } - -protected: - HANDLE mThread; -}; - -DWORD __stdcall ThreadCallback(void* threadData) -{ - WorkerThread* thread = static_cast(threadData); - thread->Run(); - return 0; -} - -WorkerThread* -WorkerThread::Create(MultiThreadedJobQueue* aJobQueue) -{ - return new WorkerThreadWin32(aJobQueue); -} - -bool -MultiThreadedJobQueue::PopJob(Job*& aOutJob, AccessType aAccess) -{ - for (;;) { - while (aAccess == BLOCKING && mJobs.empty()) { - { - CriticalSectionAutoEnter lock(&mSection); - if (mShuttingDown) { - return false; - } - } - - HANDLE handles[] = { mAvailableEvent, mShutdownEvent }; - ::WaitForMultipleObjects(2, handles, FALSE, INFINITE); - } - - CriticalSectionAutoEnter lock(&mSection); - - if (mShuttingDown) { - return false; - } - - if (mJobs.empty()) { - if (aAccess == NON_BLOCKING) { - return false; - } - continue; - } - - Job* task = mJobs.front(); - MOZ_ASSERT(task); - - mJobs.pop_front(); - - if (mJobs.empty()) { - ::ResetEvent(mAvailableEvent); - } - - aOutJob = task; - return true; - } -} - -void -MultiThreadedJobQueue::SubmitJob(Job* aJob) -{ - MOZ_ASSERT(aJob); - CriticalSectionAutoEnter lock(&mSection); - mJobs.push_back(aJob); - ::SetEvent(mAvailableEvent); -} - -void -MultiThreadedJobQueue::ShutDown() -{ - { - CriticalSectionAutoEnter lock(&mSection); - mShuttingDown = true; - } - while (mThreadsCount) { - ::SetEvent(mAvailableEvent); - ::WaitForSingleObject(mShutdownEvent, INFINITE); - } -} - -size_t -MultiThreadedJobQueue::NumJobs() -{ - CriticalSectionAutoEnter lock(&mSection); - return mJobs.size(); -} - -bool -MultiThreadedJobQueue::IsEmpty() -{ - CriticalSectionAutoEnter lock(&mSection); - return mJobs.empty(); -} - -void -MultiThreadedJobQueue::RegisterThread() -{ - mThreadsCount += 1; -} - -void -MultiThreadedJobQueue::UnregisterThread() -{ - CriticalSectionAutoEnter lock(&mSection); - mThreadsCount -= 1; - if (mThreadsCount == 0) { - ::SetEvent(mShutdownEvent); - } -} - -} // namespace -} // namespace diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h index 58c3a96e601..ca2db97dad0 100644 --- a/gfx/2d/JobScheduler_win32.h +++ b/gfx/2d/JobScheduler_win32.h @@ -10,10 +10,6 @@ #define NOT_IMPLEMENTED MOZ_CRASH("Not implemented") #include "mozilla/RefPtr.h" -#include "mozilla/gfx/CriticalSection.h" -#include -#include -#include namespace mozilla { namespace gfx { @@ -21,6 +17,14 @@ namespace gfx { class WorkerThread; class Job; +class Mutex { +public: + Mutex() { NOT_IMPLEMENTED; } + ~Mutex() { NOT_IMPLEMENTED; } + void Lock() { NOT_IMPLEMENTED; } + void Unlock() { NOT_IMPLEMENTED; } +}; + // The public interface of this class must remain identical to its equivalent // in JobScheduler_posix.h class MultiThreadedJobQueue { @@ -30,43 +34,14 @@ public: NON_BLOCKING }; - MultiThreadedJobQueue() - : mThreadsCount(0) - , mShuttingDown(false) - { - mAvailableEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); - mShutdownEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); - } - - ~MultiThreadedJobQueue() - { - ::CloseHandle(mAvailableEvent); - ::CloseHandle(mShutdownEvent); - } - - bool WaitForJob(Job*& aOutJob) { return PopJob(aOutJob, BLOCKING); } - - bool PopJob(Job*& aOutJob, AccessType aAccess); - - void SubmitJob(Job* aJob); - - void ShutDown(); - - size_t NumJobs(); - - bool IsEmpty(); - - void RegisterThread(); - - void UnregisterThread(); - -protected: - std::list mJobs; - CriticalSection mSection; - HANDLE mAvailableEvent; - HANDLE mShutdownEvent; - int32_t mThreadsCount; - bool mShuttingDown; + bool WaitForJob(Job*& aOutCommands) { NOT_IMPLEMENTED; } + bool PopJob(Job*& aOutCommands, AccessType aAccess) { NOT_IMPLEMENTED; } + void SubmitJob(Job* aCommands) { NOT_IMPLEMENTED; } + void ShutDown() { NOT_IMPLEMENTED; } + size_t NumJobs() { NOT_IMPLEMENTED; } + bool IsEmpty() { NOT_IMPLEMENTED; } + void RegisterThread() { NOT_IMPLEMENTED; } + void UnregisterThread() { NOT_IMPLEMENTED; } friend class WorkerThread; }; @@ -79,18 +54,19 @@ class EventObject : public external::AtomicRefCounted public: MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) - EventObject() { mEvent = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); } + EventObject() { NOT_IMPLEMENTED; } + ~EventObject() { NOT_IMPLEMENTED; } + void Wait() { NOT_IMPLEMENTED; } + bool Peak() { NOT_IMPLEMENTED; } + void Set() { NOT_IMPLEMENTED; } +}; - ~EventObject() { ::CloseHandle(mEvent); } - - void Wait() { ::WaitForSingleObject(mEvent, INFINITE); } - - bool Peak() { return ::WaitForSingleObject(mEvent, 0) == WAIT_OBJECT_0; } - - void Set() { ::SetEvent(mEvent); } -protected: - // TODO: it's expensive to create events so we should try to reuse them - HANDLE mEvent; +// The public interface of this class must remain identical to its equivalent +// in JobScheduler_posix.h +class WorkerThread { +public: + explicit WorkerThread(MultiThreadedJobQueue* aJobQueue) { NOT_IMPLEMENTED; } + void Run(); }; } // namespace diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index 4d64a716a1f..5480302b99d 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -20,7 +20,6 @@ EXPORTS.mozilla.gfx += [ 'Blur.h', 'BorrowedContext.h', 'Coord.h', - 'CriticalSection.h', 'DataSurfaceHelpers.h', 'DrawTargetTiled.h', 'Filters.h', @@ -65,7 +64,6 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': 'DrawTargetD2D1.cpp', 'ExtendInputEffectD2D1.cpp', 'FilterNodeD2D1.cpp', - 'JobScheduler_win32.cpp', 'PathD2D.cpp', 'RadialGradientEffectD2D1.cpp', 'ScaledFontDWrite.cpp', diff --git a/gfx/tests/gtest/TestJobScheduler.cpp b/gfx/tests/gtest/TestJobScheduler.cpp index 8c18ffcaa08..d1eed148c35 100644 --- a/gfx/tests/gtest/TestJobScheduler.cpp +++ b/gfx/tests/gtest/TestJobScheduler.cpp @@ -3,16 +3,15 @@ * http://creativecommons.org/publicdomain/zero/1.0/ */ +#ifndef WIN32 + #include "gtest/gtest.h" #include "gmock/gmock.h" #include "mozilla/gfx/JobScheduler.h" -#ifndef WIN32 #include #include -#endif - #include #include @@ -25,17 +24,15 @@ using namespace mozilla; // things more apparent (if any). void MaybeYieldThread() { -#ifndef WIN32 if (rand() % 5 == 0) { sched_yield(); } -#endif } /// Used by the TestCommand to check that tasks are processed in the right order. struct SanityChecker { std::vector mAdvancements; - mozilla::gfx::CriticalSection mSection; + mozilla::gfx::Mutex mMutex; explicit SanityChecker(uint64_t aNumCmdBuffers) { @@ -47,7 +44,7 @@ struct SanityChecker { virtual void Check(uint64_t aJobId, uint64_t aCmdId) { MaybeYieldThread(); - CriticalSectionAutoEnter lock(&mSection); + MutexAutoLock lock(&mMutex); ASSERT_EQ(mAdvancements[aJobId], aCmdId-1); mAdvancements[aJobId] = aCmdId; } @@ -245,3 +242,5 @@ TEST(Moz2D, JobScheduler_Chain) { } } } + +#endif From 6ddd381a7d0cc6e9bde42c5e9cf5a2545b368787 Mon Sep 17 00:00:00 2001 From: Wes Kocher Date: Thu, 24 Sep 2015 11:14:50 -0700 Subject: [PATCH 77/78] Backed out bug 1083101 for build bustage CLOSED TREE changesets backed out: e39dfd9e05cb, cd54e93993b4, 61db1a51a7c4, c20fd7506bb7, b5e97b0facb7, cdf356cb817e, b058a918f443, d421e79326a2, 34a0be9af3f3, bba6f89ab775, 2fd5cfcea4e5 --- gfx/2d/DrawCommand.h | 136 +++------------ gfx/2d/DrawTargetCapture.cpp | 2 +- gfx/2d/DrawingJob.cpp | 120 ------------- gfx/2d/DrawingJob.h | 157 ----------------- gfx/2d/IterableArena.h | 193 --------------------- gfx/2d/JobScheduler.cpp | 236 ------------------------- gfx/2d/JobScheduler.h | 231 ------------------------- gfx/2d/JobScheduler_posix.cpp | 202 ---------------------- gfx/2d/JobScheduler_posix.h | 187 -------------------- gfx/2d/JobScheduler_win32.h | 76 --------- gfx/2d/Types.h | 7 - gfx/2d/moz.build | 11 -- gfx/tests/gtest/TestArena.cpp | 188 -------------------- gfx/tests/gtest/TestJobScheduler.cpp | 246 --------------------------- gfx/tests/gtest/moz.build | 2 - 15 files changed, 24 insertions(+), 1970 deletions(-) delete mode 100644 gfx/2d/DrawingJob.cpp delete mode 100644 gfx/2d/DrawingJob.h delete mode 100644 gfx/2d/IterableArena.h delete mode 100644 gfx/2d/JobScheduler.cpp delete mode 100644 gfx/2d/JobScheduler.h delete mode 100644 gfx/2d/JobScheduler_posix.cpp delete mode 100644 gfx/2d/JobScheduler_posix.h delete mode 100644 gfx/2d/JobScheduler_win32.h delete mode 100644 gfx/tests/gtest/TestArena.cpp delete mode 100644 gfx/tests/gtest/TestJobScheduler.cpp diff --git a/gfx/2d/DrawCommand.h b/gfx/2d/DrawCommand.h index 135548b9c6d..228d627c088 100644 --- a/gfx/2d/DrawCommand.h +++ b/gfx/2d/DrawCommand.h @@ -6,10 +6,6 @@ #ifndef MOZILLA_GFX_DRAWCOMMAND_H_ #define MOZILLA_GFX_DRAWCOMMAND_H_ - -#define _USE_MATH_DEFINES -#include - #include "2D.h" #include "Filters.h" #include @@ -35,8 +31,7 @@ enum class CommandType : int8_t { PUSHCLIP, PUSHCLIPRECT, POPCLIP, - SETTRANSFORM, - FLUSH + SETTRANSFORM }; class DrawingCommand @@ -44,9 +39,7 @@ class DrawingCommand public: virtual ~DrawingCommand() {} - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform = nullptr) const = 0; - - virtual bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const { return false; } + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aTransform) = 0; protected: explicit DrawingCommand(CommandType aType) @@ -137,7 +130,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->DrawSurface(mSurface, mDest, mSource, mSurfOptions, mOptions); } @@ -161,7 +154,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->DrawFilter(mFilter, mSourceRect, mDestPoint, mOptions); } @@ -182,7 +175,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->ClearRect(mRect); } @@ -204,13 +197,11 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aTransform) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aTransform) { - MOZ_ASSERT(!aTransform || !aTransform->HasNonIntegerTranslation()); + MOZ_ASSERT(!aTransform.HasNonIntegerTranslation()); Point dest(Float(mDestination.x), Float(mDestination.y)); - if (aTransform) { - dest = (*aTransform) * dest; - } + dest = aTransform * dest; aDT->CopySurface(mSurface, mSourceRect, IntPoint(uint32_t(dest.x), uint32_t(dest.y))); } @@ -233,17 +224,11 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->FillRect(mRect, mPattern, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const - { - aDeviceRect = aTransform.TransformBounds(mRect); - return true; - } - private: Rect mRect; StoredPattern mPattern; @@ -263,14 +248,9 @@ public: , mStrokeOptions(aStrokeOptions) , mOptions(aOptions) { - if (aStrokeOptions.mDashLength) { - mDashes.resize(aStrokeOptions.mDashLength); - mStrokeOptions.mDashPattern = &mDashes.front(); - memcpy(&mDashes.front(), aStrokeOptions.mDashPattern, mStrokeOptions.mDashLength * sizeof(Float)); - } } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->StrokeRect(mRect, mPattern, mStrokeOptions, mOptions); } @@ -280,7 +260,6 @@ private: StoredPattern mPattern; StrokeOptions mStrokeOptions; DrawOptions mOptions; - std::vector mDashes; }; class StrokeLineCommand : public DrawingCommand @@ -300,7 +279,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->StrokeLine(mStart, mEnd, mPattern, mStrokeOptions, mOptions); } @@ -326,58 +305,17 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->Fill(mPath, mPattern, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const - { - aDeviceRect = mPath->GetBounds(aTransform); - return true; - } - private: RefPtr mPath; StoredPattern mPattern; DrawOptions mOptions; }; -#ifndef M_SQRT2 -#define M_SQRT2 1.41421356237309504880 -#endif - -#ifndef M_SQRT1_2 -#define M_SQRT1_2 0.707106781186547524400844362104849039 -#endif - -// The logic for this comes from _cairo_stroke_style_max_distance_from_path -static Rect -PathExtentsToMaxStrokeExtents(const StrokeOptions &aStrokeOptions, - const Rect &aRect, - const Matrix &aTransform) -{ - double styleExpansionFactor = 0.5f; - - if (aStrokeOptions.mLineCap == CapStyle::SQUARE) { - styleExpansionFactor = M_SQRT1_2; - } - - if (aStrokeOptions.mLineJoin == JoinStyle::MITER && - styleExpansionFactor < M_SQRT2 * aStrokeOptions.mMiterLimit) { - styleExpansionFactor = M_SQRT2 * aStrokeOptions.mMiterLimit; - } - - styleExpansionFactor *= aStrokeOptions.mLineWidth; - - double dx = styleExpansionFactor * hypot(aTransform._11, aTransform._21); - double dy = styleExpansionFactor * hypot(aTransform._22, aTransform._12); - - Rect result = aRect; - result.Inflate(dx, dy); - return result; -} - class StrokeCommand : public DrawingCommand { public: @@ -391,30 +329,18 @@ public: , mStrokeOptions(aStrokeOptions) , mOptions(aOptions) { - if (aStrokeOptions.mDashLength) { - mDashes.resize(aStrokeOptions.mDashLength); - mStrokeOptions.mDashPattern = &mDashes.front(); - memcpy(&mDashes.front(), aStrokeOptions.mDashPattern, mStrokeOptions.mDashLength * sizeof(Float)); - } } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->Stroke(mPath, mPattern, mStrokeOptions, mOptions); } - bool GetAffectedRect(Rect& aDeviceRect, const Matrix& aTransform) const - { - aDeviceRect = PathExtentsToMaxStrokeExtents(mStrokeOptions, mPath->GetBounds(aTransform), aTransform); - return true; - } - private: RefPtr mPath; StoredPattern mPattern; StrokeOptions mStrokeOptions; DrawOptions mOptions; - std::vector mDashes; }; class FillGlyphsCommand : public DrawingCommand @@ -435,7 +361,7 @@ public: memcpy(&mGlyphs.front(), aBuffer.mGlyphs, sizeof(Glyph) * aBuffer.mNumGlyphs); } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { GlyphBuffer buf; buf.mNumGlyphs = mGlyphs.size(); @@ -464,7 +390,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->Mask(mSource, mMask, mOptions); } @@ -490,7 +416,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->MaskSurface(mSource, mMask, mOffset, mOptions); } @@ -511,7 +437,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->PushClip(mPath); } @@ -529,7 +455,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->PushClipRect(mRect); } @@ -546,7 +472,7 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix&) { aDT->PopClip(); } @@ -561,33 +487,17 @@ public: { } - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix* aMatrix) const + virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix& aMatrix) { - if (aMatrix) { - aDT->SetTransform(mTransform * (*aMatrix)); - } else { - aDT->SetTransform(mTransform); - } + Matrix transform = mTransform; + transform *= aMatrix; + aDT->SetTransform(transform); } private: Matrix mTransform; }; -class FlushCommand : public DrawingCommand -{ -public: - explicit FlushCommand() - : DrawingCommand(CommandType::FLUSH) - { - } - - virtual void ExecuteOnDT(DrawTarget* aDT, const Matrix*) const - { - aDT->Flush(); - } -}; - } // namespace gfx } // namespace mozilla diff --git a/gfx/2d/DrawTargetCapture.cpp b/gfx/2d/DrawTargetCapture.cpp index f3dd4d0923e..84d08d7d28c 100644 --- a/gfx/2d/DrawTargetCapture.cpp +++ b/gfx/2d/DrawTargetCapture.cpp @@ -188,7 +188,7 @@ DrawTargetCaptureImpl::ReplayToDrawTarget(DrawTarget* aDT, const Matrix& aTransf uint8_t* current = start; while (current < start + mDrawCommandStorage.size()) { - reinterpret_cast(current + sizeof(uint32_t))->ExecuteOnDT(aDT, &aTransform); + reinterpret_cast(current + sizeof(uint32_t))->ExecuteOnDT(aDT, aTransform); current += *(uint32_t*)current; } } diff --git a/gfx/2d/DrawingJob.cpp b/gfx/2d/DrawingJob.cpp deleted file mode 100644 index 728e330f421..00000000000 --- a/gfx/2d/DrawingJob.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "DrawingJob.h" -#include "JobScheduler.h" -#include "mozilla/gfx/2D.h" - -namespace mozilla { -namespace gfx { - -DrawingJobBuilder::DrawingJobBuilder() -{} - -DrawingJobBuilder::~DrawingJobBuilder() -{ - MOZ_ASSERT(!mDrawTarget); -} - -void -DrawingJob::Clear() -{ - mCommandBuffer = nullptr; - mCursor = 0; -} - -void -DrawingJobBuilder::BeginDrawingJob(DrawTarget* aTarget, IntPoint aOffset, - SyncObject* aStart) -{ - MOZ_ASSERT(mCommandOffsets.empty()); - MOZ_ASSERT(aTarget); - mDrawTarget = aTarget; - mOffset = aOffset; - mStart = aStart; -} - -DrawingJob* -DrawingJobBuilder::EndDrawingJob(CommandBuffer* aCmdBuffer, - SyncObject* aCompletion, - WorkerThread* aPinToWorker) -{ - MOZ_ASSERT(mDrawTarget); - DrawingJob* task = new DrawingJob(mDrawTarget, mOffset, mStart, aCompletion, aPinToWorker); - task->mCommandBuffer = aCmdBuffer; - task->mCommandOffsets = Move(mCommandOffsets); - - mDrawTarget = nullptr; - mOffset = IntPoint(); - mStart = nullptr; - - return task; -} - -DrawingJob::DrawingJob(DrawTarget* aTarget, IntPoint aOffset, - SyncObject* aStart, SyncObject* aCompletion, - WorkerThread* aPinToWorker) -: Job(aStart, aCompletion, aPinToWorker) -, mCommandBuffer(nullptr) -, mCursor(0) -, mDrawTarget(aTarget) -, mOffset(aOffset) -{ - mCommandOffsets.reserve(64); -} - -JobStatus -DrawingJob::Run() -{ - while (mCursor < mCommandOffsets.size()) { - - const DrawingCommand* cmd = mCommandBuffer->GetDrawingCommand(mCommandOffsets[mCursor]); - - if (!cmd) { - return JobStatus::Error; - } - - cmd->ExecuteOnDT(mDrawTarget); - - ++mCursor; - } - - return JobStatus::Complete; -} - -DrawingJob::~DrawingJob() -{ - Clear(); -} - -const DrawingCommand* -CommandBuffer::GetDrawingCommand(ptrdiff_t aId) -{ - return static_cast(mStorage.GetStorage(aId)); -} - -CommandBuffer::~CommandBuffer() -{ - mStorage.ForEach([](void* item){ - static_cast(item)->~DrawingCommand(); - }); - mStorage.Clear(); -} - -void -CommandBufferBuilder::BeginCommandBuffer(size_t aBufferSize) -{ - MOZ_ASSERT(!mCommands); - mCommands = new CommandBuffer(aBufferSize); -} - -already_AddRefed -CommandBufferBuilder::EndCommandBuffer() -{ - return mCommands.forget(); -} - -} // namespace -} // namespace diff --git a/gfx/2d/DrawingJob.h b/gfx/2d/DrawingJob.h deleted file mode 100644 index 1f279bc3d86..00000000000 --- a/gfx/2d/DrawingJob.h +++ /dev/null @@ -1,157 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MOZILLA_GFX_COMMANDBUFFER_H_ -#define MOZILLA_GFX_COMMANDBUFFER_H_ - -#include - -#include "mozilla/RefPtr.h" -#include "mozilla/Assertions.h" -#include "mozilla/gfx/Matrix.h" -#include "mozilla/gfx/JobScheduler.h" -#include "mozilla/gfx/IterableArena.h" -#include "DrawCommand.h" - -namespace mozilla { -namespace gfx { - -class DrawingCommand; -class PrintCommand; -class SignalCommand; -class DrawingJob; -class WaitCommand; - -class SyncObject; -class MultiThreadedJobQueue; - -class DrawTarget; - -class DrawingJobBuilder; -class CommandBufferBuilder; - -/// Contains a sequence of immutable drawing commands that are typically used by -/// several DrawingJobs. -/// -/// CommandBuffer objects are built using CommandBufferBuilder. -class CommandBuffer : public external::AtomicRefCounted -{ -public: - MOZ_DECLARE_REFCOUNTED_TYPENAME(CommandBuffer) - - ~CommandBuffer(); - - const DrawingCommand* GetDrawingCommand(ptrdiff_t aId); - -protected: - explicit CommandBuffer(size_t aSize = 256) - : mStorage(IterableArena::GROWABLE, aSize) - {} - - IterableArena mStorage; - friend class CommandBufferBuilder; -}; - -/// Generates CommandBuffer objects. -/// -/// The builder is a separate object to ensure that commands are not added to a -/// submitted CommandBuffer. -class CommandBufferBuilder -{ -public: - void BeginCommandBuffer(size_t aBufferSize = 256); - - already_AddRefed EndCommandBuffer(); - - /// Build the CommandBuffer, command after command. - /// This must be used between BeginCommandBuffer and EndCommandBuffer. - template - ptrdiff_t AddCommand(Args&&... aArgs) - { - static_assert(IsBaseOf::value, - "T must derive from DrawingCommand"); - return mCommands->mStorage.Alloc(Forward(aArgs)...); - } - - bool HasCommands() const { return !!mCommands; } - -protected: - RefPtr mCommands; -}; - -/// Stores multiple commands to be executed sequencially. -class DrawingJob : public Job { -public: - ~DrawingJob(); - - virtual JobStatus Run() override; - -protected: - DrawingJob(DrawTarget* aTarget, - IntPoint aOffset, - SyncObject* aStart, - SyncObject* aCompletion, - WorkerThread* aPinToWorker = nullptr); - - /// Runs the tasks's destructors and resets the buffer. - void Clear(); - - std::vector mCommandOffsets; - RefPtr mCommandBuffer; - uint32_t mCursor; - - RefPtr mDrawTarget; - IntPoint mOffset; - - friend class DrawingJobBuilder; -}; - -/// Generates DrawingJob objects. -/// -/// The builder is a separate object to ensure that commands are not added to a -/// submitted DrawingJob. -class DrawingJobBuilder { -public: - DrawingJobBuilder(); - - ~DrawingJobBuilder(); - - /// Allocates a DrawingJob. - /// - /// call this method before starting to add commands. - void BeginDrawingJob(DrawTarget* aTarget, IntPoint aOffset, - SyncObject* aStart = nullptr); - - /// Build the DrawingJob, command after command. - /// This must be used between BeginDrawingJob and EndDrawingJob. - void AddCommand(ptrdiff_t offset) - { - mCommandOffsets.push_back(offset); - } - - /// Finalizes and returns the drawing task. - /// - /// If aCompletion is not null, the sync object will be signaled after the - /// task buffer is destroyed (and after the destructor of the tasks have run). - /// In most cases this means after the completion of all tasks in the task buffer, - /// but also when the task buffer is destroyed due to an error. - DrawingJob* EndDrawingJob(CommandBuffer* aCmdBuffer, - SyncObject* aCompletion = nullptr, - WorkerThread* aPinToWorker = nullptr); - - /// Returns true between BeginDrawingJob and EndDrawingJob, false otherwise. - bool HasDrawingJob() const { return !!mDrawTarget; } - -protected: - std::vector mCommandOffsets; - RefPtr mDrawTarget; - IntPoint mOffset; - RefPtr mStart; -}; - -} // namespace -} // namespace - -#endif diff --git a/gfx/2d/IterableArena.h b/gfx/2d/IterableArena.h deleted file mode 100644 index 0f993c9c110..00000000000 --- a/gfx/2d/IterableArena.h +++ /dev/null @@ -1,193 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MOZILLA_GFX_ITERABLEARENA_H_ -#define MOZILLA_GFX_ITERABLEARENA_H_ - -#include "mozilla/Move.h" -#include "mozilla/Assertions.h" -#include "mozilla/gfx/Logging.h" - -#include -#include -#include -#include - -namespace mozilla { -namespace gfx { - -/// A simple pool allocator for plain data structures. -/// -/// Beware that the pool will not attempt to run the destructors. It is the -/// responsibility of the user of this class to either use objects with no -/// destructor or to manually call the allocated objects destructors. -/// If the pool is growable, its allocated objects must be safely moveable in -/// in memory (through memcpy). -class IterableArena { -protected: - struct Header - { - size_t mBlocSize; - }; -public: - enum ArenaType { - FIXED_SIZE, - GROWABLE - }; - - IterableArena(ArenaType aType, size_t aStorageSize) - : mSize(aStorageSize) - , mCursor(0) - , mIsGrowable(aType == GROWABLE) - { - if (mSize == 0) { - mSize = 128; - } - - mStorage = (uint8_t*)malloc(mSize); - if (mStorage == nullptr) { - gfxCriticalError() << "Not enough Memory allocate a memory pool of size " << aStorageSize; - MOZ_CRASH(); - } - } - - ~IterableArena() - { - free(mStorage); - } - - /// Constructs a new item in the pool and returns a positive offset in case of - /// success. - /// - /// The offset never changes even if the storage is reallocated, so users - /// of this class should prefer storing offsets rather than direct pointers - /// to the allocated objects. - /// Alloc can cause the storage to be reallocated if the pool was initialized - /// with IterableArena::GROWABLE. - /// If for any reason the pool fails to allocate enough space for the new item - /// Alloc returns a negative offset and the object's constructor is not called. - template - ptrdiff_t - Alloc(Args&&... aArgs) - { - void* storage = nullptr; - auto offset = AllocRaw(sizeof(T), &storage); - if (offset < 0) { - return offset; - } - new (storage) T(Forward(aArgs)...); - return offset; - } - - ptrdiff_t AllocRaw(size_t aSize, void** aOutPtr = nullptr) - { - const size_t blocSize = AlignedSize(sizeof(Header) + aSize); - - if (AlignedSize(mCursor + blocSize) > mSize) { - if (!mIsGrowable) { - return -1; - } - - size_t newSize = mSize * 2; - while (AlignedSize(mCursor + blocSize) > newSize) { - newSize *= 2; - } - - uint8_t* newStorage = (uint8_t*)realloc(mStorage, newSize); - if (!newStorage) { - gfxCriticalError() << "Not enough Memory to grow the memory pool, size: " << newSize; - return -1; - } - - mStorage = newStorage; - mSize = newSize; - } - ptrdiff_t offset = mCursor; - GetHeader(offset)->mBlocSize = blocSize; - mCursor += blocSize; - if (aOutPtr) { - *aOutPtr = GetStorage(offset); - } - return offset; - } - - /// Get access to an allocated item at a given offset (only use offsets returned - /// by Alloc or AllocRaw). - /// - /// If the pool is growable, the returned pointer is only valid temporarily. The - /// underlying storage can be reallocated in Alloc or AllocRaw, so do not keep - /// these pointers around and store the offset instead. - void* GetStorage(ptrdiff_t offset = 0) - { - MOZ_ASSERT(offset >= 0); - MOZ_ASSERT(offset < mCursor); - return offset >= 0 ? mStorage + offset + sizeof(Header) : nullptr; - } - - /// Clears the storage without running any destructor and without deallocating it. - void Clear() - { - mCursor = 0; - } - - /// Iterate over the elements allocated in this pool. - /// - /// Takes a lambda or function object accepting a void* as parameter. - template - void ForEach(Func cb) - { - Iterator it; - while (void* ptr = it.Next(this)) { - cb(ptr); - } - } - - /// A simple iterator over an arena. - class Iterator { - public: - Iterator() - : mCursor(0) - {} - - void* Next(IterableArena* aArena) - { - if (mCursor >= aArena->mCursor) { - return nullptr; - } - void* result = aArena->GetStorage(mCursor); - const size_t blocSize = aArena->GetHeader(mCursor)->mBlocSize; - MOZ_ASSERT(blocSize != 0); - mCursor += blocSize; - return result; - } - - private: - ptrdiff_t mCursor; - }; - -protected: - Header* GetHeader(ptrdiff_t offset) - { - return (Header*) (mStorage + offset); - } - - size_t AlignedSize(size_t aSize) const - { - const size_t alignment = sizeof(uintptr_t); - return aSize + (alignment - (aSize % alignment)) % alignment; - } - - uint8_t* mStorage; - uint32_t mSize; - ptrdiff_t mCursor; - bool mIsGrowable; - - friend class Iterator; -}; - -} // namespace -} // namespace - -#endif diff --git a/gfx/2d/JobScheduler.cpp b/gfx/2d/JobScheduler.cpp deleted file mode 100644 index 8d7ab727e90..00000000000 --- a/gfx/2d/JobScheduler.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "JobScheduler.h" - -namespace mozilla { -namespace gfx { - -JobScheduler* JobScheduler::sSingleton = nullptr; - -bool JobScheduler::Init(uint32_t aNumThreads, uint32_t aNumQueues) -{ - MOZ_ASSERT(!sSingleton); - MOZ_ASSERT(aNumThreads >= aNumQueues); - - sSingleton = new JobScheduler(); - sSingleton->mNextQueue = 0; - - for (uint32_t i = 0; i < aNumQueues; ++i) { - sSingleton->mDrawingQueues.push_back(new MultiThreadedJobQueue()); - } - - for (uint32_t i = 0; i < aNumThreads; ++i) { - sSingleton->mWorkerThreads.push_back(new WorkerThread(sSingleton->mDrawingQueues[i%aNumQueues])); - } - return true; -} - -void JobScheduler::ShutDown() -{ - MOZ_ASSERT(IsEnabled()); - if (!IsEnabled()) { - return; - } - - for (auto queue : sSingleton->mDrawingQueues) { - queue->ShutDown(); - delete queue; - } - - for (WorkerThread* thread : sSingleton->mWorkerThreads) { - // this will block until the thread is joined. - delete thread; - } - - sSingleton->mWorkerThreads.clear(); - delete sSingleton; - sSingleton = nullptr; -} - -JobStatus -JobScheduler::ProcessJob(Job* aJob) -{ - MOZ_ASSERT(aJob); - auto status = aJob->Run(); - if (status == JobStatus::Error || status == JobStatus::Complete) { - delete aJob; - } - return status; -} - -void -JobScheduler::SubmitJob(Job* aJob) -{ - MOZ_ASSERT(aJob); - RefPtr start = aJob->GetStartSync(); - if (start && start->Register(aJob)) { - // The Job buffer starts with a non-signaled sync object, it - // is now registered in the list of task buffers waiting on the - // sync object, so we should not place it in the queue. - return; - } - - GetQueueForJob(aJob)->SubmitJob(aJob); -} - -MultiThreadedJobQueue* -JobScheduler::GetQueueForJob(Job* aJob) -{ - return aJob->IsPinnedToAThread() ? aJob->GetWorkerThread()->GetJobQueue() - : GetDrawingQueue(); -} - -Job::Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread) -: mStartSync(aStart) -, mCompletionSync(aCompletion) -, mPinToThread(aThread) -{ - if (mStartSync) { - mStartSync->AddSubsequent(this); - } - if (mCompletionSync) { - mCompletionSync->AddPrerequisite(this); - } -} - -Job::~Job() -{ - if (mCompletionSync) { - //printf(" -- Job %p dtor completion %p\n", this, mCompletionSync); - mCompletionSync->Signal(); - mCompletionSync = nullptr; - } -} - -JobStatus -SetEventJob::Run() -{ - mEvent->Set(); - return JobStatus::Complete; -} - -SetEventJob::SetEventJob(EventObject* aEvent, - SyncObject* aStart, SyncObject* aCompletion, - WorkerThread* aWorker) -: Job(aStart, aCompletion, aWorker) -, mEvent(aEvent) -{} - -SetEventJob::~SetEventJob() -{} - -SyncObject::SyncObject(uint32_t aNumPrerequisites) -: mSignals(aNumPrerequisites) -#ifdef DEBUG -, mNumPrerequisites(aNumPrerequisites) -, mAddedPrerequisites(0) -#endif -{} - -SyncObject::~SyncObject() -{ - MOZ_ASSERT(mWaitingJobs.size() == 0); -} - -bool -SyncObject::Register(Job* aJob) -{ - MOZ_ASSERT(aJob); - - // For now, ensure that when we schedule the first subsequent, we have already - // created all of the prerequisites. This is an arbitrary restriction because - // we specify the number of prerequisites in the constructor, but in the typical - // scenario, if the assertion FreezePrerequisite blows up here it probably means - // we got the initial nmber of prerequisites wrong. We can decide to remove - // this restriction if needed. - FreezePrerequisites(); - - int32_t signals = mSignals; - - if (signals > 0) { - AddWaitingJob(aJob); - // Since Register and Signal can be called concurrently, it can happen that - // reading mSignals in Register happens before decrementing mSignals in Signal, - // but SubmitWaitingJobs happens before AddWaitingJob. This ordering means - // the SyncObject ends up in the signaled state with a task sitting in the - // waiting list. To prevent that we check mSignals a second time and submit - // again if signals reached zero in the mean time. - // We do this instead of holding a mutex around mSignals+mJobs to reduce - // lock contention. - int32_t signals2 = mSignals; - if (signals2 == 0) { - SubmitWaitingJobs(); - } - return true; - } - - return false; -} - -void -SyncObject::Signal() -{ - int32_t signals = --mSignals; - MOZ_ASSERT(signals >= 0); - - if (signals == 0) { - SubmitWaitingJobs(); - } -} - -void -SyncObject::AddWaitingJob(Job* aJob) -{ - MutexAutoLock lock(&mMutex); - mWaitingJobs.push_back(aJob); -} - -void SyncObject::SubmitWaitingJobs() -{ - std::vector tasksToSubmit; - { - // Scheduling the tasks can cause code that modifies 's reference - // count to run concurrently, and cause the caller of this function to - // be owned by another thread. We need to make sure the reference count - // does not reach 0 on another thread before mWaitingJobs.clear(), so - // hold a strong ref to prevent that! - RefPtr kungFuDeathGrip(this); - - MutexAutoLock lock(&mMutex); - tasksToSubmit = Move(mWaitingJobs); - mWaitingJobs.clear(); - } - - for (Job* task : tasksToSubmit) { - JobScheduler::GetQueueForJob(task)->SubmitJob(task); - } -} - -bool -SyncObject::IsSignaled() -{ - return mSignals == 0; -} - -void -SyncObject::FreezePrerequisites() -{ - MOZ_ASSERT(mAddedPrerequisites == mNumPrerequisites); -} - -void -SyncObject::AddPrerequisite(Job* aJob) -{ - MOZ_ASSERT(++mAddedPrerequisites <= mNumPrerequisites); -} - -void -SyncObject::AddSubsequent(Job* aJob) -{ -} - -} //namespace -} //namespace diff --git a/gfx/2d/JobScheduler.h b/gfx/2d/JobScheduler.h deleted file mode 100644 index e58cee721a1..00000000000 --- a/gfx/2d/JobScheduler.h +++ /dev/null @@ -1,231 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MOZILLA_GFX_TASKSCHEDULER_H_ -#define MOZILLA_GFX_TASKSCHEDULER_H_ - -#include "mozilla/RefPtr.h" -#include "mozilla/gfx/Types.h" - -#ifdef WIN32 -#include "mozilla/gfx/JobScheduler_win32.h" -#else -#include "mozilla/gfx/JobScheduler_posix.h" -#endif - -namespace mozilla { -namespace gfx { - -class MultiThreadedJobQueue; -class SyncObject; - -class JobScheduler { -public: - /// Return one of the queues that the drawing worker threads pull from, chosen - /// pseudo-randomly. - static MultiThreadedJobQueue* GetDrawingQueue() - { - return sSingleton->mDrawingQueues[ - sSingleton->mNextQueue++ % sSingleton->mDrawingQueues.size() - ]; - } - - /// Return one of the queues that the drawing worker threads pull from with a - /// hash to choose the queue. - /// - /// Calling this function several times with the same hash will yield the same queue. - static MultiThreadedJobQueue* GetDrawingQueue(uint32_t aHash) - { - return sSingleton->mDrawingQueues[ - aHash % sSingleton->mDrawingQueues.size() - ]; - } - - /// Return the task queue associated to the worker the task is pinned to if - /// the task is pinned to a worker, or a random queue. - static MultiThreadedJobQueue* GetQueueForJob(Job* aJob); - - /// Initialize the task scheduler with aNumThreads worker threads for drawing - /// and aNumQueues task queues. - /// - /// The number of threads must be superior or equal to the number of queues - /// (since for now a worker thread only pulls from one queue). - static bool Init(uint32_t aNumThreads, uint32_t aNumQueues); - - /// Shut the scheduler down. - /// - /// This will block until worker threads are joined and deleted. - static void ShutDown(); - - /// Returns true if there is a successfully initialized JobScheduler singleton. - static bool IsEnabled() { return !!sSingleton; } - - /// Submit a task buffer to its associated queue. - /// - /// The caller looses ownership of the task buffer. - static void SubmitJob(Job* aJobs); - - /// Process commands until the command buffer needs to block on a sync object, - /// completes, yields, or encounters an error. - /// - /// Can be used on any thread. Worker threads basically loop over this, but the - /// main thread can also dequeue pending task buffers and process them alongside - /// the worker threads if it is about to block until completion anyway. - /// - /// The caller looses ownership of the task buffer. - static JobStatus ProcessJob(Job* aJobs); - -protected: - static JobScheduler* sSingleton; - - // queues of Job that are ready to be processed - std::vector mDrawingQueues; - std::vector mWorkerThreads; - Atomic mNextQueue; -}; - -/// Jobs are not reference-counted because they don't have shared ownership. -/// The ownership of tasks can change when they are passed to certain methods -/// of JobScheduler and SyncObject. See the docuumentaion of these classes. -class Job { -public: - Job(SyncObject* aStart, SyncObject* aCompletion, WorkerThread* aThread = nullptr); - - virtual ~Job(); - - virtual JobStatus Run() = 0; - - /// For use in JobScheduler::SubmitJob. Don't use it anywhere else. - //already_AddRefed GetAndResetStartSync(); - SyncObject* GetStartSync() { return mStartSync; } - - bool IsPinnedToAThread() const { return !!mPinToThread; } - - WorkerThread* GetWorkerThread() { return mPinToThread; } - -protected: - RefPtr mStartSync; - RefPtr mCompletionSync; - WorkerThread* mPinToThread; -}; - -class EventObject; - -/// This task will set an EventObject. -/// -/// Typically used as the final task, so that the main thread can block on the -/// corresponfing EventObject until all of the tasks are processed. -class SetEventJob : public Job -{ -public: - explicit SetEventJob(EventObject* aEvent, - SyncObject* aStart, SyncObject* aCompletion = nullptr, - WorkerThread* aPinToWorker = nullptr); - - ~SetEventJob(); - - JobStatus Run() override; - - EventObject* GetEvent() { return mEvent; } - -protected: - RefPtr mEvent; -}; - -/// A synchronization object that can be used to express dependencies and ordering between -/// tasks. -/// -/// Jobs can register to SyncObjects in order to asynchronously wait for a signal. -/// In practice, Job objects usually start with a sync object (startSyc) and end -/// with another one (completionSync). -/// a Job never gets processed before its startSync is in the signaled state, and -/// signals its completionSync as soon as it finishes. This is how dependencies -/// between tasks is expressed. -class SyncObject final : public external::AtomicRefCounted { -public: - MOZ_DECLARE_REFCOUNTED_TYPENAME(SyncObject) - - /// Create a synchronization object. - /// - /// aNumPrerequisites represents the number of times the object must be signaled - /// before actually entering the signaled state (in other words, it means the - /// number of dependencies of this sync object). - /// - /// Explicitly specifying the number of prerequisites when creating sync objects - /// makes it easy to start scheduling some of the prerequisite tasks while - /// creating the others, which is how we typically use the task scheduler. - /// Automatically determining the number of prerequisites using Job's constructor - /// brings the risk that the sync object enters the signaled state while we - /// are still adding prerequisites which is hard to fix without using muteces. - explicit SyncObject(uint32_t aNumPrerequisites = 1); - - ~SyncObject(); - - /// Attempt to register a task. - /// - /// If the sync object is already in the signaled state, the buffer is *not* - /// registered and the sync object does not take ownership of the task. - /// If the object is not yet in the signaled state, it takes ownership of - /// the task and places it in a list of pending tasks. - /// Pending tasks will not be processed by the worker thread. - /// When the SyncObject reaches the signaled state, it places the pending - /// tasks back in the available buffer queue, so that they can be - /// scheduled again. - /// - /// Returns true if the SyncOject is not already in the signaled state. - /// This means that if this method returns true, the SyncObject has taken - /// ownership of the Job. - bool Register(Job* aJob); - - /// Signal the SyncObject. - /// - /// This decrements an internal counter. The sync object reaches the signaled - /// state when the counter gets to zero. - void Signal(); - - /// Returns true if mSignals is equal to zero. In other words, returns true - /// if all prerequisite tasks have already signaled the sync object. - bool IsSignaled(); - - /// Asserts that the number of added prerequisites is equal to the number - /// specified in the constructor (does nothin in release builds). - void FreezePrerequisites(); - -private: - // Called by Job's constructor - void AddSubsequent(Job* aJob); - void AddPrerequisite(Job* aJob); - - void AddWaitingJob(Job* aJob); - - void SubmitWaitingJobs(); - - std::vector mWaitingJobs; - Mutex mMutex; // for concurrent access to mWaintingJobs - Atomic mSignals; - -#ifdef DEBUG - uint32_t mNumPrerequisites; - Atomic mAddedPrerequisites; -#endif - - friend class Job; - friend class JobScheduler; -}; - - -/// RAII helper. -struct MutexAutoLock { - MutexAutoLock(Mutex* aMutex) : mMutex(aMutex) { mMutex->Lock(); } - ~MutexAutoLock() { mMutex->Unlock(); } -protected: - Mutex* mMutex; -}; - - -} // namespace -} // namespace - -#endif \ No newline at end of file diff --git a/gfx/2d/JobScheduler_posix.cpp b/gfx/2d/JobScheduler_posix.cpp deleted file mode 100644 index 0b7df32c7f6..00000000000 --- a/gfx/2d/JobScheduler_posix.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "JobScheduler.h" -#include "mozilla/gfx/Logging.h" - -using namespace std; - -namespace mozilla { -namespace gfx { - -MultiThreadedJobQueue::MultiThreadedJobQueue() -: mThreadsCount(0) -, mShuttingDown(false) -{} - -MultiThreadedJobQueue::~MultiThreadedJobQueue() -{ - MOZ_ASSERT(mJobs.empty()); -} - -bool -MultiThreadedJobQueue::WaitForJob(Job*& aOutJob) -{ - return PopJob(aOutJob, BLOCKING); -} - -bool -MultiThreadedJobQueue::PopJob(Job*& aOutJobs, AccessType aAccess) -{ - for (;;) { - MutexAutoLock lock(&mMutex); - - while (aAccess == BLOCKING && !mShuttingDown && mJobs.empty()) { - mAvailableCondvar.Wait(&mMutex); - } - - if (mShuttingDown) { - return false; - } - - if (mJobs.empty()) { - if (aAccess == NON_BLOCKING) { - return false; - } - continue; - } - - Job* task = mJobs.front(); - MOZ_ASSERT(task); - - mJobs.pop_front(); - - aOutJobs = task; - return true; - } -} - -void -MultiThreadedJobQueue::SubmitJob(Job* aJobs) -{ - MOZ_ASSERT(aJobs); - MutexAutoLock lock(&mMutex); - mJobs.push_back(aJobs); - mAvailableCondvar.Broadcast(); -} - -size_t -MultiThreadedJobQueue::NumJobs() -{ - MutexAutoLock lock(&mMutex); - return mJobs.size(); -} - -bool -MultiThreadedJobQueue::IsEmpty() -{ - MutexAutoLock lock(&mMutex); - return mJobs.empty(); -} - -void -MultiThreadedJobQueue::ShutDown() -{ - MutexAutoLock lock(&mMutex); - mShuttingDown = true; - while (mThreadsCount) { - mAvailableCondvar.Broadcast(); - mShutdownCondvar.Wait(&mMutex); - } -} - -void -MultiThreadedJobQueue::RegisterThread() -{ - mThreadsCount += 1; -} - -void -MultiThreadedJobQueue::UnregisterThread() -{ - MutexAutoLock lock(&mMutex); - mThreadsCount -= 1; - if (mThreadsCount == 0) { - mShutdownCondvar.Broadcast(); - } -} - -void* ThreadCallback(void* threadData) -{ - WorkerThread* thread = (WorkerThread*)threadData; - thread->Run(); - return nullptr; -} - -WorkerThread::WorkerThread(MultiThreadedJobQueue* aJobQueue) -: mQueue(aJobQueue) -{ - aJobQueue->RegisterThread(); - pthread_create(&mThread, nullptr, ThreadCallback, this); -} - -WorkerThread::~WorkerThread() -{ - pthread_join(mThread, nullptr); -} - -void -WorkerThread::SetName(const char* aName) -{ - // Call this from the thread itself because of Mac. -#ifdef XP_MACOSX - pthread_setname_np(aName); -#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) - pthread_set_name_np(mThread, aName); -#elif defined(__NetBSD__) - pthread_setname_np(mThread, "%s", (void*)aName); -#else - pthread_setname_np(mThread, aName); -#endif -} - -void -WorkerThread::Run() -{ - SetName("gfx worker"); - - for (;;) { - Job* commands = nullptr; - if (!mQueue->WaitForJob(commands)) { - mQueue->UnregisterThread(); - return; - } - - JobStatus status = JobScheduler::ProcessJob(commands); - - if (status == JobStatus::Error) { - // Don't try to handle errors for now, but that's open to discussions. - // I expect errors to be mostly OOM issues. - MOZ_CRASH(); - } - } -} - -EventObject::EventObject() -: mIsSet(false) -{} - -EventObject::~EventObject() -{} - -bool -EventObject::Peak() -{ - MutexAutoLock lock(&mMutex); - return mIsSet; -} - -void -EventObject::Set() -{ - MutexAutoLock lock(&mMutex); - if (!mIsSet) { - mIsSet = true; - mCond.Broadcast(); - } -} - -void -EventObject::Wait() -{ - MutexAutoLock lock(&mMutex); - if (mIsSet) { - return; - } - mCond.Wait(&mMutex); -} - -} // namespce -} // namespce diff --git a/gfx/2d/JobScheduler_posix.h b/gfx/2d/JobScheduler_posix.h deleted file mode 100644 index f4071c34e47..00000000000 --- a/gfx/2d/JobScheduler_posix.h +++ /dev/null @@ -1,187 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef WIN32 -#ifndef MOZILLA_GFX_TASKSCHEDULER_POSIX_H_ -#define MOZILLA_GFX_TASKSCHEDULER_POSIX_H_ - -#include -#include -#include -#include -#include -#include - -#include "mozilla/RefPtr.h" -#include "mozilla/DebugOnly.h" - -namespace mozilla { -namespace gfx { - -class Job; -class PosixCondVar; - -class Mutex { -public: - Mutex() { - DebugOnly err = pthread_mutex_init(&mMutex, nullptr); - MOZ_ASSERT(!err); - } - - ~Mutex() { - DebugOnly err = pthread_mutex_destroy(&mMutex); - MOZ_ASSERT(!err); - } - - void Lock() { - DebugOnly err = pthread_mutex_lock(&mMutex); - MOZ_ASSERT(!err); - } - - void Unlock() { - DebugOnly err = pthread_mutex_unlock(&mMutex); - MOZ_ASSERT(!err); - } - -protected: - pthread_mutex_t mMutex; - friend class PosixCondVar; -}; - -// posix platforms only! -class PosixCondVar { -public: - PosixCondVar() { - DebugOnly err = pthread_cond_init(&mCond, nullptr); - MOZ_ASSERT(!err); - } - - ~PosixCondVar() { - DebugOnly err = pthread_cond_destroy(&mCond); - MOZ_ASSERT(!err); - } - - void Wait(Mutex* aMutex) { - DebugOnly err = pthread_cond_wait(&mCond, &aMutex->mMutex); - MOZ_ASSERT(!err); - } - - void Broadcast() { - DebugOnly err = pthread_cond_broadcast(&mCond); - MOZ_ASSERT(!err); - } - -protected: - pthread_cond_t mCond; -}; - - -/// A simple and naive multithreaded task queue -/// -/// The public interface of this class must remain identical to its equivalent -/// in JobScheduler_win32.h -class MultiThreadedJobQueue { -public: - enum AccessType { - BLOCKING, - NON_BLOCKING - }; - - // Producer thread - MultiThreadedJobQueue(); - - // Producer thread - ~MultiThreadedJobQueue(); - - // Worker threads - bool WaitForJob(Job*& aOutJob); - - // Any thread - bool PopJob(Job*& aOutJob, AccessType aAccess); - - // Any threads - void SubmitJob(Job* aJob); - - // Producer thread - void ShutDown(); - - // Any thread - size_t NumJobs(); - - // Any thread - bool IsEmpty(); - - // Producer thread - void RegisterThread(); - - // Worker threads - void UnregisterThread(); - -protected: - - std::list mJobs; - Mutex mMutex; - PosixCondVar mAvailableCondvar; - PosixCondVar mShutdownCondvar; - int32_t mThreadsCount; - bool mShuttingDown; - - friend class WorkerThread; -}; - -/// Worker thread that continuously dequeues Jobs from a MultiThreadedJobQueue -/// and process them. -/// -/// The public interface of this class must remain identical to its equivalent -/// in JobScheduler_win32.h -class WorkerThread { -public: - explicit WorkerThread(MultiThreadedJobQueue* aJobQueue); - - ~WorkerThread(); - - void Run(); - - MultiThreadedJobQueue* GetJobQueue() { return mQueue; } -protected: - void SetName(const char* name); - - MultiThreadedJobQueue* mQueue; - pthread_t mThread; -}; - -/// An object that a thread can synchronously wait on. -/// Usually set by a SetEventJob. -class EventObject : public external::AtomicRefCounted -{ -public: - MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) - - EventObject(); - - ~EventObject(); - - /// Synchronously wait until the event is set. - void Wait(); - - /// Return true if the event is set, without blocking. - bool Peak(); - - /// Set the event. - void Set(); - -protected: - Mutex mMutex; - PosixCondVar mCond; - bool mIsSet; -}; - -} // namespace -} // namespace - -#include "JobScheduler.h" - -#endif -#endif diff --git a/gfx/2d/JobScheduler_win32.h b/gfx/2d/JobScheduler_win32.h deleted file mode 100644 index ca2db97dad0..00000000000 --- a/gfx/2d/JobScheduler_win32.h +++ /dev/null @@ -1,76 +0,0 @@ -/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifdef WIN32 -#ifndef MOZILLA_GFX_TASKSCHEDULER_WIN32_H_ -#define MOZILLA_GFX_TASKSCHEDULER_WIN32_H_ - -#define NOT_IMPLEMENTED MOZ_CRASH("Not implemented") - -#include "mozilla/RefPtr.h" - -namespace mozilla { -namespace gfx { - -class WorkerThread; -class Job; - -class Mutex { -public: - Mutex() { NOT_IMPLEMENTED; } - ~Mutex() { NOT_IMPLEMENTED; } - void Lock() { NOT_IMPLEMENTED; } - void Unlock() { NOT_IMPLEMENTED; } -}; - -// The public interface of this class must remain identical to its equivalent -// in JobScheduler_posix.h -class MultiThreadedJobQueue { -public: - enum AccessType { - BLOCKING, - NON_BLOCKING - }; - - bool WaitForJob(Job*& aOutCommands) { NOT_IMPLEMENTED; } - bool PopJob(Job*& aOutCommands, AccessType aAccess) { NOT_IMPLEMENTED; } - void SubmitJob(Job* aCommands) { NOT_IMPLEMENTED; } - void ShutDown() { NOT_IMPLEMENTED; } - size_t NumJobs() { NOT_IMPLEMENTED; } - bool IsEmpty() { NOT_IMPLEMENTED; } - void RegisterThread() { NOT_IMPLEMENTED; } - void UnregisterThread() { NOT_IMPLEMENTED; } - - friend class WorkerThread; -}; - - -// The public interface of this class must remain identical to its equivalent -// in JobScheduler_posix.h -class EventObject : public external::AtomicRefCounted -{ -public: - MOZ_DECLARE_REFCOUNTED_TYPENAME(EventObject) - - EventObject() { NOT_IMPLEMENTED; } - ~EventObject() { NOT_IMPLEMENTED; } - void Wait() { NOT_IMPLEMENTED; } - bool Peak() { NOT_IMPLEMENTED; } - void Set() { NOT_IMPLEMENTED; } -}; - -// The public interface of this class must remain identical to its equivalent -// in JobScheduler_posix.h -class WorkerThread { -public: - explicit WorkerThread(MultiThreadedJobQueue* aJobQueue) { NOT_IMPLEMENTED; } - void Run(); -}; - -} // namespace -} // namespace - -#endif -#endif diff --git a/gfx/2d/Types.h b/gfx/2d/Types.h index 145e8142969..99143b6978e 100644 --- a/gfx/2d/Types.h +++ b/gfx/2d/Types.h @@ -289,13 +289,6 @@ struct GradientStop Color color; }; -enum class JobStatus { - Complete, - Wait, - Yield, - Error -}; - } // namespace gfx } // namespace mozilla diff --git a/gfx/2d/moz.build b/gfx/2d/moz.build index 5480302b99d..b7981946bba 100644 --- a/gfx/2d/moz.build +++ b/gfx/2d/moz.build @@ -25,10 +25,6 @@ EXPORTS.mozilla.gfx += [ 'Filters.h', 'Helpers.h', 'HelpersCairo.h', - 'IterableArena.h', - 'JobScheduler.h', - 'JobScheduler_posix.h', - 'JobScheduler_win32.h', 'Logging.h', 'Matrix.h', 'NumericTools.h', @@ -74,11 +70,6 @@ elif CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': ] DEFINES['WIN32'] = True -if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'windows': - SOURCES += [ - 'JobScheduler_posix.cpp', - ] - if CONFIG['MOZ_ENABLE_SKIA']: UNIFIED_SOURCES += [ 'convolver.cpp', @@ -127,7 +118,6 @@ UNIFIED_SOURCES += [ 'DataSourceSurface.cpp', 'DataSurfaceHelpers.cpp', 'DrawEventRecorder.cpp', - 'DrawingJob.cpp', 'DrawTarget.cpp', 'DrawTargetCairo.cpp', 'DrawTargetCapture.cpp', @@ -139,7 +129,6 @@ UNIFIED_SOURCES += [ 'FilterProcessing.cpp', 'FilterProcessingScalar.cpp', 'ImageScaling.cpp', - 'JobScheduler.cpp', 'Matrix.cpp', 'Path.cpp', 'PathCairo.cpp', diff --git a/gfx/tests/gtest/TestArena.cpp b/gfx/tests/gtest/TestArena.cpp deleted file mode 100644 index 4822b32f954..00000000000 --- a/gfx/tests/gtest/TestArena.cpp +++ /dev/null @@ -1,188 +0,0 @@ -/* vim:set ts=2 sw=2 sts=2 et: */ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "mozilla/gfx/IterableArena.h" -#include - -using namespace mozilla; -using namespace mozilla::gfx; - -#ifdef A -#undef A -#endif - -#ifdef B -#undef B -#endif - -// to avoid having symbols that collide easily like A and B in the global namespace -namespace test_arena { - -class A; -class B; - -class Base { -public: - virtual ~Base() {} - virtual A* AsA() { return nullptr; } - virtual B* AsB() { return nullptr; } -}; - -static int sDtorItemA = 0; -static int sDtorItemB = 0; - -class A : public Base { -public: - virtual A* AsA() override { return this; } - - explicit A(uint64_t val) : mVal(val) {} - ~A() { ++sDtorItemA; } - - uint64_t mVal; -}; - -class B : public Base { -public: - virtual B* AsB() override { return this; } - - explicit B(const string& str) : mVal(str) {} - ~B() { ++sDtorItemB; } - - std::string mVal; -}; - -struct BigStruct { - uint64_t mVal; - uint8_t data[120]; - - explicit BigStruct(uint64_t val) : mVal(val) {} -}; - -void TestArenaAlloc(IterableArena::ArenaType aType) -{ - sDtorItemA = 0; - sDtorItemB = 0; - IterableArena arena(aType, 256); - - // An empty arena has no items to iterate over. - { - int iterations = 0; - arena.ForEach([&](void* item){ - iterations++; - }); - ASSERT_EQ(iterations, 0); - } - - auto a1 = arena.Alloc(42); - auto b1 = arena.Alloc("Obladi oblada"); - auto a2 = arena.Alloc(1337); - auto b2 = arena.Alloc("Yellow submarine"); - auto b3 = arena.Alloc("She's got a ticket to ride"); - - // Alloc returns a non-negative offset if the allocation succeeded. - ASSERT_TRUE(a1 >= 0); - ASSERT_TRUE(a2 >= 0); - ASSERT_TRUE(b1 >= 0); - ASSERT_TRUE(b2 >= 0); - ASSERT_TRUE(b3 >= 0); - - ASSERT_TRUE(arena.GetStorage(a1) != nullptr); - ASSERT_TRUE(arena.GetStorage(a2) != nullptr); - ASSERT_TRUE(arena.GetStorage(b1) != nullptr); - ASSERT_TRUE(arena.GetStorage(b2) != nullptr); - ASSERT_TRUE(arena.GetStorage(b3) != nullptr); - - ASSERT_TRUE(((Base*)arena.GetStorage(a1))->AsA() != nullptr); - ASSERT_TRUE(((Base*)arena.GetStorage(a2))->AsA() != nullptr); - - ASSERT_TRUE(((Base*)arena.GetStorage(b1))->AsB() != nullptr); - ASSERT_TRUE(((Base*)arena.GetStorage(b2))->AsB() != nullptr); - ASSERT_TRUE(((Base*)arena.GetStorage(b3))->AsB() != nullptr); - - ASSERT_EQ(((Base*)arena.GetStorage(a1))->AsA()->mVal, (uint64_t)42); - ASSERT_EQ(((Base*)arena.GetStorage(a2))->AsA()->mVal, (uint64_t)1337); - - ASSERT_EQ(((Base*)arena.GetStorage(b1))->AsB()->mVal, std::string("Obladi oblada")); - ASSERT_EQ(((Base*)arena.GetStorage(b2))->AsB()->mVal, std::string("Yellow submarine")); - ASSERT_EQ(((Base*)arena.GetStorage(b3))->AsB()->mVal, std::string("She's got a ticket to ride")); - - { - int iterations = 0; - arena.ForEach([&](void* item){ - iterations++; - }); - ASSERT_EQ(iterations, 5); - } - - // Typically, running the destructors of the elements in the arena will is done - // manually like this: - arena.ForEach([](void* item){ - ((Base*)item)->~Base(); - }); - arena.Clear(); - ASSERT_EQ(sDtorItemA, 2); - ASSERT_EQ(sDtorItemB, 3); - - // An empty arena has no items to iterate over (we just cleared it). - { - int iterations = 0; - arena.ForEach([&](void* item){ - iterations++; - }); - ASSERT_EQ(iterations, 0); - } - -} - -void TestArenaLimit(IterableArena::ArenaType aType, bool aShouldReachLimit) -{ - IterableArena arena(aType, 128); - - // A non-growable arena should return a negative offset when running out - // of space, without crashing. - // We should not run out of space with a growable arena (unless the os is - // running out of memory but this isn't expected for this test). - bool reachedLimit = false; - for (int i = 0; i < 100; ++i) { - auto offset = arena.Alloc(42); - if (offset < 0) { - reachedLimit = true; - break; - } - } - ASSERT_EQ(reachedLimit, aShouldReachLimit); -} - -} // namespace test_arena - -using namespace test_arena; - -TEST(Moz2D, FixedArena) { - TestArenaAlloc(IterableArena::FIXED_SIZE); - TestArenaLimit(IterableArena::FIXED_SIZE, true); -} - -TEST(Moz2D, GrowableArena) { - TestArenaAlloc(IterableArena::GROWABLE); - TestArenaLimit(IterableArena::GROWABLE, false); - - IterableArena arena(IterableArena::GROWABLE, 16); - // sizeof(BigStruct) is more than twice the initial capacity, make sure that - // this doesn't blow everything up, since the arena doubles its storage size each - // time it grows (until it finds a size that fits). - auto a = arena.Alloc(1); - auto b = arena.Alloc(2); - auto c = arena.Alloc(3); - - // Offsets should also still point to the appropriate values after reallocation. - ASSERT_EQ(((BigStruct*)arena.GetStorage(a))->mVal, (uint64_t)1); - ASSERT_EQ(((BigStruct*)arena.GetStorage(b))->mVal, (uint64_t)2); - ASSERT_EQ(((BigStruct*)arena.GetStorage(c))->mVal, (uint64_t)3); - - arena.Clear(); -} diff --git a/gfx/tests/gtest/TestJobScheduler.cpp b/gfx/tests/gtest/TestJobScheduler.cpp deleted file mode 100644 index d1eed148c35..00000000000 --- a/gfx/tests/gtest/TestJobScheduler.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/* vim:set ts=2 sw=2 sts=2 et: */ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ - */ - -#ifndef WIN32 - -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "mozilla/gfx/JobScheduler.h" - -#include -#include -#include -#include - -namespace test_scheduler { - -using namespace mozilla::gfx; -using namespace mozilla; - -// Artificially cause threads to yield randomly in an attempt to make racy -// things more apparent (if any). -void MaybeYieldThread() -{ - if (rand() % 5 == 0) { - sched_yield(); - } -} - -/// Used by the TestCommand to check that tasks are processed in the right order. -struct SanityChecker { - std::vector mAdvancements; - mozilla::gfx::Mutex mMutex; - - explicit SanityChecker(uint64_t aNumCmdBuffers) - { - for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { - mAdvancements.push_back(0); - } - } - - virtual void Check(uint64_t aJobId, uint64_t aCmdId) - { - MaybeYieldThread(); - MutexAutoLock lock(&mMutex); - ASSERT_EQ(mAdvancements[aJobId], aCmdId-1); - mAdvancements[aJobId] = aCmdId; - } -}; - -/// Run checks that are specific to TestSchulerJoin. -struct JoinTestSanityCheck : public SanityChecker { - bool mSpecialJobHasRun; - - explicit JoinTestSanityCheck(uint64_t aNumCmdBuffers) - : SanityChecker(aNumCmdBuffers) - , mSpecialJobHasRun(false) - {} - - virtual void Check(uint64_t aJobId, uint64_t aCmdId) override - { - // Job 0 is the special task executed when everything is joined after task 1 - if (aCmdId == 0) { - ASSERT_FALSE(mSpecialJobHasRun); - mSpecialJobHasRun = true; - for (auto advancement : mAdvancements) { - // Because of the synchronization point (beforeFilter), all - // task buffers should have run task 1 when task 0 is run. - ASSERT_EQ(advancement, (uint32_t)1); - } - } else { - // This check does not apply to task 0. - SanityChecker::Check(aJobId, aCmdId); - } - - if (aCmdId == 2) { - ASSERT_TRUE(mSpecialJobHasRun); - } - } -}; - -class TestJob : public Job -{ -public: - TestJob(uint64_t aCmdId, uint64_t aJobId, SanityChecker* aChecker, - SyncObject* aStart, SyncObject* aCompletion) - : Job(aStart, aCompletion, nullptr) - , mCmdId(aCmdId) - , mCmdBufferId(aJobId) - , mSanityChecker(aChecker) - {} - - JobStatus Run() - { - MaybeYieldThread(); - mSanityChecker->Check(mCmdBufferId, mCmdId); - MaybeYieldThread(); - return JobStatus::Complete; - } - - uint64_t mCmdId; - uint64_t mCmdBufferId; - SanityChecker* mSanityChecker; -}; - -/// This test creates aNumCmdBuffers task buffers with sync objects set up -/// so that all tasks will join after command 5 before a task buffer runs -/// a special task (task 0) after which all task buffers fork again. -/// This simulates the kind of scenario where all tiles must join at -/// a certain point to execute, say, a filter, and fork again after the filter -/// has been processed. -/// The main thread is only blocked when waiting for the completion of the entire -/// task stream (it doesn't have to wait at the filter's sync points to orchestrate it). -void TestSchedulerJoin(uint32_t aNumThreads, uint32_t aNumCmdBuffers) -{ - JoinTestSanityCheck check(aNumCmdBuffers); - - RefPtr beforeFilter = new SyncObject(aNumCmdBuffers); - RefPtr afterFilter = new SyncObject(); - RefPtr completion = new SyncObject(aNumCmdBuffers); - - - for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { - Job* t1 = new TestJob(1, i, &check, nullptr, beforeFilter); - JobScheduler::SubmitJob(t1); - MaybeYieldThread(); - } - beforeFilter->FreezePrerequisites(); - - // This task buffer is executed when all other tasks have joined after task 1 - JobScheduler::SubmitJob( - new TestJob(0, 0, &check, beforeFilter, afterFilter) - ); - afterFilter->FreezePrerequisites(); - - for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { - Job* t2 = new TestJob(2, i, &check, afterFilter, completion); - JobScheduler::SubmitJob(t2); - MaybeYieldThread(); - } - completion->FreezePrerequisites(); - - RefPtr waitForCompletion = new EventObject(); - auto evtJob = new SetEventJob(waitForCompletion, completion); - JobScheduler::SubmitJob(evtJob); - - MaybeYieldThread(); - - waitForCompletion->Wait(); - - MaybeYieldThread(); - - for (auto advancement : check.mAdvancements) { - ASSERT_TRUE(advancement == 2); - } -} - -/// This test creates several chains of 10 task, tasks of a given chain are executed -/// sequentially, and chains are exectuted in parallel. -/// This simulates the typical scenario where we want to process sequences of drawing -/// commands for several tiles in parallel. -void TestSchedulerChain(uint32_t aNumThreads, uint32_t aNumCmdBuffers) -{ - SanityChecker check(aNumCmdBuffers); - - RefPtr completion = new SyncObject(aNumCmdBuffers); - - uint32_t numJobs = 10; - - for (uint32_t i = 0; i < aNumCmdBuffers; ++i) { - - std::vector> syncs; - std::vector tasks; - syncs.reserve(numJobs); - tasks.reserve(numJobs); - - for (uint32_t t = 0; t < numJobs-1; ++t) { - syncs.push_back(new SyncObject()); - tasks.push_back(new TestJob(t+1, i, &check, t == 0 ? nullptr - : syncs[t-1].get(), - syncs[t])); - syncs.back()->FreezePrerequisites(); - } - - tasks.push_back(new TestJob(numJobs, i, &check, syncs.back(), completion)); - - if (i % 2 == 0) { - // submit half of the tasks in order - for (Job* task : tasks) { - JobScheduler::SubmitJob(task); - MaybeYieldThread(); - } - } else { - // ... and submit the other half in reverse order - for (int32_t reverse = numJobs-1; reverse >= 0; --reverse) { - JobScheduler::SubmitJob(tasks[reverse]); - MaybeYieldThread(); - } - } - } - completion->FreezePrerequisites(); - - RefPtr waitForCompletion = new EventObject(); - auto evtJob = new SetEventJob(waitForCompletion, completion); - JobScheduler::SubmitJob(evtJob); - - MaybeYieldThread(); - - waitForCompletion->Wait(); - - for (auto advancement : check.mAdvancements) { - ASSERT_TRUE(advancement == numJobs); - } -} - -} // namespace test_scheduler - -TEST(Moz2D, JobScheduler_Join) { - srand(time(nullptr)); - for (uint32_t threads = 1; threads < 8; ++threads) { - for (uint32_t queues = 1; queues < threads; ++queues) { - for (uint32_t buffers = 1; buffers < 100; buffers += 3) { - mozilla::gfx::JobScheduler::Init(threads, queues); - test_scheduler::TestSchedulerJoin(threads, buffers); - mozilla::gfx::JobScheduler::ShutDown(); - } - } - } -} - -TEST(Moz2D, JobScheduler_Chain) { - srand(time(nullptr)); - for (uint32_t threads = 1; threads < 8; ++threads) { - for (uint32_t queues = 1; queues < threads; ++queues) { - for (uint32_t buffers = 1; buffers < 100; buffers += 3) { - mozilla::gfx::JobScheduler::Init(threads, queues); - test_scheduler::TestSchedulerChain(threads, buffers); - mozilla::gfx::JobScheduler::ShutDown(); - } - } - } -} - -#endif diff --git a/gfx/tests/gtest/moz.build b/gfx/tests/gtest/moz.build index fedbbc6b06b..cda7439695d 100644 --- a/gfx/tests/gtest/moz.build +++ b/gfx/tests/gtest/moz.build @@ -8,13 +8,11 @@ UNIFIED_SOURCES += [ 'gfxSurfaceRefCountTest.cpp', # Disabled on suspicion of causing bug 904227 #'gfxWordCacheTest.cpp', - 'TestArena.cpp', 'TestBufferRotation.cpp', 'TestColorNames.cpp', 'TestCompositor.cpp', 'TestGfxPrefs.cpp', 'TestGfxWidgets.cpp', - 'TestJobScheduler.cpp', 'TestLayers.cpp', 'TestMoz2D.cpp', 'TestQcms.cpp', From 4715141da8881f4b7d6a4afaa9abde58fcc99f12 Mon Sep 17 00:00:00 2001 From: Wes Kocher Date: Thu, 24 Sep 2015 12:06:03 -0700 Subject: [PATCH 78/78] Backed out changeset 1da385d6c46b (bug 1021845) for b2g r3 and r7 permafail CLOSED TREE --- .../composite/LayerManagerComposite.cpp | 59 ++++--------------- gfx/layers/composite/LayerManagerComposite.h | 4 +- 2 files changed, 14 insertions(+), 49 deletions(-) diff --git a/gfx/layers/composite/LayerManagerComposite.cpp b/gfx/layers/composite/LayerManagerComposite.cpp index 5d0be4cb7ea..34032824b0b 100644 --- a/gfx/layers/composite/LayerManagerComposite.cpp +++ b/gfx/layers/composite/LayerManagerComposite.cpp @@ -204,31 +204,16 @@ LayerManagerComposite::BeginTransactionWithDrawTarget(DrawTarget* aTarget, const mTargetBounds = aRect; } -template -Maybe -IntersectMaybeRects(const Maybe& aRect1, const Maybe& aRect2) -{ - if (aRect1) { - if (aRect2) { - return Some(aRect1->Intersect(*aRect2)); - } - return aRect1; - } - return aRect2; -} - void -LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion, - const Maybe& aClipFromAncestors) +LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion) { nsIntRegion localOpaque; - Matrix4x4 transform = aLayer->GetLocalTransform(); Matrix transform2d; bool isTranslation = false; // If aLayer has a simple transform (only an integer translation) then we // can easily convert aOpaqueRegion into pre-transform coordinates and include // that region. - if (transform.Is2D(&transform2d)) { + if (aLayer->GetLocalTransform().Is2D(&transform2d)) { if (transform2d.IsIntegerTranslation()) { isTranslation = true; localOpaque = aOpaqueRegion; @@ -236,38 +221,19 @@ LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaque } } - LayerComposite* composite = aLayer->AsLayerComposite(); - nsIntRegion visible = composite->GetShadowVisibleRegion(); - - // Combine our clip with the one from our ancestors. Do this even if the - // transform is not just a translation. - const Maybe& layerClip = aLayer->GetEffectiveClipRect(); - Maybe outsideClip = IntersectMaybeRects(layerClip, aClipFromAncestors); - LayerRect insideClip(LayerIntRect::FromUnknownRect(visible.GetBounds())); - if (outsideClip) { - Matrix4x4 inverse = transform; - if (inverse.Invert()) { - insideClip = UntransformTo(inverse, - ParentLayerRect(*outsideClip), - insideClip).valueOr(insideClip); - } - } - LayerIntRect combinedClip = RoundedOut(insideClip); - // Subtract any areas that we know to be opaque from our // visible region. + LayerComposite *composite = aLayer->AsLayerComposite(); if (!localOpaque.IsEmpty()) { - visible.SubOut(localOpaque); + nsIntRegion visible = composite->GetShadowVisibleRegion(); + visible.Sub(visible, localOpaque); + composite->SetShadowVisibleRegion(visible); } - visible.AndWith(LayerIntRect::ToUntyped(combinedClip)); - composite->SetShadowVisibleRegion(visible); // Compute occlusions for our descendants (in front-to-back order) and allow them to // contribute to localOpaque. for (Layer* child = aLayer->GetLastChild(); child; child = child->GetPrevSibling()) { - Maybe ancestorClipForChild = - Some(ViewAs(combinedClip, PixelCastJustification::MovingDownToChildren)); - ApplyOcclusionCulling(child, localOpaque, ancestorClipForChild); + ApplyOcclusionCulling(child, localOpaque); } // If we have a simple transform, then we can add our opaque area into @@ -276,11 +242,12 @@ LayerManagerComposite::ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaque !aLayer->HasMaskLayers() && aLayer->GetLocalOpacity() == 1.0f) { if (aLayer->GetContentFlags() & Layer::CONTENT_OPAQUE) { - localOpaque.OrWith(composite->GetFullyRenderedRegion()); + localOpaque.Or(localOpaque, composite->GetFullyRenderedRegion()); } localOpaque.MoveBy(transform2d._31, transform2d._32); - if (layerClip) { - localOpaque.And(localOpaque, ParentLayerIntRect::ToUntyped(*layerClip)); + const Maybe& clip = aLayer->GetEffectiveClipRect(); + if (clip) { + localOpaque.And(localOpaque, ParentLayerIntRect::ToUntyped(*clip)); } aOpaqueRegion.Or(aOpaqueRegion, localOpaque); } @@ -338,7 +305,7 @@ LayerManagerComposite::EndTransaction(const TimeStamp& aTimeStamp, mRoot->ComputeEffectiveTransforms(gfx::Matrix4x4()); nsIntRegion opaque; - ApplyOcclusionCulling(mRoot, opaque, Nothing()); + ApplyOcclusionCulling(mRoot, opaque); Render(); #if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK) @@ -1005,7 +972,7 @@ LayerManagerComposite::RenderToPresentationSurface() mRoot->ComputeEffectiveTransforms(matrix); nsIntRegion opaque; - ApplyOcclusionCulling(mRoot, opaque, Nothing()); + ApplyOcclusionCulling(mRoot, opaque); nsIntRegion invalid; Rect bounds(0.0f, 0.0f, scale * pageWidth, (float)actualHeight); diff --git a/gfx/layers/composite/LayerManagerComposite.h b/gfx/layers/composite/LayerManagerComposite.h index 24ed16fb3db..ca83ff2e0f3 100644 --- a/gfx/layers/composite/LayerManagerComposite.h +++ b/gfx/layers/composite/LayerManagerComposite.h @@ -172,10 +172,8 @@ public: * Restricts the shadow visible region of layers that are covered with * opaque content. aOpaqueRegion is the region already known to be covered * with opaque content, in the post-transform coordinate space of aLayer. - * aClipFromAncestors is an optional clip rect. */ - void ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion, - const Maybe& aClipFromAncestors); + void ApplyOcclusionCulling(Layer* aLayer, nsIntRegion& aOpaqueRegion); /** * RAII helper class to add a mask effect with the compositable from aMaskLayer