Bug 1127888: Fix causes of async plugin init hangs; r=jimm

dom/plugins/ipc/BrowserStreamParent.cpp

@@ -72,11 +72,7 @@ BrowserStreamParent::RecvAsyncNPP_NewStreamResult(const NPError& rv,
   }
 
   if (error != NPERR_NO_ERROR) {
-    // streamListener was suspended during async init. We must resume the stream
-    // request prior to calling _destroystream for cleanup to work correctly.
-    streamListener->ResumeRequest();
-    // We need to clean up the stream
-    parent::_destroystream(mNPP->GetNPP(), mStream, NPRES_DONE);
+    surrogate->DestroyAsyncStream(mStream);
     unused << PBrowserStreamParent::Send__delete__(this);
   }
 

dom/plugins/ipc/PluginAsyncSurrogate.cpp

@@ -358,15 +358,33 @@ PluginAsyncSurrogate::PendingNewStreamCall::PendingNewStreamCall(
 {
 }
 
-/* static */ bool
-PluginAsyncSurrogate::SetStreamType(NPStream* aStream, uint16_t aStreamType)
+/* static */ nsNPAPIPluginStreamListener*
+PluginAsyncSurrogate::GetStreamListener(NPStream* aStream)
 {
   nsNPAPIStreamWrapper* wrapper =
     reinterpret_cast<nsNPAPIStreamWrapper*>(aStream->ndata);
   if (!wrapper) {
-    return false;
+    return nullptr;
   }
-  nsNPAPIPluginStreamListener* streamListener = wrapper->GetStreamListener();
+  return wrapper->GetStreamListener();
+}
+
+void
+PluginAsyncSurrogate::DestroyAsyncStream(NPStream* aStream)
+{
+  MOZ_ASSERT(aStream);
+  nsNPAPIPluginStreamListener* streamListener = GetStreamListener(aStream);
+  MOZ_ASSERT(streamListener);
+  // streamListener was suspended during async init. We must resume the stream
+  // request prior to calling _destroystream for cleanup to work correctly.
+  streamListener->ResumeRequest();
+  parent::_destroystream(mInstance, aStream, NPRES_DONE);
+}
+
+/* static */ bool
+PluginAsyncSurrogate::SetStreamType(NPStream* aStream, uint16_t aStreamType)
+{
+  nsNPAPIPluginStreamListener* streamListener = GetStreamListener(aStream);
   if (!streamListener) {
     return false;
   }
@@ -433,6 +451,12 @@ PluginAsyncSurrogate::WaitForInit()
     mozilla::ipc::MessageChannel* contentChannel = cp->GetIPCChannel();
     MOZ_ASSERT(contentChannel);
     while (!mParent->mNPInitialized) {
+      if (mParent->mShutdown) {
+        // Since we are pumping the message channel for events, it may be
+        // possible for module initialization to fail during this loop. We must
+        // return false if this happens or else we'll be permanently stuck.
+        return false;
+      }
       result = contentChannel->WaitForIncomingMessage();
       if (!result) {
         return result;
@@ -442,6 +466,12 @@ PluginAsyncSurrogate::WaitForInit()
   mozilla::ipc::MessageChannel* channel = mParent->GetIPCChannel();
   MOZ_ASSERT(channel);
   while (!mAcceptCalls) {
+    if (mInitCancelled) {
+      // Since we are pumping the message channel for events, it may be
+      // possible for plugin instantiation to fail during this loop. We must
+      // return false if this happens or else we'll be permanently stuck.
+      return false;
+    }
     result = channel->WaitForIncomingMessage();
     if (!result) {
       break;
@@ -474,7 +504,7 @@ PluginAsyncSurrogate::NotifyAsyncInitFailed()
   // Clean up any pending NewStream requests
   for (uint32_t i = 0, len = mPendingNewStreamCalls.Length(); i < len; ++i) {
     PendingNewStreamCall& curPendingCall = mPendingNewStreamCalls[i];
-    parent::_destroystream(mInstance, curPendingCall.mStream, NPRES_DONE);
+    DestroyAsyncStream(curPendingCall.mStream);
   }
   mPendingNewStreamCalls.Clear();
 

dom/plugins/ipc/PluginAsyncSurrogate.h

@@ -79,6 +79,7 @@ public:
   void AsyncCallArriving();
 
   void NotifyAsyncInitFailed();
+  void DestroyAsyncStream(NPStream* aStream);
 
 private:
   explicit PluginAsyncSurrogate(PluginModuleParent* aParent);
@@ -117,6 +118,7 @@ private:
                                   uint32_t* aCount);
   static bool ScriptableConstruct(NPObject* aObject, const NPVariant* aArgs,
                                   uint32_t aArgCount, NPVariant* aResult);
+  static nsNPAPIPluginStreamListener* GetStreamListener(NPStream* aStream);
 
 private:
   struct PendingNewStreamCall

dom/plugins/ipc/PluginInstanceParent.cpp

@@ -1704,14 +1704,10 @@ PluginInstanceParent::RecvAsyncNPP_NewResult(const NPError& aResult)
     }
 
     nsPluginInstanceOwner* owner = GetOwner();
-    if (!owner) {
-        // This is possible in async plugin land; the instance may outlive
-        // the owner
-        return true;
-    }
-
-    if (aResult != NPERR_NO_ERROR) {
-        owner->NotifyHostAsyncInitFailed();
+    // It is possible for a plugin instance to outlive its owner when async
+    // plugin init is turned on, so we need to handle that case.
+    if (aResult != NPERR_NO_ERROR || !owner) {
+        mSurrogate->NotifyAsyncInitFailed();
         return true;
     }
 

dom/plugins/ipc/PluginModuleParent.cpp

@@ -97,6 +97,7 @@ mozilla::plugins::SetupBridge(uint32_t aPluginId,
                               bool aForceBridgeNow,
                               nsresult* rv)
 {
+    PluginModuleChromeParent::ClearInstantiationFlag();
     nsRefPtr<nsPluginHost> host = nsPluginHost::GetInst();
     nsRefPtr<nsNPAPIPlugin> plugin;
     *rv = host->GetPluginForContentProcess(aPluginId, getter_AddRefs(plugin));
@@ -105,7 +106,8 @@ mozilla::plugins::SetupBridge(uint32_t aPluginId,
     }
     PluginModuleChromeParent* chromeParent = static_cast<PluginModuleChromeParent*>(plugin->GetLibrary());
     chromeParent->SetContentParent(aContentParent);
-    if (!aForceBridgeNow && chromeParent->IsStartingAsync()) {
+    if (!aForceBridgeNow && chromeParent->IsStartingAsync() &&
+        PluginModuleChromeParent::DidInstantiate()) {
         // We'll handle the bridging asynchronously
         return true;
     }
@@ -561,6 +563,8 @@ PluginModuleContentParent::~PluginModuleContentParent()
     Preferences::UnregisterCallback(TimeoutChanged, kContentTimeoutPref, this);
 }
 
+bool PluginModuleChromeParent::sInstantiated = false;
+
 PluginModuleChromeParent::PluginModuleChromeParent(const char* aFilePath, uint32_t aPluginId)
     : PluginModuleParent(true)
     , mSubprocess(new PluginProcessParent(aFilePath))
@@ -588,6 +592,7 @@ PluginModuleChromeParent::PluginModuleChromeParent(const char* aFilePath, uint32
     , mIsFlashPlugin(false)
 {
     NS_ASSERTION(mSubprocess, "Out of memory!");
+    sInstantiated = true;
 
     RegisterSettingsCallbacks();
 
@@ -1382,7 +1387,7 @@ NP_END_MACRO
 
 NPError
 PluginModuleParent::NPP_Destroy(NPP instance,
-                                NPSavedData** /*saved*/)
+                                NPSavedData** saved)
 {
     // FIXME/cjones:
     //  (1) send a "destroy" message to the child
@@ -1391,7 +1396,12 @@ PluginModuleParent::NPP_Destroy(NPP instance,
     //  (4) free parent
 
     PLUGIN_LOG_DEBUG_FUNCTION;
-    PluginInstanceParent* parentInstance = PluginInstanceParent::Cast(instance);
+    PluginAsyncSurrogate* surrogate = nullptr;
+    PluginInstanceParent* parentInstance =
+        PluginInstanceParent::Cast(instance, &surrogate);
+    if (surrogate && (!parentInstance || parentInstance->UseSurrogate())) {
+        return surrogate->NPP_Destroy(saved);
+    }
 
     if (!parentInstance)
         return NPERR_NO_ERROR;
@@ -1642,6 +1652,8 @@ PluginModuleParent::OnInitFailure()
         Close();
     }
 
+    mShutdown = true;
+
     if (mIsStartingAsync) {
         /* If we've failed then we need to enumerate any pending NPP_New calls
            and clean them up. */
@@ -1768,13 +1780,18 @@ PluginModuleParent::NP_Initialize(NPNetscapeFuncs* bFuncs, NPPluginFuncs* pFuncs
         return NS_ERROR_FAILURE;
     }
 
+    *error = NPERR_NO_ERROR;
     if (mIsStartingAsync) {
-        PluginAsyncSurrogate::NP_GetEntryPoints(pFuncs);
+        if (GetIPCChannel()->CanSend()) {
+            // We're already connected, so we may call this immediately.
+            RecvNP_InitializeResult(*error);
+        } else {
+            PluginAsyncSurrogate::NP_GetEntryPoints(pFuncs);
+        }
     } else {
         SetPluginFuncs(pFuncs);
     }
 
-    *error = NPERR_NO_ERROR;
     return NS_OK;
 }
 
@@ -1890,6 +1907,18 @@ PluginModuleParent::NP_Initialize(NPNetscapeFuncs* bFuncs, NPError* error)
     return NS_OK;
 }
 
+nsresult
+PluginModuleContentParent::NP_Initialize(NPNetscapeFuncs* bFuncs, NPError* error)
+{
+    PLUGIN_LOG_DEBUG_METHOD;
+    nsresult rv = PluginModuleParent::NP_Initialize(bFuncs, error);
+    if (mIsStartingAsync && GetIPCChannel()->CanSend()) {
+        // We're already connected, so we may call this immediately.
+        RecvNP_InitializeResult(*error);
+    }
+    return rv;
+}
+
 nsresult
 PluginModuleChromeParent::NP_Initialize(NPNetscapeFuncs* bFuncs, NPError* error)
 {

dom/plugins/ipc/PluginModuleParent.h

@@ -315,6 +315,10 @@ class PluginModuleContentParent : public PluginModuleParent
 
     virtual ~PluginModuleContentParent();
 
+#if defined(XP_WIN) || defined(XP_MACOSX)
+    nsresult NP_Initialize(NPNetscapeFuncs* bFuncs, NPError* error) MOZ_OVERRIDE;
+#endif
+
   private:
     virtual bool ShouldContinueFromReplyTimeout() MOZ_OVERRIDE;
     virtual void OnExitedSyncSend() MOZ_OVERRIDE;
@@ -342,6 +346,14 @@ class PluginModuleChromeParent
     static PluginLibrary* LoadModule(const char* aFilePath, uint32_t aPluginId,
                                      nsPluginTag* aPluginTag);
 
+    /**
+     * The following two functions are called by SetupBridge to determine
+     * whether an existing plugin module was reused, or whether a new module
+     * was instantiated by the plugin host.
+     */
+    static void ClearInstantiationFlag() { sInstantiated = false; }
+    static bool DidInstantiate() { return sInstantiated; }
+
     virtual ~PluginModuleChromeParent();
 
     void TerminateChildProcess(MessageLoop* aMsgLoop);
@@ -518,6 +530,7 @@ private:
     nsCOMPtr<nsIObserver> mOfflineObserver;
     bool mIsFlashPlugin;
     bool mIsBlocklisted;
+    static bool sInstantiated;
 };
 
 } // namespace plugins