Prevent deadlocks when sending sync messages during urgent in-calls (bug 900062, r=cjones).

ipc/glue/SyncChannel.cpp

@@ -102,7 +102,13 @@ SyncChannel::ProcessUrgentMessages()
 bool
 SyncChannel::Send(Message* _msg, Message* reply)
 {
-    MOZ_ASSERT(!mPendingReply);
+    if (mPendingReply) {
+        // This is a temporary hack in place, for e10s CPOWs, until bug 901789
+        // and the new followup RPC protocol land. Eventually this will become
+        // an assert again. See bug 900062 for details.
+        NS_ERROR("Nested sync messages are not supported");
+        return false;
+    }
 
     nsAutoPtr<Message> msg(_msg);
 

ipc/ipdl/test/cxx/PTestUrgency.ipdl

@@ -7,11 +7,14 @@ parent:
     sync Test1() returns (uint32_t result);
     async Test2();
     sync Test3() returns (uint32_t result);
+    sync Test4_Begin();
+    sync Test4_NestedSync();
 
 child:
     async Start();
     urgent Reply1() returns (uint32_t result);
     urgent Reply2() returns (uint32_t result);
+    urgent Test4_Reenter();
 };
 
 } // namespace _ipdltest

ipc/ipdl/test/cxx/TestUrgency.cpp

@@ -59,6 +59,21 @@ TestUrgencyParent::RecvTest3(uint32_t *value)
   return true;
 }
 
+bool
+TestUrgencyParent::RecvTest4_Begin()
+{
+  if (!CallTest4_Reenter())
+    fail("call Test4_Reenter");
+  return true;
+}
+
+bool
+TestUrgencyParent::RecvTest4_NestedSync()
+{
+  fail("nested sync not supported");
+  return false;
+}
+
 //-----------------------------------------------------------------------------
 // child
 
@@ -97,6 +112,9 @@ TestUrgencyChild::RecvStart()
   if (result != 1000)
     fail("wrong value from test3");
 
+  if (!SendTest4_Begin())
+    fail("calling SendTest4_Begin");
+
   Close();
 
   return true;
@@ -127,6 +145,14 @@ TestUrgencyChild::AnswerReply2(uint32_t *reply)
   return true;
 }
 
+bool
+TestUrgencyChild::AnswerTest4_Reenter()
+{
+  if (SendTest4_NestedSync())
+    fail("sending nested sync messages not supported");
+  return true;
+}
+
 TestUrgencyChild::TestUrgencyChild()
   : test_(0)
 {

ipc/ipdl/test/cxx/TestUrgency.h

@@ -25,6 +25,8 @@ public:
     bool RecvTest1(uint32_t *value);
     bool RecvTest2();
     bool RecvTest3(uint32_t *value);
+    bool RecvTest4_Begin();
+    bool RecvTest4_NestedSync();
 
     virtual void ActorDestroy(ActorDestroyReason why) MOZ_OVERRIDE
     {
@@ -48,6 +50,7 @@ public:
     bool RecvStart();
     bool AnswerReply1(uint32_t *reply);
     bool AnswerReply2(uint32_t *reply);
+    bool AnswerTest4_Reenter();
 
     virtual void ActorDestroy(ActorDestroyReason why) MOZ_OVERRIDE
     {