Reland Bug 1063281, Part 9: Switch Gecko from NSS to CheckCertHostname, r=keeler

--HG--
extra : rebase_source : 3a5e3bc2e113035e9c88b571bac68f3dbe2c8f04

config/external/nss/nss.def

@@ -146,7 +146,6 @@ CERT_StartCertificateRequestAttributes
 CERT_SubjectPublicKeyInfoTemplate DATA
 CERT_TimeChoiceTemplate DATA
 CERT_VerifyCertificate
-CERT_VerifyCertName
 CERT_VerifySignedDataWithPublicKeyInfo
 DER_AsciiToTime_Util
 DER_DecodeTimeChoice_Util

security/certverifier/CertVerifier.cpp

@@ -438,7 +438,7 @@ CertVerifier::VerifySSLServerCert(CERTCertificate* peerCert,
   }
 
   ScopedCERTCertList builtChainTemp;
-  // CreateCertErrorRunnable assumes that CERT_VerifyCertName is only called
+  // CreateCertErrorRunnable assumes that CheckCertHostname is only called
   // if VerifyCert succeeded.
   SECStatus rv = VerifyCert(peerCert, certificateUsageSSLServer, time, pinarg,
                             hostname, flags, stapledOCSPResponse,
@@ -447,9 +447,23 @@ CertVerifier::VerifySSLServerCert(CERTCertificate* peerCert,
     return rv;
   }
 
-  rv = CERT_VerifyCertName(peerCert, hostname);
-  if (rv != SECSuccess) {
-    return rv;
+  Input peerCertInput;
+  Result result = peerCertInput.Init(peerCert->derCert.data,
+                                     peerCert->derCert.len);
+  if (result != Success) {
+    PR_SetError(MapResultToPRErrorCode(result), 0);
+    return SECFailure;
+  }
+  Input hostnameInput;
+  result = hostnameInput.Init(uint8_t_ptr_cast(hostname), strlen(hostname));
+  if (result != Success) {
+    PR_SetError(SEC_ERROR_INVALID_ARGS, 0);
+    return SECFailure;
+  }
+  result = CheckCertHostname(peerCertInput, hostnameInput);
+  if (result != Success) {
+    PR_SetError(MapResultToPRErrorCode(result), 0);
+    return SECFailure;
   }
 
   if (saveIntermediatesInPermanentDatabase) {

security/manager/ssl/src/SSLServerCertVerification.cpp

@@ -96,7 +96,7 @@
 
 #include <cstring>
 
-#include "pkix/pkixtypes.h"
+#include "pkix/pkix.h"
 #include "pkix/pkixnss.h"
 #include "pkix/ScopedPtr.h"
 #include "CertVerifier.h"
@@ -328,7 +328,7 @@ DetermineCertOverrideErrors(CERTCertificate* cert, const char* hostName,
   MOZ_ASSERT(errorCodeExpired == 0);
 
   // Assumes the error prioritization described in mozilla::pkix's
-  // BuildForward function. Also assumes that CERT_VerifyCertName was only
+  // BuildForward function. Also assumes that CheckCertHostname was only
   // called if CertVerifier::VerifyCert succeeded.
   switch (defaultErrorCodeToReport) {
     case SEC_ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED:
@@ -373,14 +373,25 @@ DetermineCertOverrideErrors(CERTCertificate* cert, const char* hostName,
   }
 
   if (defaultErrorCodeToReport != SSL_ERROR_BAD_CERT_DOMAIN) {
-    if (CERT_VerifyCertName(cert, hostName) != SECSuccess) {
-      if (PR_GetError() != SSL_ERROR_BAD_CERT_DOMAIN) {
-        PR_SetError(defaultErrorCodeToReport, 0);
-        return SECFailure;
-      }
-
+    Input certInput;
+    if (certInput.Init(cert->derCert.data, cert->derCert.len) != Success) {
+      PR_SetError(SEC_ERROR_BAD_DER, 0);
+      return SECFailure;
+    }
+    Input hostnameInput;
+    Result result = hostnameInput.Init(uint8_t_ptr_cast(hostName),
+                                       strlen(hostName));
+    if (result != Success) {
+      PR_SetError(SEC_ERROR_INVALID_ARGS, 0);
+      return SECFailure;
+    }
+    result = CheckCertHostname(certInput, hostnameInput);
+    if (result == Result::ERROR_BAD_CERT_DOMAIN) {
       collectedErrors |= nsICertOverrideService::ERROR_MISMATCH;
       errorCodeMismatch = SSL_ERROR_BAD_CERT_DOMAIN;
+    } else if (result != Success) {
+      PR_SetError(defaultErrorCodeToReport, 0);
+      return SECFailure;
     }
   }
 

security/nss/coreconf/coreconf.dep

@@ -10,4 +10,3 @@
  */
 
 #error "Do not include this header file."
-