[release-branch.go1.6] crypto/x509: read Darwin trust settings for root CAs

Darwin separately stores bits indicating whether a root certificate
should be trusted; this changes Go to read and use those when
initializing SystemCertPool.

Unfortunately, the trust API is very slow. To avoid a delay of up to
0.5s in initializing the system cert pool, we assume that
the trust settings found in kSecTrustSettingsDomainSystem will always
indicate trust. (That is, all root certs Apple distributes are trusted.)
This is not guaranteed by the API but is true in practice.

In the non-cgo codepath, we do not have that benefit, so we must check
the trust status of every certificate. This causes about 0.5s of delay
in initializing the SystemCertPool.

On OS X 10.11 and older, the "security" command requires a certificate
to be provided in a file and not on stdin, so the non-cgo codepath
creates temporary files for each certificate, further slowing initialization.

Updates #18141.

Change-Id: If681c514047afe5e1a68de6c9d40ceabbce54755
Reviewed-on: https://go-review.googlesource.com/33721
Run-TryBot: Quentin Smith <quentin@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>
Reviewed-on: https://go-review.googlesource.com/33728

4 files changed, 296 insertions, 18 deletions

diff --git a/src/crypto/x509/cert_pool.go b/src/crypto/x509/cert_pool.go
index 2362e84688..dc88ad45a0 100644
--- a/src/crypto/x509/cert_pool.go
+++ b/src/crypto/x509/cert_pool.go
@@ -52,6 +52,21 @@ func (s *CertPool) findVerifiedParents(cert *Certificate) (parents []int, errCer
 	return
 }
 
+func (s *CertPool) contains(cert *Certificate) bool {
+	if s == nil {
+		return false
+	}
+
+	candidates := s.byName[string(cert.RawSubject)]
+	for _, c := range candidates {
+		if s.certs[c].Equal(cert) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // AddCert adds a certificate to a pool.
 func (s *CertPool) AddCert(cert *Certificate) {
 	if cert == nil {
diff --git a/src/crypto/x509/root_cgo_darwin.go b/src/crypto/x509/root_cgo_darwin.go
index bf4a5cdfee..ee94dadcde 100644
--- a/src/crypto/x509/root_cgo_darwin.go
+++ b/src/crypto/x509/root_cgo_darwin.go
@@ -7,30 +7,28 @@
 package x509
 
 /*
-#cgo CFLAGS: -mmacosx-version-min=10.6 -D__MAC_OS_X_VERSION_MAX_ALLOWED=1060
+#cgo CFLAGS: -mmacosx-version-min=10.6 -D__MAC_OS_X_VERSION_MAX_ALLOWED=1080
 #cgo LDFLAGS: -framework CoreFoundation -framework Security
 
+#include <errno.h>
+#include <sys/sysctl.h>
+
 #include <CoreFoundation/CoreFoundation.h>
 #include <Security/Security.h>
 
-// FetchPEMRoots fetches the system's list of trusted X.509 root certificates.
-//
-// On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root
-// certificates of the system. On failure, the function returns -1.
-//
-// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after
-// we've consumed its content.
-int FetchPEMRoots(CFDataRef *pemRoots) {
+// FetchPEMRoots_MountainLion is the version of FetchPEMRoots from Go 1.6
+// which still works on OS X 10.8 (Mountain Lion).
+// It lacks support for admin & user cert domains.
+// See golang.org/issue/16473
+int FetchPEMRoots_MountainLion(CFDataRef *pemRoots) {
 	if (pemRoots == NULL) {
 		return -1;
 	}
-
 	CFArrayRef certs = NULL;
 	OSStatus err = SecTrustCopyAnchorCertificates(&certs);
 	if (err != noErr) {
 		return -1;
 	}
-
 	CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
 	int i, ncerts = CFArrayGetCount(certs);
 	for (i = 0; i < ncerts; i++) {
@@ -39,7 +37,6 @@ int FetchPEMRoots(CFDataRef *pemRoots) {
 		if (cert == NULL) {
 			continue;
 		}
-
 		// Note: SecKeychainItemExport is deprecated as of 10.7 in favor of SecItemExport.
 		// Once we support weak imports via cgo we should prefer that, and fall back to this
 		// for older systems.
@@ -47,16 +44,157 @@ int FetchPEMRoots(CFDataRef *pemRoots) {
 		if (err != noErr) {
 			continue;
 		}
-
 		if (data != NULL) {
 			CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data));
 			CFRelease(data);
 		}
 	}
-
 	CFRelease(certs);
+	*pemRoots = combinedData;
+	return 0;
+}
+
+// useOldCode reports whether the running machine is OS X 10.8 Mountain Lion
+// or older. We only support Mountain Lion and higher, but we'll at least try our
+// best on older machines and continue to use the old code path.
+//
+// See golang.org/issue/16473
+int useOldCode() {
+	char str[256];
+	size_t size = sizeof(str);
+	memset(str, 0, size);
+	sysctlbyname("kern.osrelease", str, &size, NULL, 0);
+	// OS X 10.8 is osrelease "12.*", 10.7 is 11.*, 10.6 is 10.*.
+	// We never supported things before that.
+	return memcmp(str, "12.", 3) == 0 || memcmp(str, "11.", 3) == 0 || memcmp(str, "10.", 3) == 0;
+}
 
+// FetchPEMRoots fetches the system's list of trusted X.509 root certificates.
+//
+// On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root
+// certificates of the system. On failure, the function returns -1.
+// Additionally, it fills untrustedPemRoots with certs that must be removed from pemRoots.
+//
+// Note: The CFDataRef returned in pemRoots and untrustedPemRoots must
+// be released (using CFRelease) after we've consumed its content.
+int FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots) {
+	if (useOldCode()) {
+		return FetchPEMRoots_MountainLion(pemRoots);
+	}
+
+	// Get certificates from all domains, not just System, this lets
+	// the user add CAs to their "login" keychain, and Admins to add
+	// to the "System" keychain
+	SecTrustSettingsDomain domains[] = { kSecTrustSettingsDomainSystem,
+					     kSecTrustSettingsDomainAdmin,
+					     kSecTrustSettingsDomainUser };
+
+	int numDomains = sizeof(domains)/sizeof(SecTrustSettingsDomain);
+	if (pemRoots == NULL) {
+		return -1;
+	}
+
+	// kSecTrustSettingsResult is defined as CFSTR("kSecTrustSettingsResult"),
+	// but the Go linker's internal linking mode can't handle CFSTR relocations.
+	// Create our own dynamic string instead and release it below.
+	CFStringRef policy = CFStringCreateWithCString(NULL, "kSecTrustSettingsResult", kCFStringEncodingUTF8);
+
+	CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
+	CFMutableDataRef combinedUntrustedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
+	for (int i = 0; i < numDomains; i++) {
+		CFArrayRef certs = NULL;
+		OSStatus err = SecTrustSettingsCopyCertificates(domains[i], &certs);
+		if (err != noErr) {
+			continue;
+		}
+
+		CFIndex numCerts = CFArrayGetCount(certs);
+		for (int j = 0; j < numCerts; j++) {
+			CFDataRef data = NULL;
+			CFErrorRef errRef = NULL;
+			CFArrayRef trustSettings = NULL;
+			SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, j);
+			if (cert == NULL) {
+				continue;
+			}
+			// We only want trusted certs.
+			int untrusted = 0;
+			if (i != 0) {
+				// Certs found in the system domain are always trusted. If the user
+				// configures "Never Trust" on such a cert, it will also be found in the
+				// admin or user domain, causing it to be added to untrustedPemRoots. The
+				// Go code will then clean this up.
+
+				// Trust may be stored in any of the domains. According to Apple's
+				// SecTrustServer.c, "user trust settings overrule admin trust settings",
+				// so take the last trust settings array we find.
+				// Skip the system domain since it is always trusted.
+				for (int k = 1; k < numDomains; k++) {
+					CFArrayRef domainTrustSettings = NULL;
+					err = SecTrustSettingsCopyTrustSettings(cert, domains[k], &domainTrustSettings);
+					if (err == errSecSuccess && domainTrustSettings != NULL) {
+						if (trustSettings) {
+							CFRelease(trustSettings);
+						}
+						trustSettings = domainTrustSettings;
+					}
+				}
+				if (trustSettings == NULL) {
+					// "this certificate must be verified to a known trusted certificate"; aka not a root.
+					continue;
+				}
+				for (CFIndex k = 0; k < CFArrayGetCount(trustSettings); k++) {
+					CFNumberRef cfNum;
+					CFDictionaryRef tSetting = (CFDictionaryRef)CFArrayGetValueAtIndex(trustSettings, k);
+					if (CFDictionaryGetValueIfPresent(tSetting, policy, (const void**)&cfNum)){
+						SInt32 result = 0;
+						CFNumberGetValue(cfNum, kCFNumberSInt32Type, &result);
+						// TODO: The rest of the dictionary specifies conditions for evaluation.
+						if (result == kSecTrustSettingsResultDeny) {
+							untrusted = 1;
+						}
+					}
+				}
+				CFRelease(trustSettings);
+			}
+			// We only want to add Root CAs, so make sure Subject and Issuer Name match
+			CFDataRef subjectName = SecCertificateCopyNormalizedSubjectContent(cert, &errRef);
+			if (errRef != NULL) {
+				CFRelease(errRef);
+				continue;
+			}
+			CFDataRef issuerName = SecCertificateCopyNormalizedIssuerContent(cert, &errRef);
+			if (errRef != NULL) {
+				CFRelease(subjectName);
+				CFRelease(errRef);
+				continue;
+			}
+			Boolean equal = CFEqual(subjectName, issuerName);
+			CFRelease(subjectName);
+			CFRelease(issuerName);
+			if (!equal) {
+				continue;
+			}
+
+			// Note: SecKeychainItemExport is deprecated as of 10.7 in favor of SecItemExport.
+			// Once we support weak imports via cgo we should prefer that, and fall back to this
+			// for older systems.
+			err = SecKeychainItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data);
+			if (err != noErr) {
+				continue;
+			}
+
+			if (data != NULL) {
+				CFMutableDataRef appendTo = untrusted ? combinedUntrustedData : combinedData;
+				CFDataAppendBytes(appendTo, CFDataGetBytePtr(data), CFDataGetLength(data));
+				CFRelease(data);
+			}
+		}
+		CFRelease(certs);
+	}
+	CFRelease(policy);
 	*pemRoots = combinedData;
+	*untrustedPemRoots = combinedUntrustedData;
 	return 0;
 }
 */
@@ -67,7 +205,8 @@ func initSystemRoots() {
 	roots := NewCertPool()
 
 	var data C.CFDataRef = nil
-	err := C.FetchPEMRoots(&data)
+	var untrustedData C.CFDataRef = nil
+	err := C.FetchPEMRoots(&data, &untrustedData)
 	if err == -1 {
 		return
 	}
@@ -75,5 +214,20 @@ func initSystemRoots() {
 	defer C.CFRelease(C.CFTypeRef(data))
 	buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data)))
 	roots.AppendCertsFromPEM(buf)
+	if untrustedData == nil {
+		systemRoots = roots
+		return
+	}
+	defer C.CFRelease(C.CFTypeRef(untrustedData))
+	buf = C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(untrustedData)), C.int(C.CFDataGetLength(untrustedData)))
+	untrustedRoots := NewCertPool()
+	untrustedRoots.AppendCertsFromPEM(buf)
+
+	trustedRoots := NewCertPool()
+	for _, c := range roots.certs {
+		if !untrustedRoots.contains(c) {
+			trustedRoots.AddCert(c)
+		}
+	}
 	systemRoots = roots
 }
diff --git a/src/crypto/x509/root_darwin.go b/src/crypto/x509/root_darwin.go
index 78de56c221..59b303d64f 100644
--- a/src/crypto/x509/root_darwin.go
+++ b/src/crypto/x509/root_darwin.go
@@ -6,12 +6,27 @@
 
 package x509
 
-import "os/exec"
+import (
+	"bytes"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"sync"
+	"syscall"
+)
 
 func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
 	return nil, nil
 }
 
+// This code is only used when compiling without cgo.
+// It is here, instead of root_nocgo_darwin.go, so that tests can check it
+// even if the tests are run with cgo enabled.
+// The linker will not include these unused functions in binaries built with cgo enabled.
+
 func execSecurityRoots() (*CertPool, error) {
 	cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain")
 	data, err := cmd.Output()
@@ -19,7 +34,100 @@ func execSecurityRoots() (*CertPool, error) {
 		return nil, err
 	}
 
-	roots := NewCertPool()
-	roots.AppendCertsFromPEM(data)
+	var (
+		mu    sync.Mutex
+		roots = NewCertPool()
+	)
+	add := func(cert *Certificate) {
+		mu.Lock()
+		defer mu.Unlock()
+		roots.AddCert(cert)
+	}
+	blockCh := make(chan *pem.Block)
+	var wg sync.WaitGroup
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for block := range blockCh {
+				verifyCertWithSystem(block, add)
+			}
+		}()
+	}
+	for len(data) > 0 {
+		var block *pem.Block
+		block, data = pem.Decode(data)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+			continue
+		}
+		blockCh <- block
+	}
+	close(blockCh)
+	wg.Wait()
 	return roots, nil
 }
+
+func verifyCertWithSystem(block *pem.Block, add func(*Certificate)) {
+	data := pem.EncodeToMemory(block)
+	var cmd *exec.Cmd
+	if needsTmpFiles() {
+		f, err := ioutil.TempFile("", "cert")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "can't create temporary file for cert: %v", err)
+			return
+		}
+		defer os.Remove(f.Name())
+		if _, err := f.Write(data); err != nil {
+			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+			return
+		}
+		if err := f.Close(); err != nil {
+			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+			return
+		}
+		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", f.Name(), "-l")
+	} else {
+		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", "/dev/stdin", "-l")
+		cmd.Stdin = bytes.NewReader(data)
+	}
+	if cmd.Run() == nil {
+		// Non-zero exit means untrusted
+		cert, err := ParseCertificate(block.Bytes)
+		if err != nil {
+			return
+		}
+
+		add(cert)
+	}
+}
+
+var versionCache struct {
+	sync.Once
+	major int
+}
+
+// needsTmpFiles reports whether the OS is <= 10.11 (which requires real
+// files as arguments to the security command).
+func needsTmpFiles() bool {
+	versionCache.Do(func() {
+		release, err := syscall.Sysctl("kern.osrelease")
+		if err != nil {
+			return
+		}
+		for i, c := range release {
+			if c == '.' {
+				release = release[:i]
+				break
+			}
+		}
+		major, err := strconv.Atoi(release)
+		if err != nil {
+			return
+		}
+		versionCache.major = major
+	})
+	return versionCache.major <= 15
+}
diff --git a/src/crypto/x509/root_darwin_test.go b/src/crypto/x509/root_darwin_test.go
index cc6d23c505..38f314b87b 100644
--- a/src/crypto/x509/root_darwin_test.go
+++ b/src/crypto/x509/root_darwin_test.go
@@ -29,6 +29,7 @@ func TestSystemRoots(t *testing.T) {
 		// On Mavericks, there are 212 bundled certs; require only
 		// 150 here, since this is just a sanity check, and the
 		// exact number will vary over time.
+		t.Logf("got %d roots", len(tt.certs))
 		if want, have := 150, len(tt.certs); have < want {
 			t.Fatalf("want at least %d system roots, have %d", want, have)
 		}