Merge TLS1.3 Feature Branch into Main

ct/scanner/scanner.go

@@ -13,8 +13,20 @@ import (
 	"github.com/zmap/zcrypto/ct"
 	"github.com/zmap/zcrypto/ct/client"
 	"github.com/zmap/zcrypto/ct/x509"
+	"github.com/zmap/zcrypto/encoding/asn1"
+	"github.com/zmap/zcrypto/x509/pkix"
 )
 
+// ASN1Certificate holds the top-level asn1 fields in a certificate.
+//
+// It is used to determine if a certificate contains well-formed asn1 data or is corrupted.
+type ASN1Certificate struct {
+	Raw                asn1.RawContent
+	TBSCertificate     asn1.RawValue
+	SignatureAlgorithm pkix.AlgorithmIdentifier
+	SignatureValue     asn1.BitString
+}
+
 // Clients wishing to implement their own Matchers should implement this interface:
 type Matcher interface {
 	// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
@@ -137,6 +149,9 @@ type ScannerOptions struct {
 	Name string
 
 	MaximumIndex int64
+
+	// Always output encountered certificates, so long as they are valid ASN.1
+	IgnoreParsingErrors bool
 }
 
 // Creates a new ScannerOptions struct with sensible defaults
@@ -188,30 +203,87 @@ type fetchRange struct {
 	end   int64
 }
 
-// Takes the error returned by either x509.ParseCertificate() or
-// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
-// In the case of non-fatal errors, the error will be logged,
-// entriesWithNonFatalErrors will be incremented, and the return value will be
-// nil.
-// Fatal errors will be logged, unparsableEntires will be incremented, and the
+// parseCertificate takes a raw certificate, parses it, and if there is an error,
+// determines if it is fatal. In the case of non-fatal errors, the error will be logged,
+// entriesWithNonFatalErrors will be incremented, and the returned error will be nil.
+//
+// Fatal parse errors will be logged, unparsableEntries will be incremented, and the
 // fatal error itself will be returned.
-// When |err| is nil, this method does nothing.
-func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
+//
+// This function does NOT promise that the returned certificate will be non-nil
+// whenever err is non-nil.
+func (s *Scanner) parseCertificate(
+	precert bool,
+	entryType ct.LogEntryType,
+	index int64,
+	raw []byte,
+) (*x509.Certificate, error) {
+	var cert *x509.Certificate
+	var err error
+	if precert {
+		cert, err = x509.ParseTBSCertificate(raw)
+	} else {
+		cert, err = x509.ParseCertificate(raw)
+	}
+
 	if err == nil {
 		// No error to handle
-		return nil
+		return cert, nil
 	}
+
+	var isFatal bool
+
 	switch err.(type) {
 	case x509.NonFatalErrors:
 		s.entriesWithNonFatalErrors++
-		// We'll make a note, but continue.
-		s.logger.Warnf("Non-fatal error in %+v at index %d of log at %s: %s", entryType, index, s.logClient.Uri, err)
 	default:
 		s.unparsableEntries++
-		s.logger.Warnf("Failed to parse in %+v at index %d of log at %s: %s", entryType, index, s.logClient.Uri, err)
-		return err
+		isFatal = true
+	}
+
+	if !isFatal {
+		s.logger.Warnf(
+			"Ignored non-fatal error in %+v at index %d of log at %s: %s",
+			entryType,
+			index,
+			s.logClient.Uri,
+			err,
+		)
+		return cert, nil
+	}
+
+	if !s.opts.IgnoreParsingErrors {
+		s.logger.Errorf(
+			"Fatal parse error in %+v at index %d of log at %s: %s",
+			entryType,
+			index,
+			s.logClient.Uri,
+			err,
+		)
+		return nil, err
 	}
-	return nil
+
+	var asn1cert ASN1Certificate
+	if _, perr := asn1.Unmarshal(raw, &asn1cert); perr != nil {
+		s.logger.Warnf(
+			"Corrupted cert ASN.1 in %+v at index %d of log %s: %s",
+			entryType,
+			index,
+			s.logClient.Uri,
+			perr,
+		)
+		return nil, perr
+	}
+
+	s.logger.Warnf(
+		"Ignored fatal parse error in %+v at index %d of log %s: %s",
+		entryType,
+		index,
+		s.logClient.Uri,
+		err,
+	)
+
+	return cert, nil
 }
 
 // Processes the given |entry| in the specified log.
@@ -223,29 +295,52 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry, s
 			// Only interested in precerts and this is an X.509 cert, early-out.
 			return
 		}
-		cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
-		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
-			// We hit an unparseable entry, already logged inside handleParseEntryError()
+		cert, err := s.parseCertificate(
+			false,
+			entry.Leaf.TimestampedEntry.EntryType,
+			entry.Index,
+			entry.Leaf.TimestampedEntry.X509Entry,
+		)
+		if err != nil {
 			return
 		}
-		if s.opts.Matcher.CertificateMatches(cert) {
-			entry.X509Cert = cert
+		if cert != nil {
+			if s.opts.Matcher.CertificateMatches(cert) {
+				entry.RawCert = entry.Leaf.TimestampedEntry.X509Entry
+				entry.X509Cert = cert
+				foundCert(&entry, s.opts.Name)
+			}
+		} else {
+			entry.RawCert = entry.Leaf.TimestampedEntry.X509Entry
 			foundCert(&entry, s.opts.Name)
 		}
 	case ct.PrecertLogEntryType:
-		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
-		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
-			// We hit an unparseable entry, already logged inside handleParseEntryError()
+		cert, err := s.parseCertificate(
+			true,
+			entry.Leaf.TimestampedEntry.EntryType,
+			entry.Index,
+			entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate,
+		)
+		if err != nil {
 			return
 		}
 		precert := &ct.Precertificate{
 			Raw:            entry.Chain[0],
-			TBSCertificate: *c,
+			TBSCertificate: cert,
 			IssuerKeyHash:  entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
-		if s.opts.Matcher.PrecertificateMatches(precert) {
-			entry.Precert = precert
+
+		entry.IsPrecert = true
+		entry.RawCert = entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate
+		entry.Precert = precert
+
+		if cert != nil {
+			if s.opts.Matcher.PrecertificateMatches(precert) {
+				foundPrecert(&entry, s.opts.Name)
+			}
+		} else {
 			foundPrecert(&entry, s.opts.Name)
 		}
+
 		s.precertsSeen++
 	}
 }

ct/types.go

@@ -237,12 +237,16 @@ func (d *DigitallySigned) UnmarshalJSON(b []byte) error {
 
 // LogEntry represents the contents of an entry in a CT log, see section 3.1.
 type LogEntry struct {
-	Index    int64
-	Leaf     MerkleTreeLeaf
+	Index     int64
+	Leaf      MerkleTreeLeaf
+	RawCert   []byte
+	IsPrecert bool
+	Chain     []ASN1Cert
+	Server    string
+	// When the raw cert is parseable, and not a precert,
+	// X509Cert contains the parsed certificate for convenience.
 	X509Cert *x509.Certificate
 	Precert  *Precertificate
-	Chain    []ASN1Cert
-	Server   string
 }
 
 // SHA256Hash represents the output from the SHA256 hash function.
@@ -348,8 +352,8 @@ type Precertificate struct {
 	// SHA256 hash of the issuing key
 	IssuerKeyHash [issuerKeyHashLength]byte
 	// Parsed TBSCertificate structure (held in an x509.Certificate for ease of
-	// access.
-	TBSCertificate x509.Certificate
+	// access), when possible.
+	TBSCertificate *x509.Certificate
 }
 
 // X509Certificate returns the X.509 Certificate contained within the

ct/x509/sec1.go

@@ -7,6 +7,7 @@ package x509
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
+
 	// START CT CHANGES
 	"github.com/zmap/zcrypto/ct/asn1"
 	// START CT CHANGES

data/test/certificates/fpki.go

@@ -119,6 +119,31 @@ fOs/QbP1b0s6Xq5vk3aY0vGZnUXEjnI=
 -----END CERTIFICATE-----
 `
 
+// CorruptCertificate is a mangled certificate which is not valid ASN.1
+const CorruptCertificate = `
+-----BEGIN CERTIFICATE-----
+MIFDczCCAlugAwIBAgIBATANBgkqhkiG9w0BAQsFADBbMQswCQYDVQQGEwJVUzEY
+MBYGA1UEChMPVS5TLiBHb3Zlcm5tZW50MQwwCgYDVQQLEwNEb0QxDDAKBgNVBAsT
+A1BLSTEWMBQGA1UEAxMNRG9EIFJvb3QgQ0EgMzAeFw0xMjAzMjAxODQ2NDFaFw0y
+OTEyMzAxODQ2NDFaMFsxCzAJBgNVBAYTAlVTMRgwFgYDVQQKEw9VLlMuIEdvdmVy
+bm1lbnQFFFFFFFNVBAsTA0RvRDEMMAoGA1UECxMDUEtJMRYwFAYDVQQDEw1Eb0Qg
+Um9vdCBDQSAzMIIBIjANBgQQQQQQQQQQQQQQAAOCAQ8AMIIBCgKCAQEAqewUcoro
+S3Cj2hADhKb7pzYNKjpSFr8wFVKGBUcgz6qmzXXEZG7v8WAjywpmQK60yGgqAFFo
+STfpWTJNlbxDJ+lAjToQzhS8Qxih+d7M54V2c14YGiNbvT8f8u2NGcwD0UCkj6cg
+AkwnWnk29qM3IY4AWgYWytNVlm8xKbtyDsviSFHy1DekNdZv7hezsQarCxmG6CNt
+MRsoeGXF3mJSvMF96+6gIHOPETHISWORKSGCTPC/unRAOwwERYBnXMXrolfDGn8K
+Lb1/udzBmbDIB+QMhjaUOiUv8n3mlzwblLSXWQbJOuQL2erp/DtzNG/955jk86HC
+kF8c9T8u1xnTfwIDAQABo0IwQDAdBgNVHQ4EFgQUbIqUonexgHIdgXoWqvLczmbu
+RcAwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAJ9xpMC2ltKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASDFLiH+H
+9Jj1qMYJyR/wLB/sgrj0pUc4wTMr30x+mr4LC7HLD3xQKBDPio2i6bqshtfUsZNf
+Io+WBbRODHWRfdPy55TClBR2T48MqxCHWDKFB3WGEgte6lO0CshMhJIf6+hBhjy6
+9E5BStFsWEdBw4Za8u7p8pgnguouNtb4Bl6C8aBSk0QJutKpGVpYo6hdIG1PZPgw
+hxuQE0iBzcqQxw3B1Jg/jvIOV2gzEo6ZCbHw5PYQ9DbySb3qozjIVkEjg5rfoRs1
+fOs/QbP1b0s6Xq5vk3aY0vGZnUXEjnI=
+-----END CERTIFICATE-----
+`
+
 // HexHashDoDRootCA3SignedBySelf is the hex SHA256 fingerprint of
 // DoDRootCA3SignedBySelf.
 const HexHashDoDRootCA3SignedBySelf = "b107b33f453e5510f68e513110c6f6944bacc263df0137f821c1b3c2f8f863d2"

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.10.0
 	github.com/weppos/publicsuffix-go v0.40.3-0.20250127173806-e489a31678ca
-	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248
 	github.com/zmap/zcertificate v0.0.1
 	golang.org/x/crypto v0.32.0
 	golang.org/x/net v0.34.0

go.sum

@@ -37,8 +37,6 @@ github.com/weppos/publicsuffix-go v0.40.3-0.20250127173806-e489a31678ca h1:Cno+V
 github.com/weppos/publicsuffix-go v0.40.3-0.20250127173806-e489a31678ca/go.mod h1:43Dfyxu2dpmLg56at26Q4k9gwf3yWSUiwk8kGnwzULk=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
-github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
-github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
 github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
 github.com/zmap/zcertificate v0.0.1 h1:2X15TRx4Fr6qzKItfwUdww294OeRSmHILLa+Xn2Uv+s=
 github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk=

internal/cfg/cfg.go

@@ -0,0 +1,65 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cfg holds configuration shared by the Go command and internal/testenv.
+// Definitions that don't need to be exposed outside of cmd/go should be in
+// cmd/go/internal/cfg instead of this package.
+package cfg
+
+// KnownEnv is a list of environment variables that affect the operation
+// of the Go command.
+const KnownEnv = `
+	AR
+	CC
+	CGO_CFLAGS
+	CGO_CFLAGS_ALLOW
+	CGO_CFLAGS_DISALLOW
+	CGO_CPPFLAGS
+	CGO_CPPFLAGS_ALLOW
+	CGO_CPPFLAGS_DISALLOW
+	CGO_CXXFLAGS
+	CGO_CXXFLAGS_ALLOW
+	CGO_CXXFLAGS_DISALLOW
+	CGO_ENABLED
+	CGO_FFLAGS
+	CGO_FFLAGS_ALLOW
+	CGO_FFLAGS_DISALLOW
+	CGO_LDFLAGS
+	CGO_LDFLAGS_ALLOW
+	CGO_LDFLAGS_DISALLOW
+	CXX
+	FC
+	GCCGO
+	GO111MODULE
+	GO386
+	GOARCH
+	GOARM
+	GOBIN
+	GOCACHE
+	GOENV
+	GOEXE
+	GOFLAGS
+	GOGCCFLAGS
+	GOHOSTARCH
+	GOHOSTOS
+	GOINSECURE
+	GOMIPS
+	GOMIPS64
+	GOMODCACHE
+	GONOPROXY
+	GONOSUMDB
+	GOOS
+	GOPATH
+	GOPPC64
+	GOPRIVATE
+	GOPROXY
+	GOROOT
+	GOSUMDB
+	GOTMPDIR
+	GOTOOLDIR
+	GOVCS
+	GOWASM
+	GO_EXTLINK_ENABLED
+	PKG_CONFIG
+`