Files
tailscale/prober/tls_test.go
T
Thomas Desrosiers 7add2af9ec prober: cache CRLs across TLS probes (#21286)
The TLS probe fetched and parsed the leaf certificate's CRL on every
run. That is fine when the CRL is small, but some CAs publish CRLs of
several megabytes: the one for the AWS ACM R2M04 intermediate is about
2.5MB, which at the default 15s interval is a continuous 170kB/s per
probed node.

Cache parsed CRLs by distribution point URL and reuse each for up to
an hour, or until its NextUpdate if that comes first. An hour is the
HTTP max-age Let's Encrypt serves on its root CRL. A CRL is cached
only after its signature verifies, every use still re-verifies it
against the probing leaf's issuer (the cache is keyed by URL alone),
and a CRL without a NextUpdate is never cached since it declares no
validity window.

Concurrent probes fetch through singleflight.DoChanContext to avoid
re-fetching a single CRL, with each waiter keeping its own deadline. A
caller that missed the cache re-checks it inside the singleflight
closure, since singleflight dedupes only calls that overlap.

Leaf certificates whose issuer is missing from the presented chain now
fail before any fetch. Previously the probe downloaded the CRL and then
panicked in CheckSignatureFrom, which the prober recovered and recorded
as a probe failure.

Also update the TLS probe's doc comments, which said OCSP where the
code checks a CRL.

Fixes #21310

Signed-off-by: Thomas Desrosiers <git@hive.pw>
2026-09-16 00:08:19 -04:00

538 lines
15 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package prober
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"tailscale.com/tstest"
)
var leafCert = x509.Certificate{
SerialNumber: big.NewInt(10001),
Subject: pkix.Name{CommonName: "tlsprobe.test"},
SignatureAlgorithm: x509.SHA256WithRSA,
PublicKeyAlgorithm: x509.RSA,
Version: 3,
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
NotBefore: time.Now().Add(-5 * time.Minute),
NotAfter: time.Now().Add(60 * 24 * time.Hour),
SubjectKeyId: []byte{1, 2, 3},
AuthorityKeyId: []byte{1, 2, 3, 4, 5}, // issuerCert below
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
var issuerCertTpl = x509.Certificate{
SerialNumber: big.NewInt(10002),
Subject: pkix.Name{CommonName: "tlsprobe.ca.test"},
SignatureAlgorithm: x509.SHA256WithRSA,
PublicKeyAlgorithm: x509.RSA,
Version: 3,
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
NotBefore: time.Now().Add(-5 * time.Minute),
NotAfter: time.Now().Add(60 * 24 * time.Hour),
SubjectKeyId: []byte{1, 2, 3, 4, 5},
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
func simpleCert() (tls.Certificate, error) {
certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return tls.Certificate{}, err
}
certPrivKeyPEM := new(bytes.Buffer)
pem.Encode(certPrivKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
})
certBytes, err := x509.CreateCertificate(rand.Reader, &leafCert, &leafCert, &certPrivKey.PublicKey, certPrivKey)
if err != nil {
return tls.Certificate{}, err
}
certPEM := new(bytes.Buffer)
pem.Encode(certPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
return tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes())
}
func TestTLSConnection(t *testing.T) {
crt, err := simpleCert()
if err != nil {
t.Fatal(err)
}
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
srv.TLS = &tls.Config{Certificates: []tls.Certificate{crt}}
srv.StartTLS()
defer srv.Close()
err = probeTLS(context.Background(), &tls.Config{ServerName: "fail.example.com"}, srv.Listener.Addr().String())
// The specific error message here is platform-specific ("certificate is not trusted"
// on macOS and "certificate signed by unknown authority" on Linux), so only check
// that it contains the word 'certificate'.
if err == nil || !strings.Contains(err.Error(), "certificate") {
t.Errorf("unexpected error: %q", err)
}
}
func TestCertExpiration(t *testing.T) {
for _, tt := range []struct {
name string
cert func() *x509.Certificate
wantErr string
}{
{
"cert not valid yet",
func() *x509.Certificate {
c := leafCert
c.NotBefore = time.Now().Add(time.Hour)
return &c
},
"one of the certs has NotBefore in the future",
},
{
"cert expiring soon",
func() *x509.Certificate {
c := leafCert
c.NotAfter = time.Now().Add(time.Hour)
return &c
},
"one of the certs expires in",
},
} {
t.Run(tt.name, func(t *testing.T) {
cs := &tls.ConnectionState{PeerCertificates: []*x509.Certificate{tt.cert()}}
err := validateConnState(context.Background(), &crlCache{now: time.Now}, cs)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Errorf("unexpected error %q; want %q", err, tt.wantErr)
}
})
}
}
type CRLServer struct {
crlBytes []byte
requests atomic.Int32 // total requests served
}
func (s *CRLServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.requests.Add(1)
if s.crlBytes == nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/pkix-crl")
w.WriteHeader(http.StatusOK)
w.Write(s.crlBytes)
}
// someECDSAKey{1,2,3} are different EC private keys in PEM format
// as generated by:
//
// openssl ecparam -name prime256v1 -genkey -noout -out -
//
// They're used in tests to avoid burning CPU at test time to just
// to make some arbitrary test keys.
const (
someECDSAKey1 = `
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIDKggO47Si0/JgqF0q9m0HfQ92lbERWsBaKS5YihtuheoAoGCCqGSM49
AwEHoUQDQgAE/JtNZkfFmAGQJHW5Xgz0Eoyi9MKVxl77sXjIFDMX233QDIWPEM/B
vmNMvdFkuYBjwbq6H+SNf1NXRNladEGU/Q==
-----END EC PRIVATE KEY-----
`
someECDSAKey2 = `
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIPIJhRf4MpzLil1ZKcRqMx+jPeJXw96KtYYzV2AcgBzgoAoGCCqGSM49
AwEHoUQDQgAEhA9CSWFmUvdvXMzyt+as+6f+0luydHU1x/gEksVByYIgYxahaGts
xbSKj6F2WgAN/ok1gFLqhH3UWMNVthM1wA==
-----END EC PRIVATE KEY-----
`
someECDSAKey3 = `
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIKgZ1OJjK2St9O0i52N1K+IgSiu2/NSMk9Yt2+kDMHd7oAoGCCqGSM49
AwEHoUQDQgAExFp80etkjy/AEUtSgJjXRA39jTU7eiEmCGRREewFQhwcEscBEfrg
6NN31r9YlEs+hZ8gXE1L3Deu6jn5jW3pig==
-----END EC PRIVATE KEY-----
`
)
// parseECKey parses an EC private key from a PEM-encoded string.
func parseECKey(t *testing.T, pemPriv string) *ecdsa.PrivateKey {
t.Helper()
block, _ := pem.Decode([]byte(pemPriv))
if block == nil {
t.Fatal("failed to decode PEM")
}
key, err := x509.ParseECPrivateKey(block.Bytes)
if err != nil {
t.Fatalf("failed to parse EC key: %v", err)
}
return key
}
// crlTestPKI is a CA that can sign CRLs plus a leaf certificate it issued.
type crlTestPKI struct {
caCert *x509.Certificate
caKey *ecdsa.PrivateKey
leaf *x509.Certificate
}
func newCRLTestPKI(t *testing.T) crlTestPKI {
t.Helper()
// Generate CA key and self-signed CA cert
caKey := parseECKey(t, someECDSAKey1)
caTpl := issuerCertTpl
caTpl.BasicConstraintsValid = true
caTpl.IsCA = true
caTpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature
caTpl.SignatureAlgorithm = x509.ECDSAWithSHA256
caBytes, err := x509.CreateCertificate(rand.Reader, &caTpl, &caTpl, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
caCert, err := x509.ParseCertificate(caBytes)
if err != nil {
t.Fatal(err)
}
// Issue a leaf cert signed by the CA
leaf := leafCert
leaf.SerialNumber = big.NewInt(20001)
leaf.SignatureAlgorithm = x509.ECDSAWithSHA256
leaf.Issuer = caCert.Subject
leafKey := parseECKey(t, someECDSAKey2)
leafBytes, err := x509.CreateCertificate(rand.Reader, &leaf, caCert, &leafKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
leafCertParsed, err := x509.ParseCertificate(leafBytes)
if err != nil {
t.Fatal(err)
}
return crlTestPKI{caCert: caCert, caKey: caKey, leaf: leafCertParsed}
}
// crl signs a CRL from the test CA revoking the given serials as of
// thisUpdate. A zero nextUpdate omits the field and requires a zero
// thisUpdate, since x509.CreateRevocationList rejects a NextUpdate before
// ThisUpdate.
func (p crlTestPKI) crl(t *testing.T, thisUpdate, nextUpdate time.Time, revoked ...*big.Int) []byte {
t.Helper()
rl := x509.RevocationList{
SignatureAlgorithm: p.caCert.SignatureAlgorithm,
Issuer: p.caCert.Subject,
ThisUpdate: thisUpdate,
NextUpdate: nextUpdate,
Number: big.NewInt(1),
}
for _, serial := range revoked {
rl.RevokedCertificateEntries = append(rl.RevokedCertificateEntries, x509.RevocationListEntry{
SerialNumber: serial,
RevocationTime: thisUpdate,
ReasonCode: 1, // Key compromise
})
}
b, err := x509.CreateRevocationList(rand.Reader, &rl, p.caCert, p.caKey)
if err != nil {
t.Fatal(err)
}
return b
}
func TestCRL(t *testing.T) {
pki := newCRLTestPKI(t)
caCert, caKey, leafCertParsed := pki.caCert, pki.caKey, pki.leaf
// Catch no CRL set by Let's Encrypt date.
noCRLCert := leafCert
noCRLCert.SerialNumber = big.NewInt(20002)
noCRLCert.CRLDistributionPoints = []string{}
noCRLCert.NotBefore = time.Unix(letsEncryptStartedStaplingCRL, 0).Add(-48 * time.Hour)
noCRLCert.Issuer = caCert.Subject
noCRLCert.SignatureAlgorithm = x509.ECDSAWithSHA256
noCRLCertKey := parseECKey(t, someECDSAKey3)
noCRLStapledBytes, err := x509.CreateCertificate(rand.Reader, &noCRLCert, caCert, &noCRLCertKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
noCRLStapledParsed, err := x509.ParseCertificate(noCRLStapledBytes)
if err != nil {
t.Fatal(err)
}
crlServer := &CRLServer{crlBytes: nil}
srv := httptest.NewServer(crlServer)
defer srv.Close()
// Create a CRL that revokes the leaf cert using x509.CreateRevocationList
now := time.Now()
revoked := []x509.RevocationListEntry{{
SerialNumber: leafCertParsed.SerialNumber,
RevocationTime: now,
ReasonCode: 1, // Key compromise
}}
rl := x509.RevocationList{
SignatureAlgorithm: caCert.SignatureAlgorithm,
Issuer: caCert.Subject,
ThisUpdate: now,
NextUpdate: now.Add(24 * time.Hour),
RevokedCertificateEntries: revoked,
Number: big.NewInt(1),
}
rlBytes, err := x509.CreateRevocationList(rand.Reader, &rl, caCert, caKey)
if err != nil {
t.Fatal(err)
}
emptyRlBytes, err := x509.CreateRevocationList(rand.Reader, &x509.RevocationList{Number: big.NewInt(2)}, caCert, caKey)
if err != nil {
t.Fatal(err)
}
for _, tt := range []struct {
name string
cert *x509.Certificate
crlBytes []byte
issuer pkix.Name
wantErr string
}{
{
"ValidCert",
leafCertParsed,
emptyRlBytes,
caCert.Issuer,
"",
},
{
"RevokedCert",
leafCertParsed,
rlBytes,
caCert.Issuer,
"has been revoked on",
},
{
"EmptyCRL",
leafCertParsed,
emptyRlBytes,
caCert.Issuer,
"",
},
{
"NoCRLLetsEncrypt",
leafCertParsed,
nil,
pkix.Name{CommonName: "tlsprobe.test", Organization: []string{"Let's Encrypt"}},
"no CRL server presented in leaf cert for",
},
{
"NoCRLOtherCA",
leafCertParsed,
nil,
caCert.Issuer,
"",
},
{
"NotBeforeCRLStaplingDate",
noCRLStapledParsed,
nil,
caCert.Issuer,
"",
},
} {
t.Run(tt.name, func(t *testing.T) {
tt.cert.Issuer = tt.issuer
cs := &tls.ConnectionState{PeerCertificates: []*x509.Certificate{tt.cert, caCert}}
if tt.crlBytes != nil {
crlServer.crlBytes = tt.crlBytes
tt.cert.CRLDistributionPoints = []string{srv.URL}
} else {
crlServer.crlBytes = nil
tt.cert.CRLDistributionPoints = []string{}
}
// Each subtest needs its own cache: they share one CRL server
// URL, which is the cache key, but swap the bytes it serves.
err := validateConnState(context.Background(), &crlCache{now: time.Now}, cs)
if err == nil && tt.wantErr == "" {
return
}
if err == nil || tt.wantErr == "" || !strings.Contains(err.Error(), tt.wantErr) {
t.Errorf("unexpected error %q; want %q", err, tt.wantErr)
}
})
}
}
func TestCRLCache(t *testing.T) {
pki := newCRLTestPKI(t)
start := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)
newServer := func(t *testing.T, s *CRLServer) *httptest.Server {
srv := httptest.NewServer(s)
t.Cleanup(srv.Close)
return srv
}
check := func(c *crlCache, srv *httptest.Server) error {
return c.checkCertCRL(t.Context(), srv.URL, pki.leaf, pki.caCert)
}
mustCheck := func(t *testing.T, c *crlCache, srv *httptest.Server) {
t.Helper()
if err := check(c, srv); err != nil {
t.Fatalf("checkCertCRL: %v", err)
}
}
wantRequests := func(t *testing.T, s *CRLServer, want int32) {
t.Helper()
if got := s.requests.Load(); got != want {
t.Errorf("CRL server saw %d requests; want %d", got, want)
}
}
t.Run("ReusedAcrossProbes", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour))}
srv := newServer(t, s)
for range 5 {
mustCheck(t, c, srv)
clock.Advance(15 * time.Second)
}
wantRequests(t, s, 1)
})
t.Run("RefetchedAfterRefreshInterval", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour))}
srv := newServer(t, s)
mustCheck(t, c, srv)
clock.Advance(crlRefreshInterval - time.Minute)
mustCheck(t, c, srv)
wantRequests(t, s, 1)
clock.Advance(2 * time.Minute)
mustCheck(t, c, srv)
wantRequests(t, s, 2)
})
t.Run("RefetchedAfterNextUpdate", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(crlRefreshInterval/2))}
srv := newServer(t, s)
mustCheck(t, c, srv)
clock.Advance(crlRefreshInterval/2 - time.Minute)
mustCheck(t, c, srv)
wantRequests(t, s, 1)
clock.Advance(2 * time.Minute)
mustCheck(t, c, srv)
wantRequests(t, s, 2)
})
t.Run("ConcurrentCallersShareOneFetch", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour))}
srv := newServer(t, s)
var wg sync.WaitGroup
errs := make([]error, 8)
for i := range errs {
wg.Go(func() {
errs[i] = check(c, srv)
})
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Errorf("caller %d: %v", i, err)
}
}
wantRequests(t, s, 1)
})
t.Run("RevokedFromCache", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour), pki.leaf.SerialNumber)}
srv := newServer(t, s)
for range 3 {
err := check(c, srv)
if err == nil || !strings.Contains(err.Error(), "has been revoked on") {
t.Fatalf("unexpected error %q; want revoked", err)
}
clock.Advance(15 * time.Second)
}
wantRequests(t, s, 1)
})
t.Run("WrongIssuerNotCached", func(t *testing.T) {
clock := tstest.NewClock(tstest.ClockOpts{Start: start})
c := &crlCache{now: clock.Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour))}
srv := newServer(t, s)
// The leaf is not a CA, so it cannot have signed the CRL.
err := c.checkCertCRL(t.Context(), srv.URL, pki.leaf, pki.leaf)
if err == nil || !strings.Contains(err.Error(), "could not verify CRL signature") {
t.Fatalf("unexpected error %q; want signature failure", err)
}
wantRequests(t, s, 1)
mustCheck(t, c, srv)
wantRequests(t, s, 2)
})
t.Run("NoNextUpdateNotCached", func(t *testing.T) {
c := &crlCache{now: tstest.NewClock(tstest.ClockOpts{Start: start}).Now}
s := &CRLServer{crlBytes: pki.crl(t, time.Time{}, time.Time{})}
srv := newServer(t, s)
for range 3 {
mustCheck(t, c, srv)
}
wantRequests(t, s, 3)
})
t.Run("NilIssuerReturnsError", func(t *testing.T) {
c := &crlCache{now: tstest.NewClock(tstest.ClockOpts{Start: start}).Now}
s := &CRLServer{crlBytes: pki.crl(t, start, start.Add(7*24*time.Hour))}
srv := newServer(t, s)
err := c.checkCertCRL(t.Context(), srv.URL, pki.leaf, nil)
if err == nil || !strings.Contains(err.Error(), "not in presented chain") {
t.Fatalf("unexpected error %q; want missing issuer", err)
}
wantRequests(t, s, 0)
})
}