tailscale/kube/certs/certs.go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause

// Package certs implements logic to help multiple Kubernetes replicas share TLS
// certs for a common Tailscale Service.
package certs

import (
	"context"
	"errors"
	"fmt"
	"net"
	"slices"
	"sync"
	"syscall"
	"time"

	"tailscale.com/client/local"
	"tailscale.com/ipn"
	"tailscale.com/kube/localclient"
	"tailscale.com/types/logger"
	"tailscale.com/util/goroutines"
	"tailscale.com/util/mak"
)

// CertManager is responsible for issuing certificates for known domains and for
// maintaining a loop that re-attempts issuance daily.
// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
// HA Ingress HTTPS endpoints ('write' replicas).
type CertManager struct {
	lc      localclient.LocalClient
	logf    logger.Logf
	tracker goroutines.Tracker // tracks running goroutines
	mu      sync.Mutex         // guards the following
	// certLoops contains a map of DNS names, for which we currently need to
	// manage certs to cancel functions that allow stopping a goroutine when
	// we no longer need to manage certs for the DNS name.
	certLoops map[string]context.CancelFunc
}

func NewCertManager(lc localclient.LocalClient, logf logger.Logf) *CertManager {
	return &CertManager{
		lc:   lc,
		logf: logf,
	}
}

// EnsureCertLoops ensures that, for all currently managed Service HTTPS
// endpoints, there is a cert loop responsible for issuing and ensuring the
// renewal of the TLS certs.
// ServeConfig must not be nil.
func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
	if sc == nil {
		return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
	}
	currentDomains := make(map[string]bool)
	const httpsPort = "443"
	for _, service := range sc.Services {
		// L7 Web handlers (HA Ingress).
		for hostPort := range service.Web {
			domain, port, err := net.SplitHostPort(string(hostPort))
			if err != nil {
				return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
			}
			if port != httpsPort { // HA Ingress' HTTP endpoint
				continue
			}
			currentDomains[domain] = true
		}
		// L4 TCP handlers with TLS termination (kube-apiserver proxy).
		for _, handler := range service.TCP {
			if handler != nil && handler.TerminateTLS != "" {
				currentDomains[handler.TerminateTLS] = true
			}
		}
	}
	cm.mu.Lock()
	defer cm.mu.Unlock()
	for domain := range currentDomains {
		if _, exists := cm.certLoops[domain]; !exists {
			cancelCtx, cancel := context.WithCancel(ctx)
			mak.Set(&cm.certLoops, domain, cancel)
			// Note that most of the issuance anyway happens
			// serially because the cert client has a shared lock
			// that's held during any issuance.
			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
		}
	}

	// Stop goroutines for domain names that are no longer in the config.
	for domain, cancel := range cm.certLoops {
		if !currentDomains[domain] {
			cancel()
			delete(cm.certLoops, domain)
		}
	}
	return nil
}

// isTransientCertErr reports whether err represents a failure that did not
// reach the CA (ctx timeout, LocalAPI socket unreachable). Such errors must
// not advance the loop's retryCount.
func isTransientCertErr(err error) bool {
	switch {
	case errors.Is(err, context.DeadlineExceeded),
		errors.Is(err, context.Canceled),
		errors.Is(err, syscall.ECONNREFUSED),
		errors.Is(err, syscall.ECONNRESET),
		errors.Is(err, syscall.EHOSTUNREACH),
		errors.Is(err, syscall.EPIPE):
		return true
	}
	var ne net.Error
	if errors.As(err, &ne) && ne.Timeout() {
		return true
	}
	return false
}

// nextRetryInterval picks the wait until the next CertPair attempt:
//   - success: reset retryCount, normalInterval.
//   - transient (never reached the CA): retrySchedule[0], no retryCount advance.
//   - retryAfter > 0: honour it, retryCount still advances.
//   - other: advance retryCount, walk retrySchedule.
func nextRetryInterval(err error, retryCount *int, normalInterval, retryAfter time.Duration) time.Duration {
	switch {
	case err == nil:
		*retryCount = 0
		return normalInterval
	case isTransientCertErr(err):
		return retrySchedule[0]
	}
	*retryCount++
	idx := *retryCount - 1
	if idx >= len(retrySchedule) {
		idx = len(retrySchedule) - 1
	}
	interval := retrySchedule[idx]
	if retryAfter > 0 {
		interval = retryAfter
	}
	return interval
}

// retrySchedule is the wait between successive failed issuance attempts,
// following LE's recommended schedule.
// https://letsencrypt.org/docs/integration-guide/#retrying-failures
var retrySchedule = []time.Duration{
	1 * time.Minute,
	10 * time.Minute,
	100 * time.Minute,
	24 * time.Hour,
}

// runCertLoop:
// - calls localAPI certificate endpoint to ensure that certs are issued for the
// given domain name
// - calls localAPI certificate endpoint daily to ensure that certs are renewed
// - if certificate issuance failed, retries on the schedule defined by
// [retrySchedule]; resets to the start once issuance succeeds.
// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
// replica does not get any HTTPS requests.
func (cm *CertManager) runCertLoop(ctx context.Context, domain string) {
	const normalInterval = 24 * time.Hour // regular renewal check

	if err := cm.waitForCertDomain(ctx, domain); err != nil {
		// Best-effort, log and continue with the issuing loop.
		cm.logf("error waiting for cert domain %s: %v", domain, err)
	}

	timer := time.NewTimer(0) // fire off timer immediately
	defer timer.Stop()
	retryCount := 0
	for {
		select {
		case <-ctx.Done():
			return
		case <-timer.C:
			// We call the certificate endpoint, but don't do anything with the
			// returned certs here. The call to the certificate endpoint will
			// ensure that certs are issued/renewed as needed and stored in the
			// relevant state store. For example, for HA Ingress 'write' replica,
			// the cert and key will be stored in a Kubernetes Secret named after
			// the domain for which we are issuing.
			//
			// Note that renewals triggered by the call to the certificates
			// endpoint here and by renewal check triggered during a call to
			// node's HTTPS endpoint share the same state/renewal lock mechanism,
			// so we should not run into redundant issuances during concurrent
			// renewal checks.
			//
			// Long enough to cover queue contention behind tailscaled's
			// shared cert mutex; if it fires, something is wedged.
			ctxT, cancel := context.WithTimeout(ctx, 30*time.Minute)
			_, _, err := cm.lc.CertPair(ctxT, domain)
			cancel()
			retryAfter, _ := local.RateLimitRetryAfter(err)
			nextInterval := nextRetryInterval(err, &retryCount, normalInterval, retryAfter)
			if err != nil {
				cm.logf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
					domain, retryCount, err, nextInterval)
			}
			timer.Reset(nextInterval)
		}
	}
}

// domains before issuing the cert for the first time. It uses the IPN bus
// only as a wake-up trigger (the initial status and subsequent
// Notify.SelfChange messages) and queries the current cert domains
// explicitly via [LocalClient.CertDomains].
func (cm *CertManager) waitForCertDomain(ctx context.Context, domain string) error {
	w, err := cm.lc.WatchIPNBus(ctx, ipn.NotifyInitialStatus)
	if err != nil {
		return fmt.Errorf("error watching IPN bus: %w", err)
	}
	defer w.Close()

	for {
		n, err := w.Next()
		if err != nil {
			return err
		}
		if n.SelfChange == nil && n.InitialStatus == nil {
			continue
		}
		domains, err := cm.lc.CertDomains(ctx)
		if err != nil {
			continue
		}
		if slices.Contains(domains, domain) {
			return nil
		}
	}
}