ipnlocal: try ACME TLS-ALPN for Funnel renewals

Use TLS-ALPN-01 for Funnel certificate renewals only when the node
already has a cached certificate, and fall back to DNS-01 with a fresh
order if the ALPN path is unavailable or fails.

Dynamically advertise acme-tls/1 only while an ACME challenge
certificate is pending, and add client metrics for DNS-01 and
TLS-ALPN-01 start/success/failure paths.

Updates tailscale/corp#41736
Fixes tailscale/corp#42320

Change-Id: I5adc6ea129237f9ef592f84fc1a8953c80bc9d5c
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

ipn/ipnlocal/cert.go

@@ -44,6 +44,7 @@
 	"tailscale.com/tailcfg"
 	"tailscale.com/tempfork/acme"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/testenv"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -65,6 +66,63 @@ func init() {
 	renewCertAt = map[string]time.Time{}
 )
 
+var (
+	metricACMEDNS01Start       = clientmetric.NewCounter("cert_acme_dns01_start")
+	metricACMEDNS01Success     = clientmetric.NewCounter("cert_acme_dns01_success")
+	metricACMEDNS01Failure     = clientmetric.NewCounter("cert_acme_dns01_failure")
+	metricACMETLSALPN01Start   = clientmetric.NewCounter("cert_acme_tls_alpn01_start")
+	metricACMETLSALPN01Success = clientmetric.NewCounter("cert_acme_tls_alpn01_success")
+	metricACMETLSALPN01Failure = clientmetric.NewCounter("cert_acme_tls_alpn01_failure")
+)
+
+type acmeChallengeType string
+
+const (
+	acmeChallengeDNS01     acmeChallengeType = "dns-01"
+	acmeChallengeTLSALPN01 acmeChallengeType = "tls-alpn-01"
+)
+
+// serveTLSNextProtos returns the baseline ALPN protocols for ordinary Serve
+// TLS traffic. ACME tls-alpn-01 is intentionally not advertised here; it is
+// added dynamically by serveTLSConfig only while a matching challenge
+// certificate is pending.
+func serveTLSNextProtos() []string {
+	return []string{"h2", "http/1.1"}
+}
+
+// getACMETLSALPNCert returns the short-lived ACME challenge certificate for
+// hi.ServerName. The ok result reports whether hi offered acme-tls/1 and an
+// ACME order is actively waiting on that challenge for hi.ServerName.
+func (b *LocalBackend) getACMETLSALPNCert(hi *tls.ClientHelloInfo) (cert *tls.Certificate, ok bool) {
+	if hi == nil || hi.ServerName == "" || !slices.Contains(hi.SupportedProtos, acme.ALPNProto) {
+		return nil, false
+	}
+	cert, ok = b.pendingACMETLSALPNCerts.Load(hi.ServerName)
+	return cert, ok
+}
+
+// getACMETLSALPNProto reports whether serveTLSConfig should advertise an ACME
+// ALPN protocol for this ClientHello. The proto result is the protocol to
+// advertise, and ok reports whether hi offered acme-tls/1 and an ACME order is
+// actively waiting on that challenge for hi.ServerName. It is separate from
+// getACMETLSALPNCert because Go selects ALPN before calling GetCertificate;
+// both hooks must agree for the challenge handshake to complete.
+func (b *LocalBackend) getACMETLSALPNProto(hi *tls.ClientHelloInfo) (proto string, ok bool) {
+	if _, ok := b.getACMETLSALPNCert(hi); !ok {
+		return "", false
+	}
+	return acme.ALPNProto, true
+}
+
+// storeACMETLSALPNCert publishes cert to Serve TLS handshakes for domain until
+// the returned cleanup function is called.
+func (b *LocalBackend) storeACMETLSALPNCert(domain string, cert *tls.Certificate) (cleanup func()) {
+	b.pendingACMETLSALPNCerts.Store(domain, cert)
+	return func() {
+		b.pendingACMETLSALPNCerts.Delete(domain)
+	}
+}
+
 // certDir returns (creating if needed) the directory in which cached
 // cert keypairs are stored.
 func (b *LocalBackend) certDir() (string, error) {
@@ -244,6 +302,36 @@ func (b *LocalBackend) domainRenewalTimeByExpiry(pair *TLSCertKeyPair) (time.Tim
 	return renewAt, nil
 }
 
+func (b *LocalBackend) shouldUseACMETLSALPN01(domain string, previous *TLSCertKeyPair, logf logger.Logf) bool {
+	if isWildcardDomain(domain) {
+		logf("acme: using dns-01: tls-alpn-01 does not support wildcard certificates")
+		return false
+	}
+	if !b.hasFunnelForHostPort(domain, 443) {
+		logf("acme: using dns-01: Funnel is not enabled for %s:443", domain)
+		return false
+	}
+	if previous == nil {
+		logf("acme: using dns-01: no cached certificate for Funnel renewal")
+		return false
+	}
+	logf("acme: using tls-alpn-01")
+	return true
+}
+
+func challengeByType(challenges []*acme.Challenge, typ string) *acme.Challenge {
+	for _, ch := range challenges {
+		if ch.Type == typ {
+			return ch
+		}
+	}
+	return nil
+}
+
+func isWildcardDomain(domain string) bool {
+	return strings.HasPrefix(domain, "*.")
+}
+
 func (b *LocalBackend) domainRenewalTimeByARI(cs certStore, pair *TLSCertKeyPair) (time.Time, error) {
 	var blocks []*pem.Block
 	rest := pair.CertPEM
@@ -527,8 +615,6 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 	acmeMu.Lock()
 	defer acmeMu.Unlock()
 
-	baseDomain, isWildcard := strings.CutPrefix(domain, "*.")
-
 	// In case this method was triggered multiple times in parallel (when
 	// serving incoming requests), check whether one of the other goroutines
 	// already renewed the cert before us.
@@ -593,63 +679,113 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 		}
 	}
 
+	issueArgs := acmeCertIssueArgs{
+		cs:        cs,
+		logf:      logf,
+		traceACME: traceACME,
+		domain:    domain,
+		opts:      opts,
+	}
+	if b.shouldUseACMETLSALPN01(domain, previous, logf) {
+		issueArgs.challengeType = acmeChallengeTLSALPN01
+		pair, err := b.issueACMECert(ctx, ac, issueArgs)
+		if err == nil {
+			return pair, nil
+		}
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		logf("acme: tls-alpn-01 failed; falling back to dns-01: %v", err)
+	}
+	issueArgs.challengeType = acmeChallengeDNS01
+	return b.issueACMECert(ctx, ac, issueArgs)
+}
+
+type acmeCertIssueArgs struct {
+	cs            certStore          // certificate and ACME account storage
+	logf          logger.Logf        // logs ACME progress and failures
+	traceACME     func(any)          // optional hook for logging ACME messages
+	domain        string             // certificate domain being issued
+	opts          []acme.OrderOption // ACME order options
+	challengeType acmeChallengeType  // challenge type to fulfill
+}
+
+func (args acmeCertIssueArgs) baseDomain() string { return strings.TrimPrefix(args.domain, "*.") }
+func (args acmeCertIssueArgs) isWildcard() bool   { return isWildcardDomain(args.domain) }
+
+func (b *LocalBackend) issueACMECert(ctx context.Context, ac *acme.Client, args acmeCertIssueArgs) (ret *TLSCertKeyPair, err error) {
+	if args.traceACME == nil {
+		args.traceACME = func(any) {}
+	}
+
+	switch args.challengeType {
+	case acmeChallengeTLSALPN01:
+		metricACMETLSALPN01Start.Add(1)
+		defer func() {
+			if err == nil {
+				metricACMETLSALPN01Success.Add(1)
+			} else {
+				metricACMETLSALPN01Failure.Add(1)
+			}
+		}()
+	case acmeChallengeDNS01:
+		metricACMEDNS01Start.Add(1)
+		defer func() {
+			if err == nil {
+				metricACMEDNS01Success.Add(1)
+			} else {
+				metricACMEDNS01Failure.Add(1)
+			}
+		}()
+	default:
+		return nil, fmt.Errorf("unknown ACME challenge type %q", args.challengeType)
+	}
+
 	// For wildcards, we need to authorize both the wildcard and base domain.
 	var authzIDs []acme.AuthzID
-	if isWildcard {
+	if args.isWildcard() {
 		authzIDs = []acme.AuthzID{
-			{Type: "dns", Value: domain},
-			{Type: "dns", Value: baseDomain},
+			{Type: "dns", Value: args.domain},
+			{Type: "dns", Value: args.baseDomain()},
 		}
 	} else {
-		authzIDs = []acme.AuthzID{{Type: "dns", Value: domain}}
+		authzIDs = []acme.AuthzID{{Type: "dns", Value: args.domain}}
 	}
-	order, err := ac.AuthorizeOrder(ctx, authzIDs, opts...)
+	order, err := ac.AuthorizeOrder(ctx, authzIDs, args.opts...)
 	if err != nil {
 		return nil, err
 	}
-	traceACME(order)
+	args.traceACME(order)
 
 	for _, aurl := range order.AuthzURLs {
 		az, err := ac.GetAuthorization(ctx, aurl)
 		if err != nil {
 			return nil, err
 		}
-		traceACME(az)
-		for _, ch := range az.Challenges {
-			if ch.Type == "dns-01" {
-				rec, err := ac.DNS01ChallengeRecord(ch.Token)
-				if err != nil {
-					return nil, err
-				}
-				// For wildcards, the challenge is on the base domain.
-				// e.g., "*.node.ts.net" -> "_acme-challenge.node.ts.net"
-				key := "_acme-challenge." + strings.TrimPrefix(az.Identifier.Value, "*.")
-
-				// Do a best-effort lookup to see if we've already created this DNS name
-				// in a previous attempt. Don't burn too much time on it, though. Worst
-				// case we ask the server to create something that already exists.
-				var resolver net.Resolver
-				lookupCtx, lookupCancel := context.WithTimeout(ctx, 500*time.Millisecond)
-				txts, _ := resolver.LookupTXT(lookupCtx, key)
-				lookupCancel()
-				if slices.Contains(txts, rec) {
-					logf("TXT record already existed for %s", key)
-				} else {
-					logf("starting SetDNS call for %s...", key)
-					err = b.SetDNS(ctx, key, rec)
-					if err != nil {
-						return nil, fmt.Errorf("SetDNS %q => %q: %w", key, rec, err)
-					}
-					logf("did SetDNS for %s", key)
-				}
-
-				chal, err := ac.Accept(ctx, ch)
-				if err != nil {
-					return nil, fmt.Errorf("Accept: %v", err)
-				}
-				traceACME(chal)
-				break
+		args.traceACME(az)
+		switch args.challengeType {
+		case acmeChallengeTLSALPN01:
+			ch := challengeByType(az.Challenges, string(acmeChallengeTLSALPN01))
+			if ch == nil {
+				return nil, errors.New("tls-alpn-01 challenge not offered")
 			}
+			cert, err := ac.TLSALPN01ChallengeCert(ch.Token, az.Identifier.Value)
+			if err != nil {
+				return nil, fmt.Errorf("TLSALPN01ChallengeCert: %w", err)
+			}
+			cleanup := b.storeACMETLSALPNCert(az.Identifier.Value, &cert)
+			defer cleanup()
+			chal, err := ac.Accept(ctx, ch)
+			if err != nil {
+				return nil, fmt.Errorf("Accept: %v", err)
+			}
+			args.traceACME(chal)
+		case acmeChallengeDNS01:
+			if err := b.fulfillACMEDNS01Challenge(ctx, ac, az, args.logf, args.traceACME); err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unknown ACME challenge type %q", args.challengeType)
 		}
 	}
 
@@ -660,13 +796,13 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 			return nil, ctx.Err()
 		}
 		if oe, ok := err.(*acme.OrderError); ok {
-			logf("acme: WaitOrder: OrderError status %q", oe.Status)
+			args.logf("acme: WaitOrder: OrderError status %q", oe.Status)
 		} else {
-			logf("acme: WaitOrder error: %v", err)
+			args.logf("acme: WaitOrder error: %v", err)
 		}
 		return nil, err
 	}
-	traceACME(order)
+	args.traceACME(order)
 
 	certPrivKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
@@ -677,18 +813,18 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 		return nil, err
 	}
 
-	csr, err := certRequest(certPrivKey, domain, nil)
+	csr, err := certRequest(certPrivKey, args.domain, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	logf("requesting cert...")
-	traceACME(csr)
+	args.logf("requesting cert...")
+	args.traceACME(csr)
 	der, _, err := ac.CreateOrderCert(ctx, order.FinalizeURL, csr, true)
 	if err != nil {
 		return nil, fmt.Errorf("CreateOrder: %v", err)
 	}
-	logf("got cert")
+	args.logf("got cert")
 
 	var certPEM bytes.Buffer
 	for _, b := range der {
@@ -697,14 +833,55 @@ func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKey
 			return nil, err
 		}
 	}
-	if err := cs.WriteTLSCertAndKey(domain, certPEM.Bytes(), privPEM.Bytes()); err != nil {
+	if err := args.cs.WriteTLSCertAndKey(args.domain, certPEM.Bytes(), privPEM.Bytes()); err != nil {
 		return nil, err
 	}
-	b.domainRenewed(domain)
+	b.domainRenewed(args.domain)
 
 	return &TLSCertKeyPair{CertPEM: certPEM.Bytes(), KeyPEM: privPEM.Bytes()}, nil
 }
 
+func (b *LocalBackend) fulfillACMEDNS01Challenge(ctx context.Context, ac *acme.Client, az *acme.Authorization, logf logger.Logf, traceACME func(any)) error {
+	for _, ch := range az.Challenges {
+		if ch.Type != string(acmeChallengeDNS01) {
+			continue
+		}
+		rec, err := ac.DNS01ChallengeRecord(ch.Token)
+		if err != nil {
+			return err
+		}
+		// For wildcards, the challenge is on the base domain.
+		// e.g., "*.node.ts.net" -> "_acme-challenge.node.ts.net"
+		key := "_acme-challenge." + strings.TrimPrefix(az.Identifier.Value, "*.")
+
+		// Do a best-effort lookup to see if we've already created this DNS name
+		// in a previous attempt. Don't burn too much time on it, though. Worst
+		// case we ask the server to create something that already exists.
+		var resolver net.Resolver
+		lookupCtx, lookupCancel := context.WithTimeout(ctx, 500*time.Millisecond)
+		txts, _ := resolver.LookupTXT(lookupCtx, key)
+		lookupCancel()
+		if slices.Contains(txts, rec) {
+			logf("TXT record already existed for %s", key)
+		} else {
+			logf("starting SetDNS call for %s...", key)
+			err = b.SetDNS(ctx, key, rec)
+			if err != nil {
+				return fmt.Errorf("SetDNS %q => %q: %w", key, rec, err)
+			}
+			logf("did SetDNS for %s", key)
+		}
+
+		chal, err := ac.Accept(ctx, ch)
+		if err != nil {
+			return fmt.Errorf("Accept: %v", err)
+		}
+		traceACME(chal)
+		return nil
+	}
+	return errors.New("dns-01 challenge not offered")
+}
+
 // certRequest generates a CSR for the given domain and optional SANs.
 func certRequest(key crypto.Signer, domain string, ext []pkix.Extension) ([]byte, error) {
 	dnsNames := []string{domain}

ipn/ipnlocal/cert_disabled.go

@@ -7,6 +7,7 @@
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"io"
 	"net/http"
@@ -27,6 +28,18 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 	return nil, errNoCerts
 }
 
+func serveTLSNextProtos() []string {
+	return []string{"h2", "http/1.1"}
+}
+
+func (b *LocalBackend) getACMETLSALPNCert(hi *tls.ClientHelloInfo) (*tls.Certificate, bool) {
+	return nil, false
+}
+
+func (b *LocalBackend) getACMETLSALPNProto(hi *tls.ClientHelloInfo) (string, bool) {
+	return "", false
+}
+
 var errCertExpired = errors.New("cert expired")
 
 type certStore interface{}

ipn/ipnlocal/cert_test.go

@@ -10,6 +10,7 @@
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"embed"
@@ -23,8 +24,10 @@
 
 	"github.com/google/go-cmp/cmp"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tempfork/acme"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -250,6 +253,87 @@ func TestValidLookingCertDomain(t *testing.T) {
 	}
 }
 
+func TestACMETLSALPNCertHook(t *testing.T) {
+	b := newTestLocalBackend(t)
+	cert := &tls.Certificate{}
+	cleanup := b.storeACMETLSALPNCert("example.com", cert)
+	defer cleanup()
+
+	if got, ok := b.getACMETLSALPNCert(&tls.ClientHelloInfo{
+		ServerName:      "example.com",
+		SupportedProtos: []string{acme.ALPNProto},
+	}); !ok || got != cert {
+		t.Fatalf("getACMETLSALPNCert = %v, %v; want stored cert, true", got, ok)
+	}
+	if _, ok := b.getACMETLSALPNCert(&tls.ClientHelloInfo{
+		ServerName:      "example.com",
+		SupportedProtos: []string{"http/1.1"},
+	}); ok {
+		t.Fatal("getACMETLSALPNCert without acme ALPN = ok, want false")
+	}
+	if _, ok := b.getACMETLSALPNCert(&tls.ClientHelloInfo{
+		ServerName:      "other.example.com",
+		SupportedProtos: []string{acme.ALPNProto},
+	}); ok {
+		t.Fatal("getACMETLSALPNCert for other name = ok, want false")
+	}
+
+	otherBackend := newTestLocalBackend(t)
+	if _, ok := otherBackend.getACMETLSALPNCert(&tls.ClientHelloInfo{
+		ServerName:      "example.com",
+		SupportedProtos: []string{acme.ALPNProto},
+	}); ok {
+		t.Fatal("getACMETLSALPNCert on different LocalBackend = ok, want false")
+	}
+}
+
+func TestServeTLSConfigNextProtos(t *testing.T) {
+	b := newTestLocalBackend(t)
+	getCert := func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return nil, nil
+	}
+
+	httpsConfig := b.serveTLSConfig(getCert, serveTLSNextProtos())
+	if got, want := httpsConfig.NextProtos, []string{"h2", "http/1.1"}; !slices.Equal(got, want) {
+		t.Fatalf("HTTPS NextProtos = %q; want %q", got, want)
+	}
+
+	tcpForwardConfig := b.serveTLSConfig(getCert, nil)
+	if got := tcpForwardConfig.NextProtos; got != nil {
+		t.Fatalf("TLS-terminated TCP forward NextProtos = %q; want nil", got)
+	}
+}
+
+func TestShouldUseACMETLSALPN01(t *testing.T) {
+	const domain = "example.com"
+	b := newTestLocalBackend(t)
+	b.mu.Lock()
+	b.serveConfig = (&ipn.ServeConfig{
+		AllowFunnel: map[ipn.HostPort]bool{
+			domain + ":443": true,
+		},
+	}).View()
+	b.mu.Unlock()
+
+	previous := &TLSCertKeyPair{}
+	if !b.shouldUseACMETLSALPN01(domain, previous, t.Logf) {
+		t.Fatal("shouldUseACMETLSALPN01 = false, want true")
+	}
+	if b.shouldUseACMETLSALPN01(domain, nil, t.Logf) {
+		t.Fatal("shouldUseACMETLSALPN01 without cached cert = true, want false")
+	}
+	if b.shouldUseACMETLSALPN01("*."+domain, previous, t.Logf) {
+		t.Fatal("shouldUseACMETLSALPN01 for wildcard = true, want false")
+	}
+
+	b.mu.Lock()
+	b.serveConfig = (&ipn.ServeConfig{}).View()
+	b.mu.Unlock()
+	if b.shouldUseACMETLSALPN01(domain, previous, t.Logf) {
+		t.Fatal("shouldUseACMETLSALPN01 without Funnel = true, want false")
+	}
+}
+
 //go:embed testdata/*
 var certTestFS embed.FS
 

ipn/ipnlocal/local.go

@@ -9,6 +9,7 @@
 	"bufio"
 	"cmp"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -245,6 +246,13 @@ type LocalBackend struct {
 	// is never called.
 	getTCPHandlerForFunnelFlow func(srcAddr netip.AddrPort, dstPort uint16) (handler func(net.Conn))
 
+	// pendingACMETLSALPNCerts maps SNI names to short-lived ACME tls-alpn-01
+	// challenge certificates while an ACME order is waiting for validation.
+	// Entries are deleted by the cleanup function returned from
+	// storeACMETLSALPNCert after the challenge validation path finishes,
+	// whether it succeeds or fails.
+	pendingACMETLSALPNCerts syncs.Map[string, *tls.Certificate] // "foo.bar.com" => challenge cert
+
 	containsViaIPFuncAtomic                 syncs.AtomicValue[func(netip.Addr) bool]     // TODO(nickkhyl): move to nodeBackend
 	shouldInterceptTCPPortAtomic            syncs.AtomicValue[func(uint16) bool]         // TODO(nickkhyl): move to nodeBackend
 	shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] // TODO(nickkhyl): move to nodeBackend

ipn/ipnlocal/serve.go

@@ -560,68 +560,14 @@ func (b *LocalBackend) tcpHandlerForVIPService(dstAddr, srcAddr netip.AddrPort)
 		return nil
 	}
 
-	if tcph.HTTPS() || tcph.HTTP() {
-		hs := &http.Server{
-			Handler: http.HandlerFunc(b.serveWebHandler),
-			BaseContext: func(_ net.Listener) context.Context {
-				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
-					SrcAddr:       srcAddr,
-					ForVIPService: dstSvc,
-					DestPort:      dport,
-				})
-			},
-		}
-		if tcph.HTTPS() {
-			// TODO(kevinliang10): just leaving this TLS cert creation as if we don't have other
-			// hostnames, but for services this getTLSServeCetForPort will need a version that also take
-			// in the hostname. How to store the TLS cert is still being discussed.
-			hs.TLSConfig = &tls.Config{
-				GetCertificate: b.getTLSServeCertForPort(dport, dstSvc),
-			}
-			return func(c net.Conn) error {
-				return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "")
-			}
-		}
-
-		return func(c net.Conn) error {
-			return hs.Serve(netutil.NewOneConnListener(c, nil))
-		}
-	}
-
-	if backDst := tcph.TCPForward(); backDst != "" {
-		return func(conn net.Conn) error {
-			defer conn.Close()
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			backConn, err := b.dialer.SystemDial(ctx, "tcp", backDst)
-			cancel()
-			if err != nil {
-				b.logf("localbackend: failed to TCP proxy port %v (from %v) to %s: %v", dport, srcAddr, backDst, err)
-				return nil
-			}
-			defer backConn.Close()
-			if sni := tcph.TerminateTLS(); sni != "" {
-				conn = tls.Server(conn, &tls.Config{
-					GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-						ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-						defer cancel()
-						pair, err := b.GetCertPEM(ctx, sni)
-						if err != nil {
-							return nil, err
-						}
-						cert, err := tls.X509KeyPair(pair.CertPEM, pair.KeyPEM)
-						if err != nil {
-							return nil, err
-						}
-						return &cert, nil
-					},
-				})
-			}
-
-			return b.forwardTCPWithProxyProtocol(conn, backConn, tcph.ProxyProtocol(), srcAddr, dport, backDst)
-		}
-	}
-
-	return nil
+	// TODO(kevinliang10): just leaving this TLS cert creation as if we don't have other
+	// hostnames, but for services this getTLSServeCetForPort will need a version that also take
+	// in the hostname. How to store the TLS cert is still being discussed.
+	return b.tcpHandlerForServeTCP(tcph, dport, srcAddr, &serveHTTPContext{
+		SrcAddr:       srcAddr,
+		ForVIPService: dstSvc,
+		DestPort:      dport,
+	}, dstSvc)
 }
 
 // tcpHandlerForServe returns a handler for a TCP connection to be served via
@@ -641,21 +587,24 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort,
 		return nil
 	}
 
+	return b.tcpHandlerForServeTCP(tcph, dport, srcAddr, &serveHTTPContext{
+		Funnel:   f,
+		SrcAddr:  srcAddr,
+		DestPort: dport,
+	}, "")
+}
+
+func (b *LocalBackend) tcpHandlerForServeTCP(tcph ipn.TCPPortHandlerView, dport uint16, srcAddr netip.AddrPort, httpCtx *serveHTTPContext, forVIPService tailcfg.ServiceName) func(net.Conn) error {
 	if tcph.HTTPS() || tcph.HTTP() {
 		hs := &http.Server{
 			Handler: http.HandlerFunc(b.serveWebHandler),
 			BaseContext: func(_ net.Listener) context.Context {
-				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
-					Funnel:   f,
-					SrcAddr:  srcAddr,
-					DestPort: dport,
-				})
+				c := *httpCtx
+				return serveHTTPContextKey.WithValue(context.Background(), &c)
 			},
 		}
 		if tcph.HTTPS() {
-			hs.TLSConfig = &tls.Config{
-				GetCertificate: b.getTLSServeCertForPort(dport, ""),
-			}
+			hs.TLSConfig = b.serveTLSConfig(b.getTLSServeCertForPort(dport, forVIPService), serveTLSNextProtos())
 			return func(c net.Conn) error {
 				return hs.ServeTLS(netutil.NewOneConnListener(c, nil), "", "")
 			}
@@ -678,21 +627,22 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort,
 			}
 			defer backConn.Close()
 			if sni := tcph.TerminateTLS(); sni != "" {
-				conn = tls.Server(conn, &tls.Config{
-					GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-						ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-						defer cancel()
-						pair, err := b.GetCertPEM(ctx, sni)
-						if err != nil {
-							return nil, err
-						}
-						cert, err := tls.X509KeyPair(pair.CertPEM, pair.KeyPEM)
-						if err != nil {
-							return nil, err
-						}
-						return &cert, nil
-					},
-				})
+				conn = tls.Server(conn, b.serveTLSConfig(func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+					if cert, ok := b.getACMETLSALPNCert(hi); ok {
+						return cert, nil
+					}
+					ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+					defer cancel()
+					pair, err := b.GetCertPEM(ctx, sni)
+					if err != nil {
+						return nil, err
+					}
+					cert, err := tls.X509KeyPair(pair.CertPEM, pair.KeyPEM)
+					if err != nil {
+						return nil, err
+					}
+					return &cert, nil
+				}, nil))
 			}
 
 			// TODO(bradfitz): do the RegisterIPPortIdentity and
@@ -1311,6 +1261,9 @@ func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService tailcfg
 		if hi == nil || hi.ServerName == "" {
 			return nil, errors.New("no SNI ServerName")
 		}
+		if cert, ok := b.getACMETLSALPNCert(hi); ok {
+			return cert, nil
+		}
 		_, ok := b.webServerConfig(hi.ServerName, forVIPService, port)
 		if !ok {
 			return nil, errors.New("no webserver configured for name/port")
@@ -1330,6 +1283,46 @@ func (b *LocalBackend) getTLSServeCertForPort(port uint16, forVIPService tailcfg
 	}
 }
 
+// serveTLSConfig returns the TLS configuration used by Serve and TCP-forwarded
+// TLS listeners. nextProtos is the ALPN list to advertise for normal
+// handshakes; it should be serveTLSNextProtos for HTTPS Serve and nil for
+// TLS-terminated TCP forwarding where we don't know the backend protocol.
+// During an ACME tls-alpn-01 renewal, GetConfigForClient clones the base config
+// and temporarily prepends acme-tls/1, but only for the exact SNI with a pending
+// challenge certificate. This keeps ordinary Serve traffic from advertising
+// ACME support and lets Go's TLS stack negotiate the challenge protocol before
+// GetCertificate is called.
+func (b *LocalBackend) serveTLSConfig(getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error), nextProtos []string) *tls.Config {
+	base := &tls.Config{
+		GetCertificate: getCert,
+		NextProtos:     nextProtos,
+	}
+	base.GetConfigForClient = func(hi *tls.ClientHelloInfo) (*tls.Config, error) {
+		var nextProtos []string
+		if proto, ok := b.getACMETLSALPNProto(hi); ok {
+			b.logf("serve: accepting ACME tls-alpn-01 challenge for %q", hi.ServerName)
+			nextProtos = append(nextProtos, proto)
+		}
+		if len(nextProtos) == 0 {
+			return nil, nil
+		}
+		cfg := base.Clone()
+		cfg.NextProtos = append(nextProtos, base.NextProtos...)
+		return cfg, nil
+	}
+	return base
+}
+
+func (b *LocalBackend) hasFunnelForHostPort(host string, port uint16) bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if !b.serveConfig.Valid() {
+		return false
+	}
+	hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(port))))
+	return b.serveConfig.HasFunnelForTarget(hp)
+}
+
 // setServeProxyHandlersLocked ensures there is an http proxy handler for each
 // backend specified in serveConfig. It expects serveConfig to be valid and
 // up-to-date, so should be called after reloadServeConfigLocked.

ipn/ipnlocal/serve_disabled.go

@@ -28,6 +28,9 @@ type funnelFlow = struct{
 
 func (*LocalBackend) hasIngressEnabledLocked() bool         { return false }
 func (*LocalBackend) shouldWireInactiveIngressLocked() bool { return false }
+func (*LocalBackend) hasFunnelForHostPort(host string, port uint16) bool {
+	return false
+}
 
 func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcfg.VIPService {
 	return nil