cmd/k8s-operator/e2e: use Let's Encrypt staging environment (#20666)

When running k8s operator e2e tests against real tailnets, use Let's
Encrypt's staging environment to avoid production rate limits.

Updates tailscale/corp#45571

Signed-off-by: Becky Pauley <becky@tailscale.com>

cmd/k8s-operator/e2e/certs/letsencrypt-stg-root-x1.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFmDCCA4CgAwIBAgIQU9C87nMpOIFKYpfvOHFHFDANBgkqhkiG9w0BAQsFADBm
+MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
+aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
+ZWFyIFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowZjELMAkGA1UE
+BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
+YXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRlbmQgUGVhciBYMTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdDTa1QgGBWSYkyMhsc
+ZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPWnL++fgehT0FbRHZg
+jOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigkkmx8OiCO68a4QXg4
+wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZGTIf/oRt2/c+dYmD
+oaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6VP19sTGy3yfqK5tPt
+TdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkLYC0Ft2cYUyHtkstO
+fRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2UPQFxmWFRQnFjaq6
+rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/2dBZKmJqxHkxCuOQ
+FjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRMEeOXUYvbV4lqfCf8
+mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEmQWUOTWIoDQ5FOia/
+GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eVEGOIpn26bW5LKeru
+mJxa/CFBaKi4bRvmdJRLAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+Af8EBTADAQH/MB0GA1UdDgQWBBS182Xy/rAKkh/7PH3zRKCsYyXDFDANBgkqhkiG
+9w0BAQsFAAOCAgEAncDZNytDbrrVe68UT6py1lfF2h6Tm2p8ro42i87WWyP2LK8Y
+nLHC0hvNfWeWmjZQYBQfGC5c7aQRezak+tHLdmrNKHkn5kn+9E9LCjCaEsyIIn2j
+qdHlAkepu/C3KnNtVx5tW07e5bvIjJScwkCDbP3akWQixPpRFAsnP+ULx7k0aO1x
+qAeaAhQ2rgo1F58hcflgqKTXnpPM02intVfiVVkX5GXpJjK5EoQtLceyGOrkxlM/
+sTPq4UrnypmsqSagWV3HcUlYtDinc+nukFk6eR4XkzXBbwKajl0YjztfrCIHOn5Q
+CJL6TERVDbM/aAPly8kJ1sWGLuvvWYzMYgLzDul//rUF10gEMWaXVZV51KpS9DY/
+5CunuvCXmEQJHo7kGcViT7sETn6Jz9KOhvYcXkJ7po6d93A/jy4GKPIPnsKKNEmR
+xUuXY4xRdh45tMJnLTUDdC9FIU0flTeO9/vNpVA8OPU1i14vCz+MU8KX1bV3GXm/
+fxlB7VBBjX9v5oUep0o/j68R/iDlCOM4VVfRa8gX6T2FU7fNdatvGro7uQzIvWof
+gN9WUwCbEMBy/YhBSrXycKA8crgGg3x1mIsopn88JKwmMBa68oS7EHM9w7C4y71M
+7DiA+/9Qdp9RBWJpTS9i/mDnJg1xvo8Xz49mrrgfmcAXTCJqXi24NatI3Oc=
+-----END CERTIFICATE-----

cmd/k8s-operator/e2e/helpers.go

@@ -26,7 +26,8 @@ func generateName(prefix string) string {
 }
 
 // newHTTPClient returns a HTTP client for the given tailnet client.
-// When running against devcontrol, trusts Pebble testCAs.
+// When running against devcontrol, trusts Pebble testCAs. Otherwise,
+// trusts Let's Encrypt staging testCA.
 func newHTTPClient(cl *tsnet.Server) *http.Client {
 	return &http.Client{
 		Timeout: 10 * time.Second,

cmd/k8s-operator/e2e/setup.go

@@ -90,9 +90,16 @@
 	//go:embed certs/pebble.minica.crt
 	pebbleMiniCACert []byte
 
-	// Either nil (system) or pebble CAs if pebble is deployed for devcontrol.
-	// pebble has a static "mini" CA that its ACME directory URL serves a cert
-	// from, and also dynamically generates a different CA for issuing certs.
+	// Let's Encrypt staging environment root "Pretend Pear X1", used when
+	// running against real tailnets.
+	// Available from https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem
+	//go:embed certs/letsencrypt-stg-root-x1.pem
+	leStagingRootX1 []byte
+
+	// Either  pebble CAs (if pebble is deployed for devcontrol) or
+	// Let's Encrypt staging when running against real tailnets).
+	// pebble has a static "mini" CA that its ACME directory URL serves a cert from,
+	// and also dynamically generates a different CA for issuing certs.
 	testCAs *x509.CertPool
 
 	//go:embed acl.hujson
@@ -185,6 +192,7 @@ func runTests(m *testing.M) (int, error) {
 		certsDir                           = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
 		secondClientID, secondClientSecret string                        // OAuth client for the second tailnet (for the operator to use).
 	)
+	testCAs = x509.NewCertPool()
 	if *fDevcontrol {
 		// Deploy pebble and get its certs.
 		if err = applyPebbleResources(ctx, kubeClient); err != nil {
@@ -200,7 +208,6 @@ func runTests(m *testing.M) (int, error) {
 			return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err)
 		}
 
-		testCAs = x509.NewCertPool()
 		if ok := testCAs.AppendCertsFromPEM(pebbleMiniCACert); !ok {
 			return 0, fmt.Errorf("failed to parse pebble minica cert")
 		}
@@ -357,6 +364,10 @@ func runTests(m *testing.M) (int, error) {
 		}
 
 	} else {
+		if ok := testCAs.AppendCertsFromPEM(leStagingRootX1); !ok {
+			return 0, fmt.Errorf("failed to parse Let's Encrypt staging root")
+		}
+
 		clientSecret = os.Getenv("TS_API_CLIENT_SECRET")
 		if clientSecret == "" {
 			return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
@@ -456,6 +467,8 @@ func runTests(m *testing.M) (int, error) {
 	}
 	if *fDevcontrol {
 		extraEnv = append(extraEnv, map[string]any{"name": "TS_DEBUG_ACME_DIRECTORY_URL", "value": "https://pebble:14000/dir"})
+	} else {
+		extraEnv = append(extraEnv, map[string]any{"name": "TS_DEBUG_ACME_DIRECTORY_URL", "value": "https://acme-staging-v02.api.letsencrypt.org/directory"})
 	}
 	values := map[string]any{
 		"loginServer": clusterLoginServer,
@@ -685,6 +698,7 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c
 			Name: "default",
 		},
 		Spec: tsapi.ProxyClassSpec{
+			UseLetsEncryptStagingEnvironment: !*fDevcontrol,
 			StatefulSet: &tsapi.StatefulSet{
 				Pod: &tsapi.Pod{
 					TailscaleInitContainer: &tsapi.Container{