diff --git a/cmd/k8s-operator/e2e/helpers.go b/cmd/k8s-operator/e2e/helpers.go new file mode 100644 index 000000000..e01821c23 --- /dev/null +++ b/cmd/k8s-operator/e2e/helpers.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "crypto/rand" + "crypto/tls" + "fmt" + "net/http" + "strings" + "time" + + "tailscale.com/tsnet" +) + +func generateName(prefix string) string { + return fmt.Sprintf("%s-%s", prefix, strings.ToLower(rand.Text())) +} + +// newHTTPClient returns a HTTP client for the given tailnet client. +// When running against devcontrol, trusts Pebble testCAs. +func newHTTPClient(cl *tsnet.Server) *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: testCAs}, + DialContext: cl.Dial, + }, + } +} diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index a136d2ad3..4eb813a77 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -7,65 +7,37 @@ "context" "fmt" "net/http" + "strings" "testing" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" - "tailscale.com/cmd/testwrapper/flakytest" kube "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/httpm" ) // See [TestMain] for test requirements. -func TestIngress(t *testing.T) { - flakytest.Mark(t, "https://github.com/tailscale/corp/issues/37533") +func TestL3Ingress(t *testing.T) { if tnClient == nil { - t.Skip("TestIngress requires a working tailnet client") + t.Skip("TestL3Ingress requires a working tailnet client") } // Apply nginx - createAndCleanup(t, kubeClient, - &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", - Namespace: "default", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx", - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: new(int32(1)), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": "nginx", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "nginx", - Image: "nginx", - }, - }, - }, - }, - }, - }) + createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) // Apply service to expose it as ingress + name := generateName("test-ingress") svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "default", + Name: name, + Namespace: ns, Annotations: map[string]string{ "tailscale.com/expose": "true", }, @@ -86,7 +58,7 @@ func TestIngress(t *testing.T) { createAndCleanup(t, kubeClient, svc) if err := tstest.WaitFor(time.Minute, func() error { - maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err } @@ -100,24 +72,364 @@ func TestIngress(t *testing.T) { t.Fatalf("error waiting for the Service to become Ready: %v", err) } + // Get the DNS name for the Service from the associated Secret. + var fqdn string + if err := tstest.WaitFor(time.Minute, func() error { + var secrets corev1.SecretList + if err := kubeClient.List(t.Context(), &secrets, + client.InNamespace("tailscale"), + client.MatchingLabels{ + "tailscale.com/parent-resource": name, + "tailscale.com/parent-resource-ns": ns, + }, + ); err != nil { + return err + } + if len(secrets.Items) == 0 { + return fmt.Errorf("Service not ready yet") + } + fqdn = strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") + if fqdn != "" { + t.Log("Got DNS name for Service") + return nil + } + return fmt.Errorf("device FQDN not set yet") + }); err != nil { + t.Fatalf("error waiting for DNS Name for Service: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", fqdn)); err != nil { + t.Fatal(err) + } +} + +func TestL3HAIngress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL3HAIngress requires a working tailnet client") + } + + // Apply nginx. + createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + + // Create an ingress ProxyGroup. + createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + }) + + // Apply a Service to expose nginx via the ProxyGroup. + name := generateName("test-ingress") + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Annotations: map[string]string{ + "tailscale.com/proxy-group": "ingress", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + Port: 80, + }, + }, + }, + } + createAndCleanup(t, kubeClient, svc) + + var svcIPv4 string + forceReconcile := triggerReconcile(t, + client.ObjectKey{Namespace: ns, Name: name}, + &corev1.Service{}, 30*time.Second) + + // Wait for Service to be ready + if err := tstest.WaitFor(5*time.Minute, func() error { + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, name)} + forceReconcile() + if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { + return err + } + for _, cond := range maybeReadySvc.Status.Conditions { + if cond.Type == string(tsapi.IngressSvcConfigured) && cond.Status == metav1.ConditionTrue { + if len(maybeReadySvc.Status.LoadBalancer.Ingress) == 0 { + return fmt.Errorf("Service does not have an IP assigned yet") + } + svcIPv4 = maybeReadySvc.Status.LoadBalancer.Ingress[0].IP + t.Log("Service is ready") + return nil + } + } + return fmt.Errorf("Service is not ready yet") + }); err != nil { + t.Fatalf("error waiting for the Service to become ready: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", svcIPv4)); err != nil { + t.Fatal(err) + } +} + +func TestL7Ingress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL7Ingress requires a working tailnet client") + } + + // Apply nginx Deployment and Service. + createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Apply Ingress to expose nginx. + name := generateName("test-ingress") + ingress := l7Ingress(ns, name, map[string]string{}) + createAndCleanup(t, kubeClient, ingress) + + t.Log("Waiting for the Ingress to be ready...") + + hostname, err := waitForIngressHostname(t, ns, name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func TestL7HAIngress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL7HAIngress requires a working tailnet client") + } + + // Apply nginx Deployment and Service. + createAndCleanup(t, kubeClient, nginxDeployment(ns, "nginx")) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create ProxyGroup that the Ingress will reference. + createAndCleanup(t, kubeClient, &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + }) + + // Apply Ingress to expose nginx. + name := generateName("test-ingress") + ingress := l7Ingress(ns, name, map[string]string{"tailscale.com/proxy-group": "ingress"}) + createAndCleanup(t, kubeClient, ingress) + + t.Log("Waiting for the Ingress to be ready...") + + hostname, err := waitForIngressHostname(t, ns, name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func l7Ingress(namespace, name string, annotations map[string]string) *networkingv1.Ingress { + ingress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: annotations, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{name}}, + }, + Rules: []networkingv1.IngressRule{ + { + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: new(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "nginx", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + return ingress +} + +func nginxDeployment(namespace, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } +} + +// triggerReconcile triggers an expected reconcile for the given object if +// none occurs. This is needed when running some tests against devcontrol, +// where the final change that should trigger a reconcile does not always do so. +// This has not been reproducible in a real tailnet environment, so a +// workaround that runs only when using devcontrol is acceptable. +func triggerReconcile(t testing.TB, key client.ObjectKey, obj client.Object, after time.Duration) func() { + if !*fDevcontrol { + return func() {} + } + triggerAt := time.Now().Add(after) + var triggered bool + return func() { + if triggered || !time.Now().After(triggerAt) { + return + } + if err := kubeClient.Get(t.Context(), key, obj); err != nil { + t.Logf("failed to get %s: %v", key, err) + return + } + ann := obj.GetAnnotations() + if ann == nil { + ann = map[string]string{} + } + ann["tailscale.com/trigger-reconcile"] = "true" + obj.SetAnnotations(ann) + if err := kubeClient.Update(t.Context(), obj); err != nil { + t.Logf("failed to update %s: %v", key, err) + return + } + triggered = true + } +} + +func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) error { + t.Helper() var resp *http.Response if err := tstest.WaitFor(time.Minute, func() error { - // TODO(tomhjp): Get the tailnet DNS name from the associated secret instead. - // If we are not the first tailnet node with the requested name, we'll get - // a -N suffix. - req, err := http.NewRequest(httpm.GET, fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name), nil) + req, err := http.NewRequest(httpm.GET, url, nil) if err != nil { return err } - ctx, cancel := context.WithTimeout(t.Context(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() - resp, err = tnClient.HTTPClient().Do(req.WithContext(ctx)) - return err + resp, err = httpClient.Do(req.WithContext(ctx)) + if err != nil { + return err + } + resp.Body.Close() + return nil }); err != nil { - t.Fatalf("error trying to reach Service: %v", err) + return fmt.Errorf("error trying to reach %s: %w", url, err) } - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v; response body s", resp.StatusCode) + return fmt.Errorf("unexpected status from %s: %d", url, resp.StatusCode) } + return nil +} + +func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) { + t.Helper() + var hostname string + forceReconcile := triggerReconcile(t, + client.ObjectKey{Namespace: namespace, Name: name}, + &networkingv1.Ingress{}, 30*time.Second) + + if err := tstest.WaitFor(5*time.Minute, func() error { + forceReconcile() + ing := &networkingv1.Ingress{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{ + Namespace: namespace, Name: name, + }, ing); err != nil { + return err + } + if len(ing.Status.LoadBalancer.Ingress) == 0 || + ing.Status.LoadBalancer.Ingress[0].Hostname == "" { + return fmt.Errorf("Ingress not ready yet") + } + hostname = ing.Status.LoadBalancer.Ingress[0].Hostname + t.Log("Ingress is ready") + return nil + }); err != nil { + return "", fmt.Errorf("Ingress %s/%s never got a hostname: %w", namespace, name, err) + } + return hostname, nil } diff --git a/cmd/k8s-operator/e2e/proxy_test.go b/cmd/k8s-operator/e2e/proxy_test.go index 2d4fa53cc..3caf1c91d 100644 --- a/cmd/k8s-operator/e2e/proxy_test.go +++ b/cmd/k8s-operator/e2e/proxy_test.go @@ -4,10 +4,8 @@ package e2e import ( - "crypto/tls" "encoding/json" "fmt" - "net/http" "testing" "time" @@ -61,15 +59,7 @@ func TestProxy(t *testing.T) { Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)), } proxyCl, err := client.New(proxyCfg, client.Options{ - HTTPClient: &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: testCAs, - }, - DialContext: tnClient.Dial, - }, - }, + HTTPClient: newHTTPClient(tnClient), }) if err != nil { t.Fatal(err) diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index e3d7ed89b..642ee57ec 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -130,7 +130,7 @@ func runTests(m *testing.M) (int, error) { if err := kindProvider.Create(kindClusterName, cluster.CreateWithWaitForReady(5*time.Minute), cluster.CreateWithKubeconfigPath(kubeconfig), - cluster.CreateWithNodeImage("kindest/node:v1.30.0"), + cluster.CreateWithNodeImage("kindest/node:v1.35.0"), ); err != nil { return 0, fmt.Errorf("failed to create kind cluster: %w", err) } @@ -321,7 +321,6 @@ func runTests(m *testing.M) (int, error) { // An access token will last for an hour which is plenty of time for // the tests to run. No need for token refresh logic. tsClient = tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken)) - tsClient.BaseURL = "http://localhost:31544" } var ossTag string @@ -389,6 +388,15 @@ func runTests(m *testing.M) (int, error) { if err != nil { return 0, fmt.Errorf("failed to load helm chart: %w", err) } + extraEnv := []map[string]any{ + { + "name": "K8S_PROXY_IMAGE", + "value": "local/k8s-proxy:" + ossTag, + }, + } + if *fDevcontrol { + extraEnv = append(extraEnv, map[string]any{"name": "TS_DEBUG_ACME_DIRECTORY_URL", "value": "https://pebble:14000/dir"}) + } values := map[string]any{ "loginServer": clusterLoginServer, "oauth": map[string]any{ @@ -399,17 +407,8 @@ func runTests(m *testing.M) (int, error) { "mode": "true", }, "operatorConfig": map[string]any{ - "logging": "debug", - "extraEnv": []map[string]any{ - { - "name": "K8S_PROXY_IMAGE", - "value": "local/k8s-proxy:" + ossTag, - }, - { - "name": "TS_DEBUG_ACME_DIRECTORY_URL", - "value": "https://pebble:14000/dir", - }, - }, + "logging": "debug", + "extraEnv": extraEnv, "image": map[string]any{ "repo": "local/k8s-operator", "tag": ossTag, @@ -539,6 +538,15 @@ func tagForRepo(dir string) (string, error) { } func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl client.Client) error { + var env []tsapi.Env + if *fDevcontrol { + env = []tsapi.Env{ + { + Name: "TS_DEBUG_ACME_DIRECTORY_URL", + Value: "https://pebble:14000/dir", + }, + } + } pc := &tsapi.ProxyClass{ TypeMeta: metav1.TypeMeta{ APIVersion: tsapi.SchemeGroupVersion.String(), @@ -555,6 +563,7 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c }, TailscaleContainer: &tsapi.Container{ ImagePullPolicy: "IfNotPresent", + Env: env, }, }, },