mirror/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2026-02-07 14:32:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator,k8s-operator: Allow the use of multiple tailnets (#18344)

Browse Source 
This commit contains  the implementation of multi-tailnet support within the Kubernetes Operator

Each of our custom resources now expose the `spec.tailnet` field. This field is a string that must match the name of an existing `Tailnet` resource. A `Tailnet` resource looks like this:

```yaml
apiVersion: tailscale.com/v1alpha1
kind: Tailnet
metadata:
  name: example  # This is the name that must be referenced by other resources
spec:
  credentials:
    secretName: example-oauth
```

Each `Tailnet` references a `Secret` resource that contains a set of oauth credentials. This secret must be created in the same namespace as the operator:

```yaml
apiVersion: v1
kind: Secret
metadata:
  name: example-oauth # This is the name that's referenced by the Tailnet resource.
  namespace: tailscale
stringData:
  client_id: "client-id"
  client_secret: "client-secret"
```

When created, the operator performs a basic check that the oauth client has access to all required scopes. This is done using read actions on devices, keys & services. While this doesn't capture a missing "write" permission, it catches completely missing permissions. Once this check passes, the `Tailnet` moves into a ready state and can be referenced. Attempting to use a `Tailnet` in a non-ready state will stall the deployment of `Connector`s, `ProxyGroup`s and `Recorder`s until the `Tailnet` becomes ready.

The `spec.tailnet` field informs the operator that a `Connector`, `ProxyGroup`, or `Recorder` must be given an auth key generated using the specified oauth client. For backwards compatibility, the set of credentials the operator is configured with are considered the default. That is, where `spec.tailnet` is not set, the resource will be deployed in the same tailnet as the operator. 

Updates https://github.com/tailscale/corp/issues/34561
This commit is contained in:
David Bond
2026-01-21 12:35:44 +00:00
committed by GitHub
parent e30626c480
commit 2cb86cf65e
31 changed files with 1737 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/k8s-operator/connector.go
View File

@@ -25,6 +25,7 @@
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -207,6 +208,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
ProxyClassName: proxyClass,
proxyType: proxyTypeConnector,
LoginServer: a.ssr.loginServer,
Tailnet: cn.Spec.Tailnet,
}
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
@@ -276,7 +278,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
}
func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
if done, err := a.ssr.Cleanup(ctx, cn.Spec.Tailnet, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
} else if !done {
logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")

6
cmd/k8s-operator/depaware.txt
View File

@@ -725,7 +725,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/utils/net from k8s.io/apimachinery/pkg/util/net+
k8s.io/utils/ptr from k8s.io/client-go/tools/cache+
k8s.io/utils/trace from k8s.io/client-go/tools/cache
sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator
sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator+
sigs.k8s.io/controller-runtime/pkg/cache from sigs.k8s.io/controller-runtime/pkg/cluster+
sigs.k8s.io/controller-runtime/pkg/cache/internal from sigs.k8s.io/controller-runtime/pkg/cache
sigs.k8s.io/controller-runtime/pkg/certwatcher from sigs.k8s.io/controller-runtime/pkg/metrics/server+
@@ -821,10 +821,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+
tailscale.com/ipn/store/kubestore from tailscale.com/cmd/k8s-operator
tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+
tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator+
tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1
tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+
tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet
tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy
tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording
tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+

1
cmd/k8s-operator/deploy/chart/templates/.gitignore vendored
View File

@@ -8,3 +8,4 @@
/proxyclass.yaml
/proxygroup.yaml
/recorder.yaml
/tailnet.yaml

3
cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
View File

@@ -37,6 +37,9 @@ rules:
- apiGroups: ["tailscale.com"]
resources: ["dnsconfigs", "dnsconfigs/status"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["tailscale.com"]
resources: ["tailnets", "tailnets/status"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["tailscale.com"]
resources: ["recorders", "recorders/status"]
verbs: ["get", "list", "watch", "update"]

8
cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
View File

@@ -181,6 +181,14 @@ spec:
items:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
tailnet:
description: |-
Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- rule: self == oldSelf
message: Connector tailnet is immutable
x-kubernetes-validations:
- rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
message: A Connector needs to have at least one of exit node, subnet router or app connector configured.

8
cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml
View File

@@ -139,6 +139,14 @@ spec:
items:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
tailnet:
description: |-
Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- rule: self == oldSelf
message: ProxyGroup tailnet is immutable
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.

8
cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml
View File

@@ -1680,6 +1680,14 @@ spec:
items:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
tailnet:
description: |-
Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- rule: self == oldSelf
message: Recorder tailnet is immutable
x-kubernetes-validations:
- rule: '!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))'
message: S3 storage must be used when deploying multiple Recorder replicas

141
cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml Normal file
View File

@@ -0,0 +1,141 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
name: tailnets.tailscale.com
spec:
group: tailscale.com
names:
kind: Tailnet
listKind: TailnetList
plural: tailnets
shortNames:
- tn
singular: tailnet
scope: Cluster
versions:
- additionalPrinterColumns:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: Status of the deployed Tailnet resources.
jsonPath: .status.conditions[?(@.type == "TailnetReady")].reason
name: Status
type: string
name: v1alpha1
schema:
openAPIV3Schema:
type: object
required:
- metadata
- spec
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: |-
Spec describes the desired state of the Tailnet.
More info:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
type: object
required:
- credentials
properties:
credentials:
description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
type: object
required:
- secretName
properties:
secretName:
description: |-
The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
"client_secret".
type: string
loginUrl:
description: URL of the control plane to be used by all resources managed by the operator using this Tailnet.
type: string
status:
description: |-
Status describes the status of the Tailnet. This is set
and managed by the Tailscale operator.
type: object
properties:
conditions:
type: array
items:
description: Condition contains details for one aspect of the current state of this API Resource.
type: object
required:
- lastTransitionTime
- message
- reason
- status
- type
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
type: string
format: date-time
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
type: string
maxLength: 32768
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
type: integer
format: int64
minimum: 0
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
type: string
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
status:
description: status of the condition, one of True, False, Unknown.
type: string
enum:
- "True"
- "False"
- Unknown
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
type: string
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
served: true
storage: true
subresources:
status: {}

176
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@@ -206,6 +206,14 @@ spec:
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type: string
type: array
tailnet:
description: |-
Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- message: Connector tailnet is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
@@ -3135,6 +3143,14 @@ spec:
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type: string
type: array
tailnet:
description: |-
Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- message: ProxyGroup tailnet is immutable
rule: self == oldSelf
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
@@ -4950,6 +4966,14 @@ spec:
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type: string
type: array
tailnet:
description: |-
Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
type: string
x-kubernetes-validations:
- message: Recorder tailnet is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: S3 storage must be used when deploying multiple Recorder replicas
@@ -5059,6 +5083,148 @@ spec:
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
name: tailnets.tailscale.com
spec:
group: tailscale.com
names:
kind: Tailnet
listKind: TailnetList
plural: tailnets
shortNames:
- tn
singular: tailnet
scope: Cluster
versions:
- additionalPrinterColumns:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
- description: Status of the deployed Tailnet resources.
jsonPath: .status.conditions[?(@.type == "TailnetReady")].reason
name: Status
type: string
name: v1alpha1
schema:
openAPIV3Schema:
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: |-
Spec describes the desired state of the Tailnet.
More info:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
properties:
credentials:
description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
properties:
secretName:
description: |-
The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
"client_secret".
type: string
required:
- secretName
type: object
loginUrl:
description: URL of the control plane to be used by all resources managed by the operator using this Tailnet.
type: string
required:
- credentials
type: object
status:
description: |-
Status describes the status of the Tailnet. This is set
and managed by the Tailscale operator.
properties:
conditions:
items:
description: Condition contains details for one aspect of the current state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
type: object
required:
- metadata
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@@ -5141,6 +5307,16 @@ rules:
- list
- watch
- update
- apiGroups:
- tailscale.com
resources:
- tailnets
- tailnets/status
verbs:
- get
- list
- watch
- update
- apiGroups:
- tailscale.com
resources:

3
cmd/k8s-operator/generate/main.go
View File

@@ -26,12 +26,14 @@
dnsConfigCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
recorderCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
proxyGroupCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
tailnetCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml"
helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates"
connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml"
proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
dnsConfigCRDHelmTemplatePath = helmTemplatesPath + "/dnsconfig.yaml"
recorderCRDHelmTemplatePath = helmTemplatesPath + "/recorder.yaml"
proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
tailnetCRDHelmTemplatePath = helmTemplatesPath + "/tailnet.yaml"
helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
helmConditionalEnd = "{{- end -}}"
@@ -154,6 +156,7 @@ func generate(baseDir string) error {
{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
{recorderCRDPath, recorderCRDHelmTemplatePath},
{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
{tailnetCRDPath, tailnetCRDHelmTemplatePath},
} {
if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)

2
cmd/k8s-operator/ingress.go
View File

@@ -102,7 +102,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
return nil
}
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil {
if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil {
return fmt.Errorf("failed to cleanup: %w", err)
} else if !done {
logger.Debugf("cleanup not done yet, waiting for next reconcile")

12
cmd/k8s-operator/operator.go
View File

@@ -54,6 +54,7 @@
"tailscale.com/ipn/store/kubestore"
apiproxy "tailscale.com/k8s-operator/api-proxy"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailnet"
"tailscale.com/kube/kubetypes"
"tailscale.com/tsnet"
"tailscale.com/tstime"
@@ -325,6 +326,17 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create manager: %v", err)
}
tailnetOptions := tailnet.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
Clock: tstime.DefaultClock{},
Logger: opts.log,
}
if err = tailnet.NewReconciler(tailnetOptions).Register(mgr); err != nil {
startlog.Fatalf("could not register tailnet reconciler: %v", err)
}
svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc"))
// If a ProxyClass changes, enqueue all Services labeled with that

68
cmd/k8s-operator/proxygroup.go
View File

@@ -49,11 +49,12 @@
)
const (
reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
reasonProxyGroupReady = "ProxyGroupReady"
reasonProxyGroupAvailable = "ProxyGroupAvailable"
reasonProxyGroupCreating = "ProxyGroupCreating"
reasonProxyGroupInvalid = "ProxyGroupInvalid"
reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
reasonProxyGroupReady = "ProxyGroupReady"
reasonProxyGroupAvailable = "ProxyGroupAvailable"
reasonProxyGroupCreating = "ProxyGroupCreating"
reasonProxyGroupInvalid = "ProxyGroupInvalid"
reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable"
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
@@ -117,6 +118,23 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
} else if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err)
}
tailscaleClient := r.tsClient
if pg.Spec.Tailnet != "" {
tc, err := clientForTailnet(ctx, r.Client, r.tsNamespace, pg.Spec.Tailnet)
if err != nil {
oldPGStatus := pg.Status.DeepCopy()
nrr := &notReadyReason{
reason: reasonProxyGroupTailnetUnavailable,
message: err.Error(),
}
return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, make(map[string][]netip.AddrPort)))
}
tailscaleClient = tc
}
if markedForDeletion(pg) {
logger.Debugf("ProxyGroup is being deleted, cleaning up resources")
ix := xslices.Index(pg.Finalizers, FinalizerName)
@@ -125,7 +143,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
return reconcile.Result{}, nil
}
if done, err := r.maybeCleanup(ctx, pg); err != nil {
if done, err := r.maybeCleanup(ctx, tailscaleClient, pg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
@@ -144,7 +162,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
oldPGStatus := pg.Status.DeepCopy()
staticEndpoints, nrr, err := r.reconcilePG(ctx, pg, logger)
staticEndpoints, nrr, err := r.reconcilePG(ctx, tailscaleClient, pg, logger)
return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints))
}
@@ -152,7 +170,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
// for deletion. It is separated out from Reconcile to make a clear separation
// between reconciling the ProxyGroup, and posting the status of its created
// resources onto the ProxyGroup status field.
func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) {
func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) {
if !slices.Contains(pg.Finalizers, FinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
@@ -193,7 +211,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
}
staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass)
staticEndpoints, nrr, err := r.maybeProvision(ctx, tailscaleClient, pg, proxyClass)
if err != nil {
return nil, nrr, err
}
@@ -279,7 +297,7 @@ func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGrou
return errors.Join(errs...)
}
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) {
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) {
logger := r.logger(pg.Name)
r.mu.Lock()
r.ensureAddedToGaugeForProxyGroup(pg)
@@ -302,7 +320,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}
}
staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts)
staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, tailscaleClient, pg, proxyClass, svcToNodePorts)
if err != nil {
var selectorErr *FindStaticEndpointErr
if errors.As(err, &selectorErr) {
@@ -414,7 +432,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err)
}
if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil {
if err := r.cleanupDanglingResources(ctx, tailscaleClient, pg, proxyClass); err != nil {
return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err)
}
@@ -611,7 +629,7 @@ func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context,
// cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
// tailnet devices when the number of replicas specified is reduced.
func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error {
func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error {
logger := r.logger(pg.Name)
metadata, err := r.getNodeMetadata(ctx, pg)
if err != nil {
@@ -625,7 +643,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg
// Dangling resource, delete the config + state Secrets, as well as
// deleting the device from the tailnet.
if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil {
return err
}
if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) {
@@ -668,7 +686,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg
// maybeCleanup just deletes the device from the tailnet. All the kubernetes
// resources linked to a ProxyGroup will get cleaned up via owner references
// (which we can use because they are all in the same namespace).
func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) {
func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup) (bool, error) {
logger := r.logger(pg.Name)
metadata, err := r.getNodeMetadata(ctx, pg)
@@ -677,7 +695,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
}
for _, m := range metadata {
if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil {
return false, err
}
}
@@ -698,9 +716,9 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
return true, nil
}
func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error {
func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, tailscaleClient tsClient, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error {
logger.Debugf("deleting device %s from control", string(id))
if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil {
if err := tailscaleClient.DeleteDevice(ctx, string(id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
@@ -714,7 +732,13 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
return nil
}
func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) {
func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(
ctx context.Context,
tailscaleClient tsClient,
pg *tsapi.ProxyGroup,
proxyClass *tsapi.ProxyClass,
svcToNodePorts map[string]uint16,
) (endpoints map[string][]netip.AddrPort, err error) {
logger := r.logger(pg.Name)
endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name.
for i := range pgReplicas(pg) {
@@ -728,7 +752,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
var existingCfgSecret *corev1.Secret // unmodified copy of secret
if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
if err = r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil {
logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
existingCfgSecret = cfgSecret.DeepCopy()
} else if !apierrors.IsNotFound(err) {
@@ -742,7 +766,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
if len(tags) == 0 {
tags = r.defaultTags
}
key, err := newAuthKey(ctx, r.tsClient, tags)
key, err := newAuthKey(ctx, tailscaleClient, tags)
if err != nil {
return nil, err
}
@@ -757,7 +781,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
Namespace: r.tsNamespace,
},
}
if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) {
if err = r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) {
return nil, err
}

37
cmd/k8s-operator/sts.go
View File

@@ -107,6 +107,7 @@
letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory"
mainContainerName = "tailscale"
operatorTailnet = ""
)
var (
@@ -152,6 +153,9 @@ type tailscaleSTSConfig struct {
// HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the
// ordinal number generated by the StatefulSet.
HostnamePrefix string
// Tailnet specifies the Tailnet resource to use for producing auth keys.
Tailnet string
}
type connector struct {
@@ -194,6 +198,16 @@ func IsHTTPSEnabledOnTailnet(tsnetServer tsnetServer) bool {
// Provision ensures that the StatefulSet for the given service is running and
// up to date.
func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) {
tailscaleClient := a.tsClient
if sts.Tailnet != "" {
tc, err := clientForTailnet(ctx, a.Client, a.operatorNamespace, sts.Tailnet)
if err != nil {
return nil, err
}
tailscaleClient = tc
}
// Do full reconcile.
// TODO (don't create Service for the Connector)
hsvc, err := a.reconcileHeadlessService(ctx, logger, sts)
@@ -213,7 +227,7 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
}
sts.ProxyClass = proxyClass
secretNames, err := a.provisionSecrets(ctx, logger, sts, hsvc)
secretNames, err := a.provisionSecrets(ctx, tailscaleClient, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
}
@@ -237,7 +251,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
// Cleanup removes all resources associated that were created by Provision with
// the given labels. It returns true when all resources have been removed,
// otherwise it returns false and the caller should retry later.
func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) {
func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, tailnet string, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) {
tailscaleClient := a.tsClient
if tailnet != "" {
tc, err := clientForTailnet(ctx, a.Client, a.operatorNamespace, tailnet)
if err != nil {
logger.Errorf("failed to get tailscale client: %v", err)
return false, nil
}
tailscaleClient = tc
}
// Need to delete the StatefulSet first, and delete it with foreground
// cascading deletion. That way, the pod that's writing to the Secret will
// stop running before we start looking at the Secret's contents, and
@@ -279,7 +304,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
for _, dev := range devices {
if dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err = a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
if err = tailscaleClient.DeleteDevice(ctx, string(dev.id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
@@ -360,7 +385,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
}
func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, tailscaleClient tsClient, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
secretNames := make([]string, stsC.Replicas)
// Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling
@@ -403,7 +428,7 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
if len(tags) == 0 {
tags = a.defaultTags
}
authKey, err = newAuthKey(ctx, a.tsClient, tags)
authKey, err = newAuthKey(ctx, tailscaleClient, tags)
if err != nil {
return nil, err
}
@@ -477,7 +502,7 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z
if dev != nil && dev.id != "" {
var errResp *tailscale.ErrResponse
err = a.tsClient.DeleteDevice(ctx, string(dev.id))
err = tailscaleClient.DeleteDevice(ctx, string(dev.id))
switch {
case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound:
// This device has possibly already been deleted in the admin console. So we can ignore this

3
cmd/k8s-operator/svc.go
View File

@@ -23,6 +23,7 @@
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -167,7 +168,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
proxyTyp = proxyTypeIngressService
}
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil {
if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil {
return fmt.Errorf("failed to cleanup: %w", err)
} else if !done {
logger.Debugf("cleanup not done yet, waiting for next reconcile")

58
cmd/k8s-operator/tailnet.go Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"fmt"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"tailscale.com/internal/client/tailscale"
"tailscale.com/ipn"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
)
func clientForTailnet(ctx context.Context, cl client.Client, namespace, name string) (tsClient, error) {
var tn tsapi.Tailnet
if err := cl.Get(ctx, client.ObjectKey{Name: name}, &tn); err != nil {
return nil, fmt.Errorf("failed to get tailnet %q: %w", name, err)
}
if !operatorutils.TailnetIsReady(&tn) {
return nil, fmt.Errorf("tailnet %q is not ready", name)
}
var secret corev1.Secret
if err := cl.Get(ctx, client.ObjectKey{Name: tn.Spec.Credentials.SecretName, Namespace: namespace}, &secret); err != nil {
return nil, fmt.Errorf("failed to get Secret %q in namespace %q: %w", tn.Spec.Credentials.SecretName, namespace, err)
}
baseURL := ipn.DefaultControlURL
if tn.Spec.LoginURL != "" {
baseURL = tn.Spec.LoginURL
}
credentials := clientcredentials.Config{
ClientID: string(secret.Data["client_id"]),
ClientSecret: string(secret.Data["client_secret"]),
TokenURL: baseURL + "/api/v2/oauth/token",
}
source := credentials.TokenSource(ctx)
httpClient := oauth2.NewClient(ctx, source)
ts := tailscale.NewClient(defaultTailnet, nil)
ts.UserAgent = "tailscale-k8s-operator"
ts.HTTPClient = httpClient
ts.BaseURL = baseURL
return ts, nil
}

88
cmd/k8s-operator/tsrecorder.go
View File

@@ -42,10 +42,11 @@
)
const (
reasonRecorderCreationFailed = "RecorderCreationFailed"
reasonRecorderCreating = "RecorderCreating"
reasonRecorderCreated = "RecorderCreated"
reasonRecorderInvalid = "RecorderInvalid"
reasonRecorderCreationFailed = "RecorderCreationFailed"
reasonRecorderCreating = "RecorderCreating"
reasonRecorderCreated = "RecorderCreated"
reasonRecorderInvalid = "RecorderInvalid"
reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable"
currentProfileKey = "_current-profile"
)
@@ -84,27 +85,6 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
} else if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
}
if markedForDeletion(tsr) {
logger.Debugf("Recorder is being deleted, cleaning up resources")
ix := xslices.Index(tsr.Finalizers, FinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return reconcile.Result{}, nil
}
if done, err := r.maybeCleanup(ctx, tsr); err != nil {
return reconcile.Result{}, err
} else if !done {
logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
if err = r.Update(ctx, tsr); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
oldTSRStatus := tsr.Status.DeepCopy()
setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
@@ -119,6 +99,38 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return reconcile.Result{}, nil
}
tailscaleClient := r.tsClient
if tsr.Spec.Tailnet != "" {
tc, err := clientForTailnet(ctx, r.Client, r.tsNamespace, tsr.Spec.Tailnet)
if err != nil {
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error())
}
tailscaleClient = tc
}
if markedForDeletion(tsr) {
logger.Debugf("Recorder is being deleted, cleaning up resources")
ix := xslices.Index(tsr.Finalizers, FinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return reconcile.Result{}, nil
}
if done, err := r.maybeCleanup(ctx, tsr, tailscaleClient); err != nil {
return reconcile.Result{}, err
} else if !done {
logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
if err = r.Update(ctx, tsr); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
if !slices.Contains(tsr.Finalizers, FinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
@@ -137,7 +149,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
}
if err = r.maybeProvision(ctx, tsr); err != nil {
if err = r.maybeProvision(ctx, tailscaleClient, tsr); err != nil {
reason := reasonRecorderCreationFailed
message := fmt.Sprintf("failed creating Recorder: %s", err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
@@ -155,7 +167,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
}
func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error {
func (r *RecorderReconciler) maybeProvision(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error {
logger := r.logger(tsr.Name)
r.mu.Lock()
@@ -163,7 +175,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
gaugeRecorderResources.Set(int64(r.recorders.Len()))
r.mu.Unlock()
if err := r.ensureAuthSecretsCreated(ctx, tsr); err != nil {
if err := r.ensureAuthSecretsCreated(ctx, tailscaleClient, tsr); err != nil {
return fmt.Errorf("error creating secrets: %w", err)
}
@@ -241,13 +253,13 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
// If we have scaled the recorder down, we will have dangling state secrets
// that we need to clean up.
if err = r.maybeCleanupSecrets(ctx, tsr); err != nil {
if err = r.maybeCleanupSecrets(ctx, tailscaleClient, tsr); err != nil {
return fmt.Errorf("error cleaning up Secrets: %w", err)
}
var devices []tsapi.RecorderTailnetDevice
for replica := range replicas {
dev, ok, err := r.getDeviceInfo(ctx, tsr.Name, replica)
dev, ok, err := r.getDeviceInfo(ctx, tailscaleClient, tsr.Name, replica)
switch {
case err != nil:
return fmt.Errorf("failed to get device info: %w", err)
@@ -312,7 +324,7 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts
return nil
}
func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi.Recorder) error {
func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error {
options := []client.ListOption{
client.InNamespace(r.tsNamespace),
client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)),
@@ -354,7 +366,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi
var errResp *tailscale.ErrResponse
r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID)
err = r.tsClient.DeleteDevice(ctx, string(devicePrefs.Config.NodeID))
err = tailscaleClient.DeleteDevice(ctx, string(devicePrefs.Config.NodeID))
switch {
case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound:
// This device has possibly already been deleted in the admin console. So we can ignore this
@@ -375,7 +387,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi
// maybeCleanup just deletes the device from the tailnet. All the kubernetes
// resources linked to a Recorder will get cleaned up via owner references
// (which we can use because they are all in the same namespace).
func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) {
func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tailscaleClient tsClient) (bool, error) {
logger := r.logger(tsr.Name)
var replicas int32 = 1
@@ -399,7 +411,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
nodeID := string(devicePrefs.Config.NodeID)
logger.Debugf("deleting device %s from control", nodeID)
if err = r.tsClient.DeleteDevice(ctx, nodeID); err != nil {
if err = tailscaleClient.DeleteDevice(ctx, nodeID); err != nil {
errResp := &tailscale.ErrResponse{}
if errors.As(err, errResp) && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID)
@@ -425,7 +437,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
return true, nil
}
func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsr *tsapi.Recorder) error {
func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error {
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
@@ -453,7 +465,7 @@ func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsr *
return fmt.Errorf("failed to get Secret %q: %w", key.Name, err)
}
authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify())
authKey, err := newAuthKey(ctx, tailscaleClient, tags.Stringify())
if err != nil {
return err
}
@@ -555,7 +567,7 @@ func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) {
return prefs, ok, nil
}
func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tailscaleClient tsClient, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
secret, err := r.getStateSecret(ctx, tsrName, replica)
if err != nil || secret == nil {
return tsapi.RecorderTailnetDevice{}, false, err
@@ -569,7 +581,7 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string,
// TODO(tomhjp): The profile info doesn't include addresses, which is why we
// need the API. Should maybe update tsrecorder to write IPs to the state
// Secret like containerboot does.
device, err := r.tsClient.Device(ctx, string(prefs.Config.NodeID), nil)
device, err := tailscaleClient.Device(ctx, string(prefs.Config.NodeID), nil)
if err != nil {
return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
}

96
k8s-operator/api.md
View File

@@ -18,6 +18,8 @@
- [ProxyGroupList](#proxygrouplist)
- [Recorder](#recorder)
- [RecorderList](#recorderlist)
- [Tailnet](#tailnet)
- [TailnetList](#tailnetlist)
@@ -139,6 +141,7 @@ _Appears in:_
| `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is<br />configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the<br />Connector does not act as an app connector.<br />Note that you will need to manually configure the permissions and the domains for the app connector via the<br />Admin panel.<br />Note also that the main tested and supported use case of this config option is to deploy an app connector on<br />Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose<br />cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have<br />tested or optimised for.<br />If you are using the app connector to access SaaS applications because you need a predictable egress IP that<br />can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows<br />via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT<br />device with a static IP address.<br />https://tailscale.com/kb/1281/app-connectors | | |
| `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.<br />This field is mutually exclusive with the appConnector field.<br />https://tailscale.com/kb/1103/exit-nodes | | |
| `replicas` _integer_ | Replicas specifies how many devices to create. Set this to enable<br />high availability for app connectors, subnet routers, or exit nodes.<br />https://tailscale.com/kb/1115/high-availability. Defaults to 1. | | Minimum: 0 <br /> |
| `tailnet` _string_ | Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this<br />name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | |
#### ConnectorStatus
@@ -741,6 +744,7 @@ _Appears in:_
| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains<br />configuration options that should be applied to the resources created<br />for this ProxyGroup. If unset, and there is no default ProxyClass<br />configured, the operator will create resources with the default<br />configuration. | | |
| `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver<br />ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | |
| `tailnet` _string_ | Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this<br />name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | |
#### ProxyGroupStatus
@@ -901,6 +905,7 @@ _Appears in:_
| `enableUI` _boolean_ | Set to true to enable the Recorder UI. The UI lists and plays recorded sessions.<br />The UI will be served at <MagicDNS name of the recorder>:443. Defaults to false.<br />Corresponds to --ui tsrecorder flag https://tailscale.com/kb/1246/tailscale-ssh-session-recording#deploy-a-recorder-node.<br />Required if S3 storage is not set up, to ensure that recordings are accessible. | | |
| `storage` _[Storage](#storage)_ | Configure where to store session recordings. By default, recordings will<br />be stored in a local ephemeral volume, and will not be persisted past the<br />lifetime of a specific pod. | | |
| `replicas` _integer_ | Replicas specifies how many instances of tsrecorder to run. Defaults to 1. | | Minimum: 0 <br /> |
| `tailnet` _string_ | Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this<br />name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | |
#### RecorderStatefulSet
@@ -1154,6 +1159,44 @@ _Appears in:_
#### Tailnet
_Appears in:_
- [TailnetList](#tailnetlist)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | |
| `kind` _string_ | `Tailnet` | | |
| `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | |
| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | |
| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | |
| `spec` _[TailnetSpec](#tailnetspec)_ | Spec describes the desired state of the Tailnet.<br />More info:<br />https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status | | |
| `status` _[TailnetStatus](#tailnetstatus)_ | Status describes the status of the Tailnet. This is set<br />and managed by the Tailscale operator. | | |
#### TailnetCredentials
_Appears in:_
- [TailnetSpec](#tailnetspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `secretName` _string_ | The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and<br />"client_secret". | | |
#### TailnetDevice
@@ -1172,6 +1215,59 @@ _Appears in:_
| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. | | |
#### TailnetList
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | |
| `kind` _string_ | `TailnetList` | | |
| `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | |
| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | |
| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | |
| `items` _[Tailnet](#tailnet) array_ | | | |
#### TailnetSpec
_Appears in:_
- [Tailnet](#tailnet)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `loginUrl` _string_ | URL of the control plane to be used by all resources managed by the operator using this Tailnet. | | |
| `credentials` _[TailnetCredentials](#tailnetcredentials)_ | Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. | | |
#### TailnetStatus
_Appears in:_
- [Tailnet](#tailnet)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | | | |
#### TailscaleConfig

2
k8s-operator/apis/v1alpha1/register.go
View File

@@ -67,6 +67,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&RecorderList{},
&ProxyGroup{},
&ProxyGroupList{},
&Tailnet{},
&TailnetList{},
)
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

6
k8s-operator/apis/v1alpha1/types_connector.go
View File

@@ -133,6 +133,12 @@ type ConnectorSpec struct {
// +optional
// +kubebuilder:validation:Minimum=0
Replicas *int32 `json:"replicas,omitempty"`
// Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this
// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
// +optional
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Connector tailnet is immutable"
Tailnet string `json:"tailnet,omitempty"`
}
// SubnetRouter defines subnet routes that should be exposed to tailnet via a

6
k8s-operator/apis/v1alpha1/types_proxygroup.go
View File

@@ -97,6 +97,12 @@ type ProxyGroupSpec struct {
// ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
// +optional
KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"`
// Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this
// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
// +optional
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup tailnet is immutable"
Tailnet string `json:"tailnet,omitempty"`
}
type ProxyGroupStatus struct {

6
k8s-operator/apis/v1alpha1/types_recorder.go
View File

@@ -81,6 +81,12 @@ type RecorderSpec struct {
// +optional
// +kubebuilder:validation:Minimum=0
Replicas *int32 `json:"replicas,omitzero"`
// Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this
// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
// +optional
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Recorder tailnet is immutable"
Tailnet string `json:"tailnet,omitempty"`
}
type RecorderStatefulSet struct {

69
k8s-operator/apis/v1alpha1/types_tailnet.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Code comments on these types should be treated as user facing documentation-
// they will appear on the Tailnet CRD i.e. if someone runs kubectl explain tailnet.
var TailnetKind = "Tailnet"
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Cluster,shortName=tn
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "TailnetReady")].reason`,description="Status of the deployed Tailnet resources."
type Tailnet struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitzero"`
// Spec describes the desired state of the Tailnet.
// More info:
// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
Spec TailnetSpec `json:"spec"`
// Status describes the status of the Tailnet. This is set
// and managed by the Tailscale operator.
// +optional
Status TailnetStatus `json:"status"`
}
// +kubebuilder:object:root=true
type TailnetList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata"`
Items []Tailnet `json:"items"`
}
type TailnetSpec struct {
// URL of the control plane to be used by all resources managed by the operator using this Tailnet.
// +optional
LoginURL string `json:"loginUrl,omitempty"`
// Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
Credentials TailnetCredentials `json:"credentials"`
}
type TailnetCredentials struct {
// The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
// "client_secret".
SecretName string `json:"secretName"`
}
type TailnetStatus struct {
// +listType=map
// +listMapKey=type
// +optional
Conditions []metav1.Condition `json:"conditions"`
}
// TailnetReady is set to True if the Tailnet is available for use by operator workloads.
const TailnetReady ConditionType = `TailnetReady`

112
k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
View File

@@ -1365,6 +1365,48 @@ func (in Tags) DeepCopy() Tags {
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Tailnet) DeepCopyInto(out *Tailnet) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tailnet.
func (in *Tailnet) DeepCopy() *Tailnet {
if in == nil {
return nil
}
out := new(Tailnet)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *Tailnet) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailnetCredentials) DeepCopyInto(out *TailnetCredentials) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetCredentials.
func (in *TailnetCredentials) DeepCopy() *TailnetCredentials {
if in == nil {
return nil
}
out := new(TailnetCredentials)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) {
*out = *in
@@ -1390,6 +1432,76 @@ func (in *TailnetDevice) DeepCopy() *TailnetDevice {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailnetList) DeepCopyInto(out *TailnetList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]Tailnet, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetList.
func (in *TailnetList) DeepCopy() *TailnetList {
if in == nil {
return nil
}
out := new(TailnetList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *TailnetList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailnetSpec) DeepCopyInto(out *TailnetSpec) {
*out = *in
out.Credentials = in.Credentials
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetSpec.
func (in *TailnetSpec) DeepCopy() *TailnetSpec {
if in == nil {
return nil
}
out := new(TailnetSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailnetStatus) DeepCopyInto(out *TailnetStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetStatus.
func (in *TailnetStatus) DeepCopy() *TailnetStatus {
if in == nil {
return nil
}
out := new(TailnetStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) {
*out = *in

20
k8s-operator/conditions.go
View File

@@ -13,6 +13,7 @@
xslices "golang.org/x/exp/slices"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstime"
)
@@ -91,6 +92,14 @@ func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionT
pg.Status.Conditions = conds
}
// SetTailnetCondition ensures that Tailnet status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetTailnetCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger)
tn.Status.Conditions = conds
}
func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition {
newCondition := metav1.Condition{
Type: string(conditionType),
@@ -187,3 +196,14 @@ func SvcIsReady(svc *corev1.Service) bool {
cond := svc.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue
}
func TailnetIsReady(tn *tsapi.Tailnet) bool {
idx := xslices.IndexFunc(tn.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.TailnetReady)
})
if idx == -1 {
return false
}
cond := tn.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == tn.Generation
}

39
k8s-operator/reconciler/reconciler.go Normal file
View File

@@ -0,0 +1,39 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package reconciler provides utilities for working with Kubernetes resources within controller reconciliation
// loops.
package reconciler
import (
"slices"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
// FinalizerName is the common finalizer used across all Tailscale Kubernetes resources.
FinalizerName = "tailscale.com/finalizer"
)
// SetFinalizer adds the finalizer to the resource if not already present.
func SetFinalizer(obj client.Object) {
if idx := slices.Index(obj.GetFinalizers(), FinalizerName); idx >= 0 {
return
}
obj.SetFinalizers(append(obj.GetFinalizers(), FinalizerName))
}
// RemoveFinalizer removes the finalizer from the resource if present.
func RemoveFinalizer(obj client.Object) {
idx := slices.Index(obj.GetFinalizers(), FinalizerName)
if idx < 0 {
return
}
finalizers := obj.GetFinalizers()
obj.SetFinalizers(append(finalizers[:idx], finalizers[idx+1:]...))
}

42
k8s-operator/reconciler/reconciler_test.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package reconciler_test
import (
"slices"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"tailscale.com/k8s-operator/reconciler"
)
func TestFinalizers(t *testing.T) {
t.Parallel()
object := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
StringData: map[string]string{
"hello": "world",
},
}
reconciler.SetFinalizer(object)
if !slices.Contains(object.Finalizers, reconciler.FinalizerName) {
t.Fatalf("object does not have finalizer %q: %v", reconciler.FinalizerName, object.Finalizers)
}
reconciler.RemoveFinalizer(object)
if slices.Contains(object.Finalizers, reconciler.FinalizerName) {
t.Fatalf("object still has finalizer %q: %v", reconciler.FinalizerName, object.Finalizers)
}
}

45
k8s-operator/reconciler/tailnet/mocks_test.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package tailnet_test
import (
"context"
"io"
"tailscale.com/internal/client/tailscale"
)
type (
MockTailnetClient struct {
ErrorOnDevices bool
ErrorOnKeys bool
ErrorOnServices bool
}
)
func (m MockTailnetClient) Devices(_ context.Context, _ *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error) {
if m.ErrorOnDevices {
return nil, io.EOF
}
return nil, nil
}
func (m MockTailnetClient) Keys(_ context.Context) ([]string, error) {
if m.ErrorOnKeys {
return nil, io.EOF
}
return nil, nil
}
func (m MockTailnetClient) ListVIPServices(_ context.Context) (*tailscale.VIPServiceList, error) {
if m.ErrorOnServices {
return nil, io.EOF
}
return nil, nil
}

327
k8s-operator/reconciler/tailnet/tailnet.go Normal file
View File

@@ -0,0 +1,327 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package tailnet provides reconciliation logic for the Tailnet custom resource definition. It is responsible for
// ensuring the referenced OAuth credentials are valid and have the required scopes to be able to generate authentication
// keys, manage devices & manage VIP services.
package tailnet
import (
"context"
"errors"
"fmt"
"sync"
"time"
"go.uber.org/zap"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/internal/client/tailscale"
"tailscale.com/ipn"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/set"
)
type (
// The Reconciler type is a reconcile.TypedReconciler implementation used to manage the reconciliation of
// Tailnet custom resources.
Reconciler struct {
client.Client
tailscaleNamespace string
clock tstime.Clock
logger *zap.SugaredLogger
clientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
// Metrics related fields
mu sync.Mutex
tailnets set.Slice[types.UID]
}
// The ReconcilerOptions type contains configuration values for the Reconciler.
ReconcilerOptions struct {
// The client for interacting with the Kubernetes API.
Client client.Client
// The namespace the operator is installed in. This reconciler expects Tailnet OAuth credentials to be stored
// in Secret resources within this namespace.
TailscaleNamespace string
// Controls which clock to use for performing time-based functions. This is typically modified for use
// in tests.
Clock tstime.Clock
// The logger to use for this Reconciler.
Logger *zap.SugaredLogger
// ClientFunc is a function that takes tailscale credentials and returns an implementation for the Tailscale
// HTTP API. This should generally be nil unless needed for testing.
ClientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
}
// The TailscaleClient interface describes types that interact with the Tailscale HTTP API.
TailscaleClient interface {
Devices(context.Context, *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error)
Keys(ctx context.Context) ([]string, error)
ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error)
}
)
const reconcilerName = "tailnet-reconciler"
// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Tailnet custom
// resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler.
func NewReconciler(options ReconcilerOptions) *Reconciler {
return &Reconciler{
Client: options.Client,
tailscaleNamespace: options.TailscaleNamespace,
clock: options.Clock,
logger: options.Logger.Named(reconcilerName),
clientFunc: options.ClientFunc,
}
}
// Register the Reconciler onto the given manager.Manager implementation.
func (r *Reconciler) Register(mgr manager.Manager) error {
return builder.
ControllerManagedBy(mgr).
For(&tsapi.Tailnet{}).
Named(reconcilerName).
Complete(r)
}
var (
// gaugeTailnetResources tracks the overall number of Tailnet resources currently managed by this operator instance.
gaugeTailnetResources = clientmetric.NewGauge(kubetypes.MetricTailnetCount)
)
// Reconcile is invoked when a change occurs to Tailnet resources within the cluster. On create/update, the Tailnet
// resource is validated ensuring that the specified Secret exists and contains valid OAuth credentials that have
// required permissions to perform all necessary functions by the operator.
func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
var tailnet tsapi.Tailnet
err := r.Get(ctx, req.NamespacedName, &tailnet)
switch {
case apierrors.IsNotFound(err):
return reconcile.Result{}, nil
case err != nil:
return reconcile.Result{}, fmt.Errorf("failed to get Tailnet %q: %w", req.NamespacedName, err)
}
if !tailnet.DeletionTimestamp.IsZero() {
return r.delete(ctx, &tailnet)
}
return r.createOrUpdate(ctx, &tailnet)
}
func (r *Reconciler) delete(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) {
reconciler.RemoveFinalizer(tailnet)
if err := r.Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from Tailnet %q: %w", tailnet.Name, err)
}
r.mu.Lock()
r.tailnets.Remove(tailnet.UID)
r.mu.Unlock()
gaugeTailnetResources.Set(int64(r.tailnets.Len()))
return reconcile.Result{}, nil
}
// Constants for condition reasons.
const (
ReasonInvalidOAuth = "InvalidOAuth"
ReasonInvalidSecret = "InvalidSecret"
ReasonValid = "TailnetValid"
)
func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) {
r.mu.Lock()
r.tailnets.Add(tailnet.UID)
r.mu.Unlock()
gaugeTailnetResources.Set(int64(r.tailnets.Len()))
name := types.NamespacedName{Name: tailnet.Spec.Credentials.SecretName, Namespace: r.tailscaleNamespace}
var secret corev1.Secret
err := r.Get(ctx, name, &secret)
// The referenced Secret does not exist within the tailscale namespace, so we'll mark the Tailnet as not ready
// for use.
if apierrors.IsNotFound(err) {
operatorutils.SetTailnetCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
ReasonInvalidSecret,
fmt.Sprintf("referenced secret %q does not exist in namespace %q", name.Name, r.tailscaleNamespace),
r.clock,
r.logger,
)
if err = r.Status().Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
}
return reconcile.Result{}, nil
}
if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get secret %q: %w", name, err)
}
// We first ensure that the referenced secret contains the required fields. Otherwise, we set the Tailnet as
// invalid. The operator will not allow the use of this Tailnet while it is in an invalid state.
if ok := r.ensureSecret(tailnet, &secret); !ok {
if err = r.Status().Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
}
return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
}
tsClient := r.createClient(ctx, tailnet, &secret)
// Second, we ensure the OAuth credentials supplied in the secret are valid and have the required scopes to access
// the various API endpoints required by the operator.
if ok := r.ensurePermissions(ctx, tsClient, tailnet); !ok {
if err = r.Status().Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
}
// We provide a requeue duration here as a user will likely want to go and modify their scopes and come back.
// This should save them having to delete and recreate the resource.
return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
}
operatorutils.SetTailnetCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionTrue,
ReasonValid,
ReasonValid,
r.clock,
r.logger,
)
if err = r.Status().Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
}
reconciler.SetFinalizer(tailnet)
if err = r.Update(ctx, tailnet); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to add finalizer to Tailnet %q: %w", tailnet.Name, err)
}
return reconcile.Result{}, nil
}
// Constants for OAuth credential fields within the Secret referenced by the Tailnet.
const (
clientIDKey = "client_id"
clientSecretKey = "client_secret"
)
func (r *Reconciler) createClient(ctx context.Context, tailnet *tsapi.Tailnet, secret *corev1.Secret) TailscaleClient {
if r.clientFunc != nil {
return r.clientFunc(tailnet, secret)
}
baseURL := ipn.DefaultControlURL
if tailnet.Spec.LoginURL != "" {
baseURL = tailnet.Spec.LoginURL
}
credentials := clientcredentials.Config{
ClientID: string(secret.Data[clientIDKey]),
ClientSecret: string(secret.Data[clientSecretKey]),
TokenURL: baseURL + "/api/v2/oauth/token",
}
source := credentials.TokenSource(ctx)
httpClient := oauth2.NewClient(ctx, source)
tsClient := tailscale.NewClient("-", nil)
tsClient.UserAgent = "tailscale-k8s-operator"
tsClient.HTTPClient = httpClient
tsClient.BaseURL = baseURL
return tsClient
}
func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient TailscaleClient, tailnet *tsapi.Tailnet) bool {
// Perform basic list requests here to confirm that the OAuth credentials referenced on the Tailnet resource
// can perform the basic operations required for the operator to function. This has a caveat of only performing
// read actions, as we don't want to create arbitrary keys and VIP services. However, it will catch when a user
// has completely forgotten an entire scope that's required.
var errs error
if _, err := tsClient.Devices(ctx, nil); err != nil {
errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w", err))
}
if _, err := tsClient.Keys(ctx); err != nil {
errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w", err))
}
if _, err := tsClient.ListVIPServices(ctx); err != nil {
errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w", err))
}
if errs != nil {
operatorutils.SetTailnetCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
ReasonInvalidOAuth,
errs.Error(),
r.clock,
r.logger,
)
return false
}
return true
}
func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret) bool {
var message string
switch {
case len(secret.Data) == 0:
message = fmt.Sprintf("Secret %q is empty", secret.Name)
case len(secret.Data[clientIDKey]) == 0:
message = fmt.Sprintf("Secret %q is missing the client_id field", secret.Name)
case len(secret.Data[clientSecretKey]) == 0:
message = fmt.Sprintf("Secret %q is missing the client_secret field", secret.Name)
}
if message == "" {
return true
}
operatorutils.SetTailnetCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
ReasonInvalidSecret,
message,
r.clock,
r.logger,
)
return false
}

411
k8s-operator/reconciler/tailnet/tailnet_test.go Normal file
View File

@@ -0,0 +1,411 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package tailnet_test
import (
"testing"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailnet"
"tailscale.com/tstest"
)
func TestReconciler_Reconcile(t *testing.T) {
t.Parallel()
clock := tstest.NewClock(tstest.ClockOpts{})
logger, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
tt := []struct {
Name string
Request reconcile.Request
Tailnet *tsapi.Tailnet
Secret *corev1.Secret
ExpectsError bool
ExpectedConditions []metav1.Condition
ClientFunc func(*tsapi.Tailnet, *corev1.Secret) tailnet.TailscaleClient
}{
{
Name: "ignores unknown tailnet requests",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
},
{
Name: "invalid status for missing secret",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidSecret,
Message: `referenced secret "test" does not exist in namespace "tailscale"`,
},
},
},
{
Name: "invalid status for empty secret",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidSecret,
Message: `Secret "test" is empty`,
},
},
},
{
Name: "invalid status for missing client id",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_secret": []byte("test"),
},
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidSecret,
Message: `Secret "test" is missing the client_id field`,
},
},
},
{
Name: "invalid status for missing client secret",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_id": []byte("test"),
},
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidSecret,
Message: `Secret "test" is missing the client_secret field`,
},
},
},
{
Name: "invalid status for bad devices scope",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_id": []byte("test"),
"client_secret": []byte("test"),
},
},
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
return &MockTailnetClient{ErrorOnDevices: true}
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidOAuth,
Message: `failed to list devices: EOF`,
},
},
},
{
Name: "invalid status for bad services scope",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_id": []byte("test"),
"client_secret": []byte("test"),
},
},
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
return &MockTailnetClient{ErrorOnServices: true}
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidOAuth,
Message: `failed to list tailscale services: EOF`,
},
},
},
{
Name: "invalid status for bad keys scope",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "test",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_id": []byte("test"),
"client_secret": []byte("test"),
},
},
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
return &MockTailnetClient{ErrorOnKeys: true}
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionFalse,
Reason: tailnet.ReasonInvalidOAuth,
Message: `failed to list auth keys: EOF`,
},
},
},
{
Name: "ready when valid and scopes are correct",
Request: reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "default",
},
},
Tailnet: &tsapi.Tailnet{
ObjectMeta: metav1.ObjectMeta{
Name: "default",
},
Spec: tsapi.TailnetSpec{
Credentials: tsapi.TailnetCredentials{
SecretName: "test",
},
},
},
Secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "tailscale",
},
Data: map[string][]byte{
"client_id": []byte("test"),
"client_secret": []byte("test"),
},
},
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
return &MockTailnetClient{}
},
ExpectedConditions: []metav1.Condition{
{
Type: string(tsapi.TailnetReady),
Status: metav1.ConditionTrue,
Reason: tailnet.ReasonValid,
Message: tailnet.ReasonValid,
},
},
},
}
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme)
if tc.Tailnet != nil {
builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet)
}
if tc.Secret != nil {
builder = builder.WithObjects(tc.Secret)
}
fc := builder.Build()
opts := tailnet.ReconcilerOptions{
Client: fc,
Clock: clock,
Logger: logger.Sugar(),
ClientFunc: tc.ClientFunc,
TailscaleNamespace: "tailscale",
}
reconciler := tailnet.NewReconciler(opts)
_, err = reconciler.Reconcile(t.Context(), tc.Request)
if tc.ExpectsError && err == nil {
t.Fatalf("expected error, got none")
}
if !tc.ExpectsError && err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(tc.ExpectedConditions) == 0 {
return
}
var tn tsapi.Tailnet
if err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn); err != nil {
t.Fatal(err)
}
if len(tn.Status.Conditions) != len(tc.ExpectedConditions) {
t.Fatalf("expected %v condition(s), got %v", len(tc.ExpectedConditions), len(tn.Status.Conditions))
}
for i, expected := range tc.ExpectedConditions {
actual := tn.Status.Conditions[i]
if actual.Type != expected.Type {
t.Errorf("expected %v, got %v", expected.Type, actual.Type)
}
if actual.Status != expected.Status {
t.Errorf("expected %v, got %v", expected.Status, actual.Status)
}
if actual.Reason != expected.Reason {
t.Errorf("expected %v, got %v", expected.Reason, actual.Reason)
}
if actual.Message != expected.Message {
t.Errorf("expected %v, got %v", expected.Message, actual.Message)
}
}
if err = fc.Delete(t.Context(), &tn); err != nil {
t.Fatal(err)
}
if _, err = reconciler.Reconcile(t.Context(), tc.Request); err != nil {
t.Fatal(err)
}
err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn)
if !apierrors.IsNotFound(err) {
t.Fatalf("expected not found error, got %v", err)
}
})
}
}

1
kube/kubetypes/types.go
View File

@@ -33,6 +33,7 @@
MetricProxyGroupEgressCount = "k8s_proxygroup_egress_resources"
MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources"
MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources"
MetricTailnetCount = "k8s_tailnet_resources"
// Keys that containerboot writes to state file that can be used to determine its state.
// fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine