From 39ab189fae4be19d05f417c747484d2b9d813c6c Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Thu, 19 Mar 2026 18:28:35 +0100 Subject: [PATCH] Share NBResource between different routes (#149) This changes the behavior of the Netbird resource so that they can be shared between routes rather than having one unique created per route. This is important as we may want to have different route types pointing to the same service. Signed-off-by: Philip Laine --- cmd/main.go | 43 +-- examples/gateway-api-public/README.md | 39 +++ .../gateway.yaml | 2 +- .../nginx.yaml | 2 +- examples/gateway-api-public/values.yaml | 14 + .../templates/deployment.yaml | 3 + helm/kubernetes-operator/templates/rbac.yaml | 21 ++ helm/kubernetes-operator/values.yaml | 9 +- internal/controller/httproute_controller.go | 273 +++++++++--------- 9 files changed, 250 insertions(+), 156 deletions(-) create mode 100644 examples/gateway-api-public/README.md rename examples/{gateway-api => gateway-api-public}/gateway.yaml (97%) rename examples/{gateway-api => gateway-api-public}/nginx.yaml (98%) create mode 100644 examples/gateway-api-public/values.yaml diff --git a/cmd/main.go b/cmd/main.go index fa1ba85..1f8d596 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -75,6 +75,7 @@ func main() { netbirdAPIKey string allowAutomaticPolicyCreation bool defaultLabels string + gatewayAPIEnabled bool ) flag.StringVar(&runtimeNamespace, "runtime-namespace", "", "Namespace the controller is running in") flag.StringVar(&managementURL, "netbird-management-url", "https://api.netbird.io", "Management service URL") @@ -105,6 +106,8 @@ func main() { "", "Default labels used for all resources, in format key=value,key=value", ) + flag.BoolVar(&gatewayAPIEnabled, "gateway-api-enabled", false, "When true Gateway API resources will be reconciled.") + // Controller generic flags var ( metricsAddr string @@ -272,25 +275,27 @@ func main() { } } - if err = (&controller.GatewayClassReconciler{ - Client: mgr.GetClient(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") - os.Exit(1) - } - if err = (&controller.GatewayReconciler{ - Client: mgr.GetClient(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Gateway") - os.Exit(1) - } - if err = (&controller.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Netbird: netbird, - ClusterDNS: clusterDNS, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") - os.Exit(1) + if gatewayAPIEnabled { + if err = (&controller.GatewayClassReconciler{ + Client: mgr.GetClient(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") + os.Exit(1) + } + if err = (&controller.GatewayReconciler{ + Client: mgr.GetClient(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Gateway") + os.Exit(1) + } + if err = (&controller.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Netbird: netbird, + ClusterDNS: clusterDNS, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") + os.Exit(1) + } } } else { setupLog.Info("netbird API key not provided, ingress capabilities disabled") diff --git a/examples/gateway-api-public/README.md b/examples/gateway-api-public/README.md new file mode 100644 index 0000000..277c772 --- /dev/null +++ b/examples/gateway-api-public/README.md @@ -0,0 +1,39 @@ +# Gateway API (Public) + +This example walks you through how to setup a Netbird Gateway API and expose Nginx through the Netbird proxy service. + +Build image locally and load it into Kind. +```shell +make docker-build IMG=docker.io/netbirdio/kubernetes-operator:dev +kind load docker-image docker.io/netbirdio/kubernetes-operator:dev +``` + +Install the Gateway API CRDs. + +```shell +kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.5.0/standard-install.yaml +``` + +Create Netbird namespace and API key secret. +```shell +kubectl create namespace netbird +kubectl -n netbird create secret generic netbird-mgmt-api-key --from-literal NB_API_KEY=${NETBIRD_API_KEY} +``` + +Install the Kubernetes Operator. Make sure to use the customized values to enable Gateway API support. This assumes you have already created a secret containing a Netbird API key. + +```shell +helm upgrade --install --create-namespace -f ./examples/gateway-api-public/values.yaml -n netbird netbird-operator ./helm/kubernetes-operator +``` + +Create the gateway along with the routing peer. This will deploy Netbird clients that route traffic into the cluster. + +```shell +kubectl apply -f ./examples/gateway-api-public/gateway.yaml +``` + +Deploy the test Nginx application along with a HTTPRoute. The HTTPRoute will expose the service through Netbirds public proxy. + +```shell +kubectl apply -f ./examples/gateway-api-public/nginx.yaml +``` diff --git a/examples/gateway-api/gateway.yaml b/examples/gateway-api-public/gateway.yaml similarity index 97% rename from examples/gateway-api/gateway.yaml rename to examples/gateway-api-public/gateway.yaml index 29714d3..b77d775 100644 --- a/examples/gateway-api/gateway.yaml +++ b/examples/gateway-api-public/gateway.yaml @@ -8,7 +8,7 @@ spec: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: - name: netbird + name: public namespace: netbird spec: gatewayClassName: public diff --git a/examples/gateway-api/nginx.yaml b/examples/gateway-api-public/nginx.yaml similarity index 98% rename from examples/gateway-api/nginx.yaml rename to examples/gateway-api-public/nginx.yaml index 71b6440..54912dd 100644 --- a/examples/gateway-api/nginx.yaml +++ b/examples/gateway-api-public/nginx.yaml @@ -9,7 +9,7 @@ spec: hostnames: - nginx-test-app.eu1.netbird.services parentRefs: - - name: netbird + - name: public namespace: netbird rules: - backendRefs: diff --git a/examples/gateway-api-public/values.yaml b/examples/gateway-api-public/values.yaml new file mode 100644 index 0000000..583a093 --- /dev/null +++ b/examples/gateway-api-public/values.yaml @@ -0,0 +1,14 @@ +gatewayAPI: + enabled: true + +webhook: + enableCertManager: false + +netbirdAPI: + keyFromSecret: + name: "netbird-mgmt-api-key" + key: "NB_API_KEY" + +operator: + image: + tag: dev diff --git a/helm/kubernetes-operator/templates/deployment.yaml b/helm/kubernetes-operator/templates/deployment.yaml index 5ce4605..27e03c0 100644 --- a/helm/kubernetes-operator/templates/deployment.yaml +++ b/helm/kubernetes-operator/templates/deployment.yaml @@ -46,6 +46,9 @@ spec: - --health-probe-bind-address=:{{ .Values.operator.livenessProbe.port }} - --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs - --runtime-namespace=$(POD_NAMESPACE) + {{- if .Values.gatewayAPI.enabled }} + - --gateway-api-enabled=true + {{- end }} {{- if .Values.managementURL }} - --netbird-management-url={{.Values.managementURL}} {{- end }} diff --git a/helm/kubernetes-operator/templates/rbac.yaml b/helm/kubernetes-operator/templates/rbac.yaml index dfeb05f..683a381 100644 --- a/helm/kubernetes-operator/templates/rbac.yaml +++ b/helm/kubernetes-operator/templates/rbac.yaml @@ -124,6 +124,27 @@ rules: - delete {{- end }} {{- end }} +{{- if .Values.gatewayAPI.enabled }} +- apiGroups: ["gateway.networking.k8s.io"] + resources: + - gatewayclasses + - gateways + - httproutes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: ["gateway.networking.k8s.io"] + resources: + - gatewayclasses/status + - gateways/status + - httproutes/status + verbs: + - update + - patch +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/helm/kubernetes-operator/values.yaml b/helm/kubernetes-operator/values.yaml index 9bf315c..f773c93 100644 --- a/helm/kubernetes-operator/values.yaml +++ b/helm/kubernetes-operator/values.yaml @@ -30,7 +30,7 @@ webhook: # operator: NotIn # values: # - foo - + # Failure Policy for webhook failurePolicy: Fail @@ -184,6 +184,9 @@ ingress: # sourceGroups: # - All +gatewayAPI: + enabled: false + cluster: # Cluster DNS name (used for webhooks certificates and for network resource DNS names) dns: svc.cluster.local @@ -196,8 +199,8 @@ netbirdAPI: {} #keyFromSecret: # name: "Secret name" # key: "NB_API_KEY" - -#routingClientImage: "netbirdio/netbird:latest" + +#routingClientImage: "netbirdio/netbird:latest" general: # General labels, applied to all created K8s resources diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index a8e6ec7..34f832e 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -2,13 +2,14 @@ package controller import ( "context" - "encoding/json" "fmt" - "strings" + "time" netbird "github.com/netbirdio/netbird/shared/management/client/rest" "github.com/netbirdio/netbird/shared/management/http/api" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,9 +21,7 @@ import ( ) const ( - HTTPRouteFinalizer = "gateway.netbird.io/httproute" - ResourceIDAnnotationKey = "gateway.netbird.io/resource-ids" - ProxyIDAnnotationKey = "gateway.netbird.io/proxy-ids" + HTTPRouteFinalizer = "gateway.netbird.io/httproute" ) type HTTPRouteReconciler struct { @@ -68,9 +67,8 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) { logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name) - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil } - nbrp := &netbirdiov1.NBRoutingPeer{} err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: gw.Spec.Infrastructure.ParametersRef.Name}, nbrp) if err != nil { @@ -85,75 +83,85 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Create network resources. - oldResourceIDs := map[string]string{} - if s, ok := hr.Annotations[ResourceIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &oldResourceIDs) - if err != nil { - return ctrl.Result{}, err - } - } - resourceIDs := map[string]string{} - targets := []api.ServiceTarget{} + svcIdx := map[string]corev1.Service{} for _, rule := range hr.Spec.Rules { for _, ref := range rule.BackendRefs { // TODO (phillebaba): Support reference grants. - refNamespace := hr.Namespace - - key := strings.Join([]string{string(ref.Name), refNamespace}, "/") - networkResourceReq := api.NetworkResourceRequest{ - Name: fmt.Sprintf("%s/%s/%s/%s", refNamespace, gw.Name, hr.Name, ref.Name), - Enabled: true, - Address: fmt.Sprintf("%s.%s.%s", ref.Name, refNamespace, r.ClusterDNS), - Groups: []string{}, - } - - id, err := func() (string, error) { - if id, ok := oldResourceIDs[key]; ok { - _, err := r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Get(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return "", err - } - if err == nil { - _, err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Update(ctx, id, networkResourceReq) - if err != nil { - return "", err - } - delete(oldResourceIDs, key) - return id, nil - } - } - resource, err := r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Create(ctx, networkResourceReq) - if err != nil { - return "", err - } - return resource.Id, nil - }() + key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)} + var svc corev1.Service + err := r.Client.Get(ctx, key, &svc) if err != nil { return ctrl.Result{}, err } - resourceIDs[key] = id - target := api.ServiceTarget{ - Enabled: true, - Path: nil, - TargetId: id, - Protocol: "http", - TargetType: "domain", - } - targets = append(targets, target) + svcIdx[svc.Name] = svc } } - // Create proxy service. - oldProxyIDs := map[string]string{} - if s, ok := hr.Annotations[ProxyIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &oldProxyIDs) + for _, svc := range svcIdx { + nbResource := netbirdiov1.NBResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &nbResource, func() error { + err = controllerutil.SetControllerReference(&svc, &nbResource, r.Scheme(), controllerutil.WithBlockOwnerDeletion(false)) + if err != nil { + return err + } + err = controllerutil.SetOwnerReference(&hr, &nbResource, r.Scheme()) + if err != nil { + return err + } + nbResource.Spec = netbirdiov1.NBResourceSpec{ + Name: svc.Name, + NetworkID: *nbrp.Status.NetworkID, + Address: fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, r.ClusterDNS), + Groups: []string{}, + } + return nil + }) if err != nil { return ctrl.Result{}, err } } - proxyIDs := map[string]string{} + + targets := []api.ServiceTarget{} + for _, svc := range svcIdx { + var nbResource netbirdiov1.NBResource + err := r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource) + if err != nil { + return ctrl.Result{}, err + } + ready := func() bool { + for _, cond := range nbResource.Status.Conditions { + if cond.Type == netbirdiov1.NBSetupKeyReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false + }() + if !ready { + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + + target := api.ServiceTarget{ + Enabled: true, + Path: nil, + TargetId: *nbResource.Status.NetworkResourceID, + Protocol: "http", + TargetType: "domain", + } + targets = append(targets, target) + } + + // Create proxy service. + proxyServices, err := r.Netbird.ReverseProxyServices.List(ctx) + if err != nil { + return ctrl.Result{}, err + } for _, hostname := range hr.Spec.Hostnames { - proxyCreate := api.PostApiReverseProxiesServicesJSONRequestBody{ + proxyReq := api.PostApiReverseProxiesServicesJSONRequestBody{ Auth: api.ServiceAuthConfig{}, Domain: string(hostname), Enabled: true, @@ -163,59 +171,25 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Targets: targets, } - id, err := func() (string, error) { - if id, ok := oldProxyIDs[string(hostname)]; ok { - _, err := r.Netbird.ReverseProxyServices.Get(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return "", err + err := func() error { + for _, proxyService := range proxyServices { + if proxyService.Domain != string(hostname) { + continue } - if err == nil { - _, err := r.Netbird.ReverseProxyServices.Update(ctx, id, proxyCreate) - if err != nil { - return "", nil - } - delete(oldProxyIDs, string(hostname)) - return id, nil + _, err := r.Netbird.ReverseProxyServices.Update(ctx, proxyService.Id, proxyReq) + if err != nil { + return err } } - proxy, err := r.Netbird.ReverseProxyServices.Create(ctx, proxyCreate) + _, err := r.Netbird.ReverseProxyServices.Create(ctx, proxyReq) if err != nil { - return "", err + return err } - return proxy.Id, nil + return nil }() if err != nil { return ctrl.Result{}, err } - proxyIDs[string(hostname)] = id - } - - for _, id := range oldResourceIDs { - err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - for _, id := range oldProxyIDs { - err = r.Netbird.ReverseProxyServices.Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - - b, err := json.Marshal(resourceIDs) - if err != nil { - return ctrl.Result{}, err - } - hr.Annotations[ResourceIDAnnotationKey] = string(b) - b, err = json.Marshal(proxyIDs) - if err != nil { - return ctrl.Result{}, err - } - hr.Annotations[ProxyIDAnnotationKey] = string(b) - err = r.Client.Update(ctx, &hr) - if err != nil { - return ctrl.Result{}, err } } @@ -223,6 +197,16 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.HTTPRoute) (ctrl.Result, error) { + // Index all proxy services. + proxyServices, err := r.Netbird.ReverseProxyServices.List(ctx) + if err != nil { + return ctrl.Result{}, err + } + proxyIdx := map[string]string{} + for _, proxyService := range proxyServices { + proxyIdx[proxyService.Domain] = proxyService.Id + } + for _, parent := range hr.Spec.ParentRefs { parentNamespace := hr.Namespace if parent.Namespace != nil { @@ -233,46 +217,71 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1. if err != nil { return ctrl.Result{}, err } - - nbrp := &netbirdiov1.NBRoutingPeer{} - err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: gw.Spec.Infrastructure.ParametersRef.Name}, nbrp) + gwc := &gatewayv1.GatewayClass{} + err = r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc) if err != nil { return ctrl.Result{}, err } + if gwc.Spec.ControllerName != GatewayControllerName { + continue + } - proxyIDs := map[string]string{} - if s, ok := hr.Annotations[ProxyIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &proxyIDs) + // Remove the resource from the resource. + svcIdx := map[string]corev1.Service{} + for _, rule := range hr.Spec.Rules { + for _, ref := range rule.BackendRefs { + // TODO (phillebaba): Support reference grants. + key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)} + var svc corev1.Service + err := r.Client.Get(ctx, key, &svc) + if err != nil { + return ctrl.Result{}, err + } + svcIdx[svc.Name] = svc + } + } + for _, svc := range svcIdx { + var nbResource netbirdiov1.NBResource + err = r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource) if err != nil { return ctrl.Result{}, err } + err = controllerutil.RemoveOwnerReference(&hr, &nbResource, r.Scheme()) + if err != nil { + return ctrl.Result{}, err + } + + if len(nbResource.OwnerReferences) > 1 { + err = r.Client.Update(ctx, &nbResource) + if err != nil { + return ctrl.Result{}, err + } + } else { + // TODO: Precondition that nothing has changed. + err := r.Client.Delete(ctx, &nbResource) + if err != nil { + return ctrl.Result{}, err + } + } } - for _, id := range proxyIDs { + + // Remove the target from the proxy service. + for _, hostname := range hr.Spec.Hostnames { + id, ok := proxyIdx[string(hostname)] + if !ok { + continue + } err = r.Netbird.ReverseProxyServices.Delete(ctx, id) if err != nil && !netbird.IsNotFound(err) { return ctrl.Result{}, err } } + } - resourceIDs := map[string]string{} - if s, ok := hr.Annotations[ResourceIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &resourceIDs) - if err != nil { - return ctrl.Result{}, err - } - } - for _, id := range resourceIDs { - err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - - if controllerutil.RemoveFinalizer(&hr, HTTPRouteFinalizer) { - err := r.Client.Update(ctx, &hr) - if err != nil { - return ctrl.Result{}, err - } + if controllerutil.RemoveFinalizer(&hr, HTTPRouteFinalizer) { + err := r.Client.Update(ctx, &hr) + if err != nil { + return ctrl.Result{}, err } } return ctrl.Result{}, nil