diff --git a/cmd/main.go b/cmd/main.go index fa1ba85..1f8d596 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -75,6 +75,7 @@ func main() { netbirdAPIKey string allowAutomaticPolicyCreation bool defaultLabels string + gatewayAPIEnabled bool ) flag.StringVar(&runtimeNamespace, "runtime-namespace", "", "Namespace the controller is running in") flag.StringVar(&managementURL, "netbird-management-url", "https://api.netbird.io", "Management service URL") @@ -105,6 +106,8 @@ func main() { "", "Default labels used for all resources, in format key=value,key=value", ) + flag.BoolVar(&gatewayAPIEnabled, "gateway-api-enabled", false, "When true Gateway API resources will be reconciled.") + // Controller generic flags var ( metricsAddr string @@ -272,25 +275,27 @@ func main() { } } - if err = (&controller.GatewayClassReconciler{ - Client: mgr.GetClient(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") - os.Exit(1) - } - if err = (&controller.GatewayReconciler{ - Client: mgr.GetClient(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Gateway") - os.Exit(1) - } - if err = (&controller.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Netbird: netbird, - ClusterDNS: clusterDNS, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") - os.Exit(1) + if gatewayAPIEnabled { + if err = (&controller.GatewayClassReconciler{ + Client: mgr.GetClient(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") + os.Exit(1) + } + if err = (&controller.GatewayReconciler{ + Client: mgr.GetClient(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Gateway") + os.Exit(1) + } + if err = (&controller.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Netbird: netbird, + ClusterDNS: clusterDNS, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") + os.Exit(1) + } } } else { setupLog.Info("netbird API key not provided, ingress capabilities disabled") diff --git a/examples/gateway-api-public/README.md b/examples/gateway-api-public/README.md new file mode 100644 index 0000000..277c772 --- /dev/null +++ b/examples/gateway-api-public/README.md @@ -0,0 +1,39 @@ +# Gateway API (Public) + +This example walks you through how to setup a Netbird Gateway API and expose Nginx through the Netbird proxy service. + +Build image locally and load it into Kind. +```shell +make docker-build IMG=docker.io/netbirdio/kubernetes-operator:dev +kind load docker-image docker.io/netbirdio/kubernetes-operator:dev +``` + +Install the Gateway API CRDs. + +```shell +kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.5.0/standard-install.yaml +``` + +Create Netbird namespace and API key secret. +```shell +kubectl create namespace netbird +kubectl -n netbird create secret generic netbird-mgmt-api-key --from-literal NB_API_KEY=${NETBIRD_API_KEY} +``` + +Install the Kubernetes Operator. Make sure to use the customized values to enable Gateway API support. This assumes you have already created a secret containing a Netbird API key. + +```shell +helm upgrade --install --create-namespace -f ./examples/gateway-api-public/values.yaml -n netbird netbird-operator ./helm/kubernetes-operator +``` + +Create the gateway along with the routing peer. This will deploy Netbird clients that route traffic into the cluster. + +```shell +kubectl apply -f ./examples/gateway-api-public/gateway.yaml +``` + +Deploy the test Nginx application along with a HTTPRoute. The HTTPRoute will expose the service through Netbirds public proxy. + +```shell +kubectl apply -f ./examples/gateway-api-public/nginx.yaml +``` diff --git a/examples/gateway-api/gateway.yaml b/examples/gateway-api-public/gateway.yaml similarity index 97% rename from examples/gateway-api/gateway.yaml rename to examples/gateway-api-public/gateway.yaml index 29714d3..b77d775 100644 --- a/examples/gateway-api/gateway.yaml +++ b/examples/gateway-api-public/gateway.yaml @@ -8,7 +8,7 @@ spec: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: - name: netbird + name: public namespace: netbird spec: gatewayClassName: public diff --git a/examples/gateway-api/nginx.yaml b/examples/gateway-api-public/nginx.yaml similarity index 98% rename from examples/gateway-api/nginx.yaml rename to examples/gateway-api-public/nginx.yaml index 71b6440..54912dd 100644 --- a/examples/gateway-api/nginx.yaml +++ b/examples/gateway-api-public/nginx.yaml @@ -9,7 +9,7 @@ spec: hostnames: - nginx-test-app.eu1.netbird.services parentRefs: - - name: netbird + - name: public namespace: netbird rules: - backendRefs: diff --git a/examples/gateway-api-public/values.yaml b/examples/gateway-api-public/values.yaml new file mode 100644 index 0000000..583a093 --- /dev/null +++ b/examples/gateway-api-public/values.yaml @@ -0,0 +1,14 @@ +gatewayAPI: + enabled: true + +webhook: + enableCertManager: false + +netbirdAPI: + keyFromSecret: + name: "netbird-mgmt-api-key" + key: "NB_API_KEY" + +operator: + image: + tag: dev diff --git a/helm/kubernetes-operator/templates/deployment.yaml b/helm/kubernetes-operator/templates/deployment.yaml index 5ce4605..27e03c0 100644 --- a/helm/kubernetes-operator/templates/deployment.yaml +++ b/helm/kubernetes-operator/templates/deployment.yaml @@ -46,6 +46,9 @@ spec: - --health-probe-bind-address=:{{ .Values.operator.livenessProbe.port }} - --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs - --runtime-namespace=$(POD_NAMESPACE) + {{- if .Values.gatewayAPI.enabled }} + - --gateway-api-enabled=true + {{- end }} {{- if .Values.managementURL }} - --netbird-management-url={{.Values.managementURL}} {{- end }} diff --git a/helm/kubernetes-operator/templates/rbac.yaml b/helm/kubernetes-operator/templates/rbac.yaml index dfeb05f..683a381 100644 --- a/helm/kubernetes-operator/templates/rbac.yaml +++ b/helm/kubernetes-operator/templates/rbac.yaml @@ -124,6 +124,27 @@ rules: - delete {{- end }} {{- end }} +{{- if .Values.gatewayAPI.enabled }} +- apiGroups: ["gateway.networking.k8s.io"] + resources: + - gatewayclasses + - gateways + - httproutes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: ["gateway.networking.k8s.io"] + resources: + - gatewayclasses/status + - gateways/status + - httproutes/status + verbs: + - update + - patch +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/helm/kubernetes-operator/values.yaml b/helm/kubernetes-operator/values.yaml index 9bf315c..f773c93 100644 --- a/helm/kubernetes-operator/values.yaml +++ b/helm/kubernetes-operator/values.yaml @@ -30,7 +30,7 @@ webhook: # operator: NotIn # values: # - foo - + # Failure Policy for webhook failurePolicy: Fail @@ -184,6 +184,9 @@ ingress: # sourceGroups: # - All +gatewayAPI: + enabled: false + cluster: # Cluster DNS name (used for webhooks certificates and for network resource DNS names) dns: svc.cluster.local @@ -196,8 +199,8 @@ netbirdAPI: {} #keyFromSecret: # name: "Secret name" # key: "NB_API_KEY" - -#routingClientImage: "netbirdio/netbird:latest" + +#routingClientImage: "netbirdio/netbird:latest" general: # General labels, applied to all created K8s resources diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index a8e6ec7..34f832e 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -2,13 +2,14 @@ package controller import ( "context" - "encoding/json" "fmt" - "strings" + "time" netbird "github.com/netbirdio/netbird/shared/management/client/rest" "github.com/netbirdio/netbird/shared/management/http/api" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,9 +21,7 @@ import ( ) const ( - HTTPRouteFinalizer = "gateway.netbird.io/httproute" - ResourceIDAnnotationKey = "gateway.netbird.io/resource-ids" - ProxyIDAnnotationKey = "gateway.netbird.io/proxy-ids" + HTTPRouteFinalizer = "gateway.netbird.io/httproute" ) type HTTPRouteReconciler struct { @@ -68,9 +67,8 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayv1.GatewayConditionProgrammed)) { logger.Info("gateway is not ready", "name", gw.ObjectMeta.Name) - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil } - nbrp := &netbirdiov1.NBRoutingPeer{} err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: gw.Spec.Infrastructure.ParametersRef.Name}, nbrp) if err != nil { @@ -85,75 +83,85 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Create network resources. - oldResourceIDs := map[string]string{} - if s, ok := hr.Annotations[ResourceIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &oldResourceIDs) - if err != nil { - return ctrl.Result{}, err - } - } - resourceIDs := map[string]string{} - targets := []api.ServiceTarget{} + svcIdx := map[string]corev1.Service{} for _, rule := range hr.Spec.Rules { for _, ref := range rule.BackendRefs { // TODO (phillebaba): Support reference grants. - refNamespace := hr.Namespace - - key := strings.Join([]string{string(ref.Name), refNamespace}, "/") - networkResourceReq := api.NetworkResourceRequest{ - Name: fmt.Sprintf("%s/%s/%s/%s", refNamespace, gw.Name, hr.Name, ref.Name), - Enabled: true, - Address: fmt.Sprintf("%s.%s.%s", ref.Name, refNamespace, r.ClusterDNS), - Groups: []string{}, - } - - id, err := func() (string, error) { - if id, ok := oldResourceIDs[key]; ok { - _, err := r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Get(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return "", err - } - if err == nil { - _, err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Update(ctx, id, networkResourceReq) - if err != nil { - return "", err - } - delete(oldResourceIDs, key) - return id, nil - } - } - resource, err := r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Create(ctx, networkResourceReq) - if err != nil { - return "", err - } - return resource.Id, nil - }() + key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)} + var svc corev1.Service + err := r.Client.Get(ctx, key, &svc) if err != nil { return ctrl.Result{}, err } - resourceIDs[key] = id - target := api.ServiceTarget{ - Enabled: true, - Path: nil, - TargetId: id, - Protocol: "http", - TargetType: "domain", - } - targets = append(targets, target) + svcIdx[svc.Name] = svc } } - // Create proxy service. - oldProxyIDs := map[string]string{} - if s, ok := hr.Annotations[ProxyIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &oldProxyIDs) + for _, svc := range svcIdx { + nbResource := netbirdiov1.NBResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, &nbResource, func() error { + err = controllerutil.SetControllerReference(&svc, &nbResource, r.Scheme(), controllerutil.WithBlockOwnerDeletion(false)) + if err != nil { + return err + } + err = controllerutil.SetOwnerReference(&hr, &nbResource, r.Scheme()) + if err != nil { + return err + } + nbResource.Spec = netbirdiov1.NBResourceSpec{ + Name: svc.Name, + NetworkID: *nbrp.Status.NetworkID, + Address: fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, r.ClusterDNS), + Groups: []string{}, + } + return nil + }) if err != nil { return ctrl.Result{}, err } } - proxyIDs := map[string]string{} + + targets := []api.ServiceTarget{} + for _, svc := range svcIdx { + var nbResource netbirdiov1.NBResource + err := r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource) + if err != nil { + return ctrl.Result{}, err + } + ready := func() bool { + for _, cond := range nbResource.Status.Conditions { + if cond.Type == netbirdiov1.NBSetupKeyReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false + }() + if !ready { + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + + target := api.ServiceTarget{ + Enabled: true, + Path: nil, + TargetId: *nbResource.Status.NetworkResourceID, + Protocol: "http", + TargetType: "domain", + } + targets = append(targets, target) + } + + // Create proxy service. + proxyServices, err := r.Netbird.ReverseProxyServices.List(ctx) + if err != nil { + return ctrl.Result{}, err + } for _, hostname := range hr.Spec.Hostnames { - proxyCreate := api.PostApiReverseProxiesServicesJSONRequestBody{ + proxyReq := api.PostApiReverseProxiesServicesJSONRequestBody{ Auth: api.ServiceAuthConfig{}, Domain: string(hostname), Enabled: true, @@ -163,59 +171,25 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( Targets: targets, } - id, err := func() (string, error) { - if id, ok := oldProxyIDs[string(hostname)]; ok { - _, err := r.Netbird.ReverseProxyServices.Get(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return "", err + err := func() error { + for _, proxyService := range proxyServices { + if proxyService.Domain != string(hostname) { + continue } - if err == nil { - _, err := r.Netbird.ReverseProxyServices.Update(ctx, id, proxyCreate) - if err != nil { - return "", nil - } - delete(oldProxyIDs, string(hostname)) - return id, nil + _, err := r.Netbird.ReverseProxyServices.Update(ctx, proxyService.Id, proxyReq) + if err != nil { + return err } } - proxy, err := r.Netbird.ReverseProxyServices.Create(ctx, proxyCreate) + _, err := r.Netbird.ReverseProxyServices.Create(ctx, proxyReq) if err != nil { - return "", err + return err } - return proxy.Id, nil + return nil }() if err != nil { return ctrl.Result{}, err } - proxyIDs[string(hostname)] = id - } - - for _, id := range oldResourceIDs { - err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - for _, id := range oldProxyIDs { - err = r.Netbird.ReverseProxyServices.Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - - b, err := json.Marshal(resourceIDs) - if err != nil { - return ctrl.Result{}, err - } - hr.Annotations[ResourceIDAnnotationKey] = string(b) - b, err = json.Marshal(proxyIDs) - if err != nil { - return ctrl.Result{}, err - } - hr.Annotations[ProxyIDAnnotationKey] = string(b) - err = r.Client.Update(ctx, &hr) - if err != nil { - return ctrl.Result{}, err } } @@ -223,6 +197,16 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1.HTTPRoute) (ctrl.Result, error) { + // Index all proxy services. + proxyServices, err := r.Netbird.ReverseProxyServices.List(ctx) + if err != nil { + return ctrl.Result{}, err + } + proxyIdx := map[string]string{} + for _, proxyService := range proxyServices { + proxyIdx[proxyService.Domain] = proxyService.Id + } + for _, parent := range hr.Spec.ParentRefs { parentNamespace := hr.Namespace if parent.Namespace != nil { @@ -233,46 +217,71 @@ func (r *HTTPRouteReconciler) reconcileDelete(ctx context.Context, hr gatewayv1. if err != nil { return ctrl.Result{}, err } - - nbrp := &netbirdiov1.NBRoutingPeer{} - err = r.Get(ctx, types.NamespacedName{Namespace: gw.Namespace, Name: gw.Spec.Infrastructure.ParametersRef.Name}, nbrp) + gwc := &gatewayv1.GatewayClass{} + err = r.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwc) if err != nil { return ctrl.Result{}, err } + if gwc.Spec.ControllerName != GatewayControllerName { + continue + } - proxyIDs := map[string]string{} - if s, ok := hr.Annotations[ProxyIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &proxyIDs) + // Remove the resource from the resource. + svcIdx := map[string]corev1.Service{} + for _, rule := range hr.Spec.Rules { + for _, ref := range rule.BackendRefs { + // TODO (phillebaba): Support reference grants. + key := client.ObjectKey{Namespace: hr.Namespace, Name: string(ref.Name)} + var svc corev1.Service + err := r.Client.Get(ctx, key, &svc) + if err != nil { + return ctrl.Result{}, err + } + svcIdx[svc.Name] = svc + } + } + for _, svc := range svcIdx { + var nbResource netbirdiov1.NBResource + err = r.Client.Get(ctx, client.ObjectKeyFromObject(&svc), &nbResource) if err != nil { return ctrl.Result{}, err } + err = controllerutil.RemoveOwnerReference(&hr, &nbResource, r.Scheme()) + if err != nil { + return ctrl.Result{}, err + } + + if len(nbResource.OwnerReferences) > 1 { + err = r.Client.Update(ctx, &nbResource) + if err != nil { + return ctrl.Result{}, err + } + } else { + // TODO: Precondition that nothing has changed. + err := r.Client.Delete(ctx, &nbResource) + if err != nil { + return ctrl.Result{}, err + } + } } - for _, id := range proxyIDs { + + // Remove the target from the proxy service. + for _, hostname := range hr.Spec.Hostnames { + id, ok := proxyIdx[string(hostname)] + if !ok { + continue + } err = r.Netbird.ReverseProxyServices.Delete(ctx, id) if err != nil && !netbird.IsNotFound(err) { return ctrl.Result{}, err } } + } - resourceIDs := map[string]string{} - if s, ok := hr.Annotations[ResourceIDAnnotationKey]; ok { - err := json.Unmarshal([]byte(s), &resourceIDs) - if err != nil { - return ctrl.Result{}, err - } - } - for _, id := range resourceIDs { - err = r.Netbird.Networks.Resources(*nbrp.Status.NetworkID).Delete(ctx, id) - if err != nil && !netbird.IsNotFound(err) { - return ctrl.Result{}, err - } - } - - if controllerutil.RemoveFinalizer(&hr, HTTPRouteFinalizer) { - err := r.Client.Update(ctx, &hr) - if err != nil { - return ctrl.Result{}, err - } + if controllerutil.RemoveFinalizer(&hr, HTTPRouteFinalizer) { + err := r.Client.Update(ctx, &hr) + if err != nil { + return ctrl.Result{}, err } } return ctrl.Result{}, nil