From 03830ae2b76621597e295a9ba2fbfae406d200ad Mon Sep 17 00:00:00 2001 From: "M. Essam" Date: Wed, 23 Apr 2025 11:05:17 +0200 Subject: [PATCH] Add E2E tests for ingress feature (#35) --- Makefile | 17 ++ test/e2e/e2e_suite_test.go | 26 +++ test/e2e/e2e_test.go | 211 +++++++++++++++++++++++- test/utils/management/compose.yml | 85 ++++++++++ test/utils/management/management.json | 67 ++++++++ test/utils/management/seed_database.sql | 15 ++ test/utils/management/setup_mgmt.sh | 27 +++ test/utils/values.yaml | 15 ++ 8 files changed, 460 insertions(+), 3 deletions(-) create mode 100644 test/utils/management/compose.yml create mode 100644 test/utils/management/management.json create mode 100644 test/utils/management/seed_database.sql create mode 100755 test/utils/management/setup_mgmt.sh create mode 100644 test/utils/values.yaml diff --git a/Makefile b/Makefile index 07d4114..10b8f2f 100644 --- a/Makefile +++ b/Makefile @@ -133,10 +133,27 @@ endif deploy: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config. $(HELM) install -n netbird --create-namespace kubernetes-operator --set operator.image.tag=$(word 2,$(subst :, ,${IMG})) --repo https://netbirdio.github.io/helms kubernetes-operator +.PHONY: deploy-e2e +deploy-e2e: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config. + $(HELM) install -n netbird --create-namespace kubernetes-operator -f ./test/utils/values.yaml --set operator.image.tag=$(word 2,$(subst :, ,${IMG})) --set managementURL=${MGMT_HOST} --repo https://netbirdio.github.io/helms kubernetes-operator + .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(HELM) uninstall -n netbird kubernetes-operator --no-hooks +.PHONY: undeploy-e2e +undeploy-e2e: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(HELM) uninstall -n netbird kubernetes-operator --no-hooks || true + kubectl get NBResource -n default -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBResource -n default -p '{"metadata":{"finalizers":null}}' --type=merge + kubectl get NBGroup -n default -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBGroup -n default -p '{"metadata":{"finalizers":null}}' --type=merge + kubectl get NBGroup -n netbird -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBGroup -n netbird -p '{"metadata":{"finalizers":null}}' --type=merge + kubectl get NBRoutingPeer -n netbird -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBRoutingPeer -n netbird -p '{"metadata":{"finalizers":null}}' --type=merge + kubectl get NBPolicy -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBPolicy -p '{"metadata":{"finalizers":null}}' --type=merge + kubectl delete NBGroup -A --all + kubectl delete NBResource -A --all + kubectl delete NBRoutingPeer -A --all + kubectl delete NBPolicy --all + ##@ Dependencies ## Location to install dependencies to diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 512f52f..cdd4a7c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -17,17 +17,22 @@ limitations under the License. package e2e import ( + "context" "fmt" "os" "os/exec" + "path" "testing" + netbird "github.com/netbirdio/netbird/management/client/rest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/netbirdio/kubernetes-operator/test/utils" ) +const apiToken = "nbp_apTmlmUXHSC4PKmHwtIZNaGr8eqcVI2gMURp" + var ( // Optional Environment Variables: // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. @@ -72,6 +77,20 @@ var _ = BeforeSuite(func() { _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") } } + + By("starting up the test management") + cmd = exec.Command("docker", "compose", "up", "-d") + curDir, err := utils.GetProjectDir() + Expect(err).NotTo(HaveOccurred()) + cmd.Dir = path.Join(curDir, "test", "utils", "management") + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(out)) + + Eventually(func() error { + client := netbird.New("http://127.0.0.1:8080", apiToken) + _, err = client.Accounts.List(context.Background()) + return err + }).Should(Succeed()) }) var _ = AfterSuite(func() { @@ -80,4 +99,11 @@ var _ = AfterSuite(func() { _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") utils.UninstallCertManager() } + + cmd := exec.Command("docker", "compose", "down") + curDir, err := utils.GetProjectDir() + Expect(err).NotTo(HaveOccurred()) + cmd.Dir = path.Join(curDir, "test", "utils", "management") + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(out)) }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 374e2a3..649876f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,11 +17,15 @@ limitations under the License. package e2e import ( + "context" + "encoding/json" "fmt" "os/exec" "strings" "time" + netbird "github.com/netbirdio/netbird/management/client/rest" + "github.com/netbirdio/netbird/management/server/http/api" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,6 +40,7 @@ const metricsServiceName = "kubernetes-operator-metrics" var _ = Describe("Manager", Ordered, func() { var controllerPodName string + var netbirdClient *netbird.Client // Before running the tests, set up the environment by creating the namespace, // enforce the restricted security policy to the namespace, installing CRDs, @@ -53,12 +58,37 @@ var _ = Describe("Manager", Ordered, func() { Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") By("deploying the kubernetes-operator") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - out, err := utils.Run(cmd) + out, err := utils.Run(exec.Command("kubectl", "get", "node", "-o", "json")) + Expect(err).NotTo(HaveOccurred(), "Failed to get nodes") + nodesGetOutput := make(map[string]any) + err = json.Unmarshal([]byte(out), &nodesGetOutput) + Expect(err).NotTo(HaveOccurred(), "Failed to get nodes") + nodeIPs := (nodesGetOutput["items"].([]any))[0].(map[string]any)["status"].(map[string]any)["addresses"].([]any) + managementIP := "" + for _, v := range nodeIPs { + addrType := v.(map[string]any)["type"] + if addrType == "InternalIP" { + managementIP = v.(map[string]any)["address"].(string) + break + } + } + + Expect(managementIP).NotTo(BeEmpty()) + managementIPParts := strings.Split(managementIP, ".") + managementIP = fmt.Sprintf("http://%s.%s.0.1:8080", managementIPParts[0], managementIPParts[1]) + cmd = exec.Command( + "make", + "deploy-e2e", + fmt.Sprintf("IMG=%s", projectImage), + fmt.Sprintf("MGMT_HOST=%s", managementIP), + ) + out, err = utils.Run(cmd) if err != nil { fmt.Println(out) } Expect(err).NotTo(HaveOccurred(), "Failed to deploy the kubernetes-operator") + + netbirdClient = netbird.New("http://127.0.0.1:8080", apiToken) }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, @@ -69,7 +99,7 @@ var _ = Describe("Manager", Ordered, func() { _, _ = utils.Run(cmd) By("undeploying the kubernetes-operator") - cmd = exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy-e2e") _, _ = utils.Run(cmd) By("removing manager namespace") @@ -449,6 +479,181 @@ var _ = Describe("Manager", Ordered, func() { }) }) + Context("Ingress", Ordered, func() { + BeforeAll(func() { + _, err := utils.Run(exec.Command("kubectl", "create", "deployment", "--image=nginx", "exposable")) + Expect(err).NotTo(HaveOccurred()) + _, err = utils.Run(exec.Command("kubectl", "expose", "deployment", "--port=80", "exposable")) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + _, err := utils.Run(exec.Command("kubectl", "delete", "deployment", "--ignore-not-found", "exposable")) + Expect(err).NotTo(HaveOccurred()) + _, err = utils.Run(exec.Command("kubectl", "delete", "service", "--ignore-not-found", "exposable")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create Network", func() { + Eventually(func(g Gomega) { + networks, err := netbirdClient.Networks.List(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(networks).To(HaveLen(1)) + }).Should(Succeed()) + }) + + networkID := func() string { + defer GinkgoHelper() + networks, err := netbirdClient.Networks.List(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(networks).To(HaveLen(1)) + return networks[0].Id + } + + It("should create NetworkRouter", func() { + Eventually(func(g Gomega) { + routers, err := netbirdClient.Networks.Routers(networkID()).List(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(routers).To(HaveLen(1)) + }).Should(Succeed()) + }) + + It("should create router deployment", func() { + Eventually(func(g Gomega) { + _, err := utils.Run(exec.Command("kubectl", "get", "deployment", "router", "-n", "netbird")) + Expect(err).NotTo(HaveOccurred()) + }).Should(Succeed()) + }) + + groupsToNames := func(i []api.Group) []string { + ret := make([]string, len(i)) + for j, k := range i { + ret[j] = k.Name + } + return ret + } + + It("should create router group", func() { + Eventually(func(g Gomega) { + groups, err := netbirdClient.Groups.List(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(groups).To(WithTransform(groupsToNames, ContainElement("kubernetes"))) + }).Should(Succeed()) + }) + + It("should not expose service by default", func() { + _, err := utils.Run(exec.Command("kubectl", "get", "NBResource", "exposable")) + Expect(err).To(HaveOccurred()) + }) + + When("Service is annotated with expose", Ordered, func() { + BeforeAll(func() { + _, err := utils.Run(exec.Command("kubectl", "annotate", "service", "exposable", "netbird.io/expose=true")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create NBResource", func() { + Eventually(func(g Gomega) { + _, err := utils.Run(exec.Command("kubectl", "get", "NBResource", "exposable")) + g.Expect(err).NotTo(HaveOccurred()) + }).Should(Succeed()) + }) + + It("should create Network Resource", func() { + Eventually(func(g Gomega) { + resources, err := netbirdClient.Networks.Resources(networkID()).List(context.Background()) + Expect(err).NotTo(HaveOccurred()) + resourcesToNames := func(r []api.NetworkResource) []string { + ret := make([]string, len(r)) + for i, j := range r { + ret[i] = j.Name + } + return ret + } + Expect(resources).To(WithTransform(resourcesToNames, ContainElement("default-exposable"))) + }).Should(Succeed()) + }) + + It("should create Group", func() { + Eventually(func(g Gomega) { + groups, err := netbirdClient.Groups.List(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(groups).To(WithTransform(groupsToNames, ContainElement("kubernetes-default-exposable"))) + }).Should(Succeed()) + }) + }) + + When("Service is annotated with policy", Ordered, func() { + BeforeAll(func() { + _, err := utils.Run(exec.Command("kubectl", "annotate", "service", "exposable", "netbird.io/policy=default")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should add service to policy status", func() { + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command( + "kubectl", "get", "NBPolicy", "default", "-o", "jsonpath={.status.managedServiceList}", + )) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring("default/exposable")) + }).Should(Succeed()) + }) + + It("should create policy on NetBird", func() { + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command( + "kubectl", "get", "NBPolicy", "default", "-o", "jsonpath={.status.tcpPolicyID}", + )) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).NotTo(BeEmpty()) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + policies, err := netbirdClient.Policies.List(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(policies).To(HaveLen(1)) + g.Expect(policies[0].Name).To(ContainSubstring("Kubernetes Default Group")) + }).Should(Succeed()) + }) + }) + + When("Service is annotated with a non-existent policy", Ordered, func() { + BeforeAll(func() { + _, err := utils.Run(exec.Command( + "kubectl", "annotate", "service", "exposable", "--overwrite", + "netbird.io/policy=custom", "netbird.io/policy-source-groups=All", "netbird.io/policy-name=custom:E2E", + )) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create NBPolicy", func() { + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command( + "kubectl", "get", "NBPolicy", "custom-default-exposable", "-o", "jsonpath={.status.managedServiceList}", + )) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(ContainSubstring("default/exposable")) + }).Should(Succeed()) + }) + + It("should create policy on NetBird", func() { + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command( + "kubectl", "get", "NBPolicy", "custom-default-exposable", "-o", "jsonpath={.status.tcpPolicyID}", + )) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).NotTo(BeEmpty()) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + policies, err := netbirdClient.Policies.List(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(policies).To(HaveLen(1)) + g.Expect(policies[0].Name).To(ContainSubstring("E2E")) + }).Should(Succeed()) + }) + }) + }) }) }) diff --git a/test/utils/management/compose.yml b/test/utils/management/compose.yml new file mode 100644 index 0000000..75bd69b --- /dev/null +++ b/test/utils/management/compose.yml @@ -0,0 +1,85 @@ +services: + mock-jwt-provider: + image: nginx:alpine + configs: + - source: nginx_conf + target: /etc/nginx/nginx.conf + networks: + transfer_net: + ipv4_address: 192.168.1.11 + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/.well-known/jwks.json" ] + interval: 3s + timeout: 1s + retries: 15 + + netbird-mgmt: + image: netbirdio/management:0.38.0 + environment: + - NB_LOG_LEVEL=debug + - NETBIRD_STORE_ENGINE=sqlite + volumes: + - ./seed_database.sql:/app/seed_database.sql + - ./management.json:/etc/netbird/management.json + - ./setup_mgmt.sh:/etc/netbird/setup_mgmt.sh + - /tmp/empty:/var/lib/netbird/GeoLite2-City_20240101.mmdb + - /tmp/empty:/var/lib/netbird/geonames_20240101.db + - netbird_data:/var/lib/netbird + entrypoint: [] + command: + - /bin/bash + - -c + - /etc/netbird/setup_mgmt.sh & /go/bin/netbird-mgmt management --log-file console + networks: + transfer_net: + ipv4_address: 192.168.1.10 + depends_on: + mock-jwt-provider: + condition: service_healthy + healthcheck: + test: [ "CMD", "sh", "-c", "grep -q '0:0050' /proc/net/tcp*" ] + interval: 3s + timeout: 1s + retries: 10 + ports: + # local debugging only + - "8080:80" + +networks: + transfer_net: + ipam: + config: + - subnet: 192.168.1.0/24 + +volumes: + netbird_data: + name: "netbird_data_${COMPOSE_PROJECT_NAME:-default}_tmp" + external: false + driver: local + driver_opts: + type: tmpfs + device: tmpfs + +configs: + nginx_conf: + content: | + user nginx; + worker_processes auto; + pid /var/run/nginx.pid; + events {} + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + sendfile on; + + server { + listen 8080; + listen [::]:8080; + server_name localhost; + + location /.well-known/jwks.json { + return 200 '{"keys":[{"kid":"mockKid","kty":"RSA","alg":"RS256","use":"sig","n":"mockModulus","e":"AQAB"}]}'; + } + } + } \ No newline at end of file diff --git a/test/utils/management/management.json b/test/utils/management/management.json new file mode 100644 index 0000000..3834bbd --- /dev/null +++ b/test/utils/management/management.json @@ -0,0 +1,67 @@ +{ + "Stuns": [ + ], + "TURNConfig": { + "TimeBasedCredentials": true, + "CredentialsTTL": "12h0m0s", + "Secret": "", + "Turns": [ + ] + }, + "Signal": { + "Proto": "http", + "URI": "192.168.1.12:80", + "Username": "", + "Password": "" + }, + "Datadir": "/var/lib/netbird", + "DataStoreEncryptionKey": "0DStg6EdSwXQDOnr7uZHNBfJmzDsoqmR", + "HttpConfig": { + "AuthIssuer": "http://192.168.1.11:8080", + "AuthAudience": "mock-audience", + "AuthUserIDClaim": "sub", + "AuthKeysLocation": "http://192.168.1.11:8080/.well-known/jwks.json", + "IdpSignKeyRefreshEnabled": false + }, + "IdpManagerConfig": { + "ManagerType": "", + "ClientConfig": { + "Issuer": "", + "TokenEndpoint": "", + "ClientID": "", + "ClientSecret": "", + "GrantType": "" + }, + "ExtraConfig": { + "Audience": "" + }, + "Auth0ClientCredentials": null, + "AzureClientCredentials": null, + "KeycloakClientCredentials": null, + "ZitadelClientCredentials": null + }, + "DeviceAuthorizationFlow": { + "Provider": "", + "ProviderConfig": { + "ClientID": "", + "ClientSecret": "", + "Domain": "", + "Audience": "", + "TokenEndpoint": "", + "DeviceAuthEndpoint": "", + "AuthorizationEndpoint": "", + "Scope": "", + "UseIDToken": false, + "RedirectURLs": null + } + }, + "PKCEAuthorizationFlow": null, + "StoreConfig": { + "Engine": "" + }, + "ReverseProxy": { + "TrustedHTTPProxies": null, + "TrustedHTTPProxiesCount": 0, + "TrustedPeers": null + } +} diff --git a/test/utils/management/seed_database.sql b/test/utils/management/seed_database.sql new file mode 100644 index 0000000..65256ff --- /dev/null +++ b/test/utils/management/seed_database.sql @@ -0,0 +1,15 @@ +-- Seed Accounts table +INSERT INTO accounts (id, created_by, created_at, domain, domain_category, is_domain_primary_account, network_identifier, network_serial, settings_peer_login_expiration_enabled, settings_peer_login_expiration, settings_regular_users_view_blocked, settings_groups_propagation_enabled, settings_jwt_groups_enabled, settings_extra_peer_approval_enabled, network_net) +VALUES ('account1', 'user1', '2024-04-17 09:35:50.651027026+00:00', 'netbird.selfhosted', 'private', 1, 'network1', 694, 1, 86400000000000, 1, 1, 0, 0, '{"IP":"100.64.0.0","Mask":"//8AAA=="}'); + +-- Seed Users table +INSERT INTO users (id, account_id, role, is_service_user, non_deletable, blocked, created_at, issued) +VALUES ('user1', 'account1', 'owner', 0, 0, 0, '2024-08-12 00:00:00', 'api'); + +-- Seed Groups table +INSERT INTO groups (id, account_id, name, issued, peers, integration_ref_id, integration_ref_integration_type) +VALUES ('group-all', 'account1', 'All', 'api', '[]', 0, NULL); + +-- Seed Personal Access Tokens (API Keys) table +INSERT INTO personal_access_tokens (id, user_id, name, hashed_token, expiration_date, created_by, created_at, last_used) +VALUES ('1', 'user1', 'Test API Key', 'smJvzexPcQ3NRezrVDUmF++0XqvFvXzx8Rsn2y9r1z0=', '2124-08-12 00:00:00', 'user1', '2024-08-12 00:00:00', NULL); diff --git a/test/utils/management/setup_mgmt.sh b/test/utils/management/setup_mgmt.sh new file mode 100755 index 0000000..d59dcda --- /dev/null +++ b/test/utils/management/setup_mgmt.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +# Install sqlite3 +apt-get update && apt-get install -y sqlite3 curl jq + +while true +do + # Check if the schema is ready + if sqlite3 /var/lib/netbird/store.db "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts';" | grep -q accounts; then + echo "Database schema detected." + + # Check if the database is empty + if [ $(sqlite3 /var/lib/netbird/store.db "SELECT COUNT(*) FROM accounts;") -eq 0 ]; then + echo "Seeding the database..." + sqlite3 /var/lib/netbird/store.db < /app/seed_database.sql + echo "Database seeded successfully." + exit 0 + else + echo "Database already contains data, skipping seeding." + exit 1 + fi + else + echo "Database schema not found. Waiting..." + fi +done \ No newline at end of file diff --git a/test/utils/values.yaml b/test/utils/values.yaml new file mode 100644 index 0000000..65631dd --- /dev/null +++ b/test/utils/values.yaml @@ -0,0 +1,15 @@ +netbirdAPI: + key: "nbp_apTmlmUXHSC4PKmHwtIZNaGr8eqcVI2gMURp" + +ingress: + enabled: true + allowAutomaticPolicyCreation: true + kubernetesAPI: + enabled: false + router: + enabled: true + policies: + default: + name: Kubernetes Default Group + sourceGroups: + - All \ No newline at end of file