netbird/kubernetes-operator
0
0
Fork 0
mirror of https://github.com/netbirdio/kubernetes-operator.git synced 2026-05-22 17:11:40 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add E2E tests for ingress feature (#35)

Browse Source
This commit is contained in:
M. Essam
2025-04-23 11:05:17 +02:00
committed by GitHub
parent 6d5b03c4ca
commit 03830ae2b7
8 changed files with 460 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+17
View File
@@ -133,10 +133,27 @@ endif
deploy: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config.
$(HELM) install -n netbird --create-namespace kubernetes-operator --set operator.image.tag=$(word 2,$(subst :, ,${IMG})) --repo https://netbirdio.github.io/helms kubernetes-operator
.PHONY: deploy-e2e
deploy-e2e: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config.
$(HELM) install -n netbird --create-namespace kubernetes-operator -f ./test/utils/values.yaml --set operator.image.tag=$(word 2,$(subst :, ,${IMG})) --set managementURL=${MGMT_HOST} --repo https://netbirdio.github.io/helms kubernetes-operator
.PHONY: undeploy
undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(HELM) uninstall -n netbird kubernetes-operator --no-hooks
.PHONY: undeploy-e2e
undeploy-e2e: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(HELM) uninstall -n netbird kubernetes-operator --no-hooks || true
kubectl get NBResource -n default -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBResource -n default -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl get NBGroup -n default -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBGroup -n default -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl get NBGroup -n netbird -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBGroup -n netbird -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl get NBRoutingPeer -n netbird -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBRoutingPeer -n netbird -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl get NBPolicy -o "custom-columns=NAME:.metadata.name" --no-headers | xargs -r -n 1 kubectl patch NBPolicy -p '{"metadata":{"finalizers":null}}' --type=merge
kubectl delete NBGroup -A --all
kubectl delete NBResource -A --all
kubectl delete NBRoutingPeer -A --all
kubectl delete NBPolicy --all
##@ Dependencies
## Location to install dependencies to
test/e2e/e2e_suite_test.go
+26
View File
@@ -17,17 +17,22 @@ limitations under the License.
package e2e
import (
"context"
"fmt"
"os"
"os/exec"
"path"
"testing"
netbird "github.com/netbirdio/netbird/management/client/rest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/netbirdio/kubernetes-operator/test/utils"
)
const apiToken = "nbp_apTmlmUXHSC4PKmHwtIZNaGr8eqcVI2gMURp"
var (
// Optional Environment Variables:
// - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup.
@@ -72,6 +77,20 @@ var _ = BeforeSuite(func() {
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n")
}
}
By("starting up the test management")
cmd = exec.Command("docker", "compose", "up", "-d")
curDir, err := utils.GetProjectDir()
Expect(err).NotTo(HaveOccurred())
cmd.Dir = path.Join(curDir, "test", "utils", "management")
out, err := cmd.CombinedOutput()
Expect(err).NotTo(HaveOccurred(), string(out))
Eventually(func() error {
client := netbird.New("http://127.0.0.1:8080", apiToken)
_, err = client.Accounts.List(context.Background())
return err
}).Should(Succeed())
})
var _ = AfterSuite(func() {
@@ -80,4 +99,11 @@ var _ = AfterSuite(func() {
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
utils.UninstallCertManager()
}
cmd := exec.Command("docker", "compose", "down")
curDir, err := utils.GetProjectDir()
Expect(err).NotTo(HaveOccurred())
cmd.Dir = path.Join(curDir, "test", "utils", "management")
out, err := cmd.CombinedOutput()
Expect(err).NotTo(HaveOccurred(), string(out))
})
test/e2e/e2e_test.go
+208 -3
View File
@@ -17,11 +17,15 @@ limitations under the License.
package e2e
import (
"context"
"encoding/json"
"fmt"
"os/exec"
"strings"
"time"
netbird "github.com/netbirdio/netbird/management/client/rest"
"github.com/netbirdio/netbird/management/server/http/api"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -36,6 +40,7 @@ const metricsServiceName = "kubernetes-operator-metrics"
var _ = Describe("Manager", Ordered, func() {
var controllerPodName string
var netbirdClient *netbird.Client
// Before running the tests, set up the environment by creating the namespace,
// enforce the restricted security policy to the namespace, installing CRDs,
@@ -53,12 +58,37 @@ var _ = Describe("Manager", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
By("deploying the kubernetes-operator")
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))
out, err := utils.Run(cmd)
out, err := utils.Run(exec.Command("kubectl", "get", "node", "-o", "json"))
Expect(err).NotTo(HaveOccurred(), "Failed to get nodes")
nodesGetOutput := make(map[string]any)
err = json.Unmarshal([]byte(out), &nodesGetOutput)
Expect(err).NotTo(HaveOccurred(), "Failed to get nodes")
nodeIPs := (nodesGetOutput["items"].([]any))[0].(map[string]any)["status"].(map[string]any)["addresses"].([]any)
managementIP := ""
for _, v := range nodeIPs {
addrType := v.(map[string]any)["type"]
if addrType == "InternalIP" {
managementIP = v.(map[string]any)["address"].(string)
break
}
}
Expect(managementIP).NotTo(BeEmpty())
managementIPParts := strings.Split(managementIP, ".")
managementIP = fmt.Sprintf("http://%s.%s.0.1:8080", managementIPParts[0], managementIPParts[1])
cmd = exec.Command(
"make",
"deploy-e2e",
fmt.Sprintf("IMG=%s", projectImage),
fmt.Sprintf("MGMT_HOST=%s", managementIP),
)
out, err = utils.Run(cmd)
if err != nil {
fmt.Println(out)
}
Expect(err).NotTo(HaveOccurred(), "Failed to deploy the kubernetes-operator")
netbirdClient = netbird.New("http://127.0.0.1:8080", apiToken)
})
// After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs,
@@ -69,7 +99,7 @@ var _ = Describe("Manager", Ordered, func() {
_, _ = utils.Run(cmd)
By("undeploying the kubernetes-operator")
cmd = exec.Command("make", "undeploy")
cmd = exec.Command("make", "undeploy-e2e")
_, _ = utils.Run(cmd)
By("removing manager namespace")
@@ -449,6 +479,181 @@ var _ = Describe("Manager", Ordered, func() {
})
})
Context("Ingress", Ordered, func() {
BeforeAll(func() {
_, err := utils.Run(exec.Command("kubectl", "create", "deployment", "--image=nginx", "exposable"))
Expect(err).NotTo(HaveOccurred())
_, err = utils.Run(exec.Command("kubectl", "expose", "deployment", "--port=80", "exposable"))
Expect(err).NotTo(HaveOccurred())
})
AfterAll(func() {
_, err := utils.Run(exec.Command("kubectl", "delete", "deployment", "--ignore-not-found", "exposable"))
Expect(err).NotTo(HaveOccurred())
_, err = utils.Run(exec.Command("kubectl", "delete", "service", "--ignore-not-found", "exposable"))
Expect(err).NotTo(HaveOccurred())
})
It("should create Network", func() {
Eventually(func(g Gomega) {
networks, err := netbirdClient.Networks.List(context.Background())
g.Expect(err).NotTo(HaveOccurred())
g.Expect(networks).To(HaveLen(1))
}).Should(Succeed())
})
networkID := func() string {
defer GinkgoHelper()
networks, err := netbirdClient.Networks.List(context.Background())
Expect(err).NotTo(HaveOccurred())
Expect(networks).To(HaveLen(1))
return networks[0].Id
}
It("should create NetworkRouter", func() {
Eventually(func(g Gomega) {
routers, err := netbirdClient.Networks.Routers(networkID()).List(context.Background())
g.Expect(err).NotTo(HaveOccurred())
g.Expect(routers).To(HaveLen(1))
}).Should(Succeed())
})
It("should create router deployment", func() {
Eventually(func(g Gomega) {
_, err := utils.Run(exec.Command("kubectl", "get", "deployment", "router", "-n", "netbird"))
Expect(err).NotTo(HaveOccurred())
}).Should(Succeed())
})
groupsToNames := func(i []api.Group) []string {
ret := make([]string, len(i))
for j, k := range i {
ret[j] = k.Name
}
return ret
}
It("should create router group", func() {
Eventually(func(g Gomega) {
groups, err := netbirdClient.Groups.List(context.Background())
g.Expect(err).NotTo(HaveOccurred())
g.Expect(groups).To(WithTransform(groupsToNames, ContainElement("kubernetes")))
}).Should(Succeed())
})
It("should not expose service by default", func() {
_, err := utils.Run(exec.Command("kubectl", "get", "NBResource", "exposable"))
Expect(err).To(HaveOccurred())
})
When("Service is annotated with expose", Ordered, func() {
BeforeAll(func() {
_, err := utils.Run(exec.Command("kubectl", "annotate", "service", "exposable", "netbird.io/expose=true"))
Expect(err).NotTo(HaveOccurred())
})
It("should create NBResource", func() {
Eventually(func(g Gomega) {
_, err := utils.Run(exec.Command("kubectl", "get", "NBResource", "exposable"))
g.Expect(err).NotTo(HaveOccurred())
}).Should(Succeed())
})
It("should create Network Resource", func() {
Eventually(func(g Gomega) {
resources, err := netbirdClient.Networks.Resources(networkID()).List(context.Background())
Expect(err).NotTo(HaveOccurred())
resourcesToNames := func(r []api.NetworkResource) []string {
ret := make([]string, len(r))
for i, j := range r {
ret[i] = j.Name
}
return ret
}
Expect(resources).To(WithTransform(resourcesToNames, ContainElement("default-exposable")))
}).Should(Succeed())
})
It("should create Group", func() {
Eventually(func(g Gomega) {
groups, err := netbirdClient.Groups.List(context.Background())
Expect(err).NotTo(HaveOccurred())
Expect(groups).To(WithTransform(groupsToNames, ContainElement("kubernetes-default-exposable")))
}).Should(Succeed())
})
})
When("Service is annotated with policy", Ordered, func() {
BeforeAll(func() {
_, err := utils.Run(exec.Command("kubectl", "annotate", "service", "exposable", "netbird.io/policy=default"))
Expect(err).NotTo(HaveOccurred())
})
It("should add service to policy status", func() {
Eventually(func(g Gomega) {
out, err := utils.Run(exec.Command(
"kubectl", "get", "NBPolicy", "default", "-o", "jsonpath={.status.managedServiceList}",
))
g.Expect(err).NotTo(HaveOccurred())
g.Expect(out).To(ContainSubstring("default/exposable"))
}).Should(Succeed())
})
It("should create policy on NetBird", func() {
Eventually(func(g Gomega) {
out, err := utils.Run(exec.Command(
"kubectl", "get", "NBPolicy", "default", "-o", "jsonpath={.status.tcpPolicyID}",
))
g.Expect(err).NotTo(HaveOccurred())
g.Expect(out).NotTo(BeEmpty())
}).Should(Succeed())
Eventually(func(g Gomega) {
policies, err := netbirdClient.Policies.List(context.Background())
g.Expect(err).NotTo(HaveOccurred())
g.Expect(policies).To(HaveLen(1))
g.Expect(policies[0].Name).To(ContainSubstring("Kubernetes Default Group"))
}).Should(Succeed())
})
})
When("Service is annotated with a non-existent policy", Ordered, func() {
BeforeAll(func() {
_, err := utils.Run(exec.Command(
"kubectl", "annotate", "service", "exposable", "--overwrite",
"netbird.io/policy=custom", "netbird.io/policy-source-groups=All", "netbird.io/policy-name=custom:E2E",
))
Expect(err).NotTo(HaveOccurred())
})
It("should create NBPolicy", func() {
Eventually(func(g Gomega) {
out, err := utils.Run(exec.Command(
"kubectl", "get", "NBPolicy", "custom-default-exposable", "-o", "jsonpath={.status.managedServiceList}",
))
g.Expect(err).NotTo(HaveOccurred())
g.Expect(out).To(ContainSubstring("default/exposable"))
}).Should(Succeed())
})
It("should create policy on NetBird", func() {
Eventually(func(g Gomega) {
out, err := utils.Run(exec.Command(
"kubectl", "get", "NBPolicy", "custom-default-exposable", "-o", "jsonpath={.status.tcpPolicyID}",
))
g.Expect(err).NotTo(HaveOccurred())
g.Expect(out).NotTo(BeEmpty())
}).Should(Succeed())
Eventually(func(g Gomega) {
policies, err := netbirdClient.Policies.List(context.Background())
g.Expect(err).NotTo(HaveOccurred())
g.Expect(policies).To(HaveLen(1))
g.Expect(policies[0].Name).To(ContainSubstring("E2E"))
}).Should(Succeed())
})
})
})
})
})
test/utils/management/compose.yml
+85
View File
@@ -0,0 +1,85 @@
services:
mock-jwt-provider:
image: nginx:alpine
configs:
- source: nginx_conf
target: /etc/nginx/nginx.conf
networks:
transfer_net:
ipv4_address: 192.168.1.11
healthcheck:
test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/.well-known/jwks.json" ]
interval: 3s
timeout: 1s
retries: 15
netbird-mgmt:
image: netbirdio/management:0.38.0
environment:
- NB_LOG_LEVEL=debug
- NETBIRD_STORE_ENGINE=sqlite
volumes:
- ./seed_database.sql:/app/seed_database.sql
- ./management.json:/etc/netbird/management.json
- ./setup_mgmt.sh:/etc/netbird/setup_mgmt.sh
- /tmp/empty:/var/lib/netbird/GeoLite2-City_20240101.mmdb
- /tmp/empty:/var/lib/netbird/geonames_20240101.db
- netbird_data:/var/lib/netbird
entrypoint: []
command:
- /bin/bash
- -c
- /etc/netbird/setup_mgmt.sh & /go/bin/netbird-mgmt management --log-file console
networks:
transfer_net:
ipv4_address: 192.168.1.10
depends_on:
mock-jwt-provider:
condition: service_healthy
healthcheck:
test: [ "CMD", "sh", "-c", "grep -q '0:0050' /proc/net/tcp*" ]
interval: 3s
timeout: 1s
retries: 10
ports:
# local debugging only
- "8080:80"
networks:
transfer_net:
ipam:
config:
- subnet: 192.168.1.0/24
volumes:
netbird_data:
name: "netbird_data_${COMPOSE_PROJECT_NAME:-default}_tmp"
external: false
driver: local
driver_opts:
type: tmpfs
device: tmpfs
configs:
nginx_conf:
content: |
user nginx;
worker_processes auto;
pid /var/run/nginx.pid;
events {}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
sendfile on;
server {
listen 8080;
listen [::]:8080;
server_name localhost;
location /.well-known/jwks.json {
return 200 '{"keys":[{"kid":"mockKid","kty":"RSA","alg":"RS256","use":"sig","n":"mockModulus","e":"AQAB"}]}';
}
}
}
test/utils/management/management.json
+67
View File
@@ -0,0 +1,67 @@
{
"Stuns": [
],
"TURNConfig": {
"TimeBasedCredentials": true,
"CredentialsTTL": "12h0m0s",
"Secret": "",
"Turns": [
]
},
"Signal": {
"Proto": "http",
"URI": "192.168.1.12:80",
"Username": "",
"Password": ""
},
"Datadir": "/var/lib/netbird",
"DataStoreEncryptionKey": "0DStg6EdSwXQDOnr7uZHNBfJmzDsoqmR",
"HttpConfig": {
"AuthIssuer": "http://192.168.1.11:8080",
"AuthAudience": "mock-audience",
"AuthUserIDClaim": "sub",
"AuthKeysLocation": "http://192.168.1.11:8080/.well-known/jwks.json",
"IdpSignKeyRefreshEnabled": false
},
"IdpManagerConfig": {
"ManagerType": "",
"ClientConfig": {
"Issuer": "",
"TokenEndpoint": "",
"ClientID": "",
"ClientSecret": "",
"GrantType": ""
},
"ExtraConfig": {
"Audience": ""
},
"Auth0ClientCredentials": null,
"AzureClientCredentials": null,
"KeycloakClientCredentials": null,
"ZitadelClientCredentials": null
},
"DeviceAuthorizationFlow": {
"Provider": "",
"ProviderConfig": {
"ClientID": "",
"ClientSecret": "",
"Domain": "",
"Audience": "",
"TokenEndpoint": "",
"DeviceAuthEndpoint": "",
"AuthorizationEndpoint": "",
"Scope": "",
"UseIDToken": false,
"RedirectURLs": null
}
},
"PKCEAuthorizationFlow": null,
"StoreConfig": {
"Engine": ""
},
"ReverseProxy": {
"TrustedHTTPProxies": null,
"TrustedHTTPProxiesCount": 0,
"TrustedPeers": null
}
}
test/utils/management/seed_database.sql
+15
View File
@@ -0,0 +1,15 @@
-- Seed Accounts table
INSERT INTO accounts (id, created_by, created_at, domain, domain_category, is_domain_primary_account, network_identifier, network_serial, settings_peer_login_expiration_enabled, settings_peer_login_expiration, settings_regular_users_view_blocked, settings_groups_propagation_enabled, settings_jwt_groups_enabled, settings_extra_peer_approval_enabled, network_net)
VALUES ('account1', 'user1', '2024-04-17 09:35:50.651027026+00:00', 'netbird.selfhosted', 'private', 1, 'network1', 694, 1, 86400000000000, 1, 1, 0, 0, '{"IP":"100.64.0.0","Mask":"//8AAA=="}');
-- Seed Users table
INSERT INTO users (id, account_id, role, is_service_user, non_deletable, blocked, created_at, issued)
VALUES ('user1', 'account1', 'owner', 0, 0, 0, '2024-08-12 00:00:00', 'api');
-- Seed Groups table
INSERT INTO groups (id, account_id, name, issued, peers, integration_ref_id, integration_ref_integration_type)
VALUES ('group-all', 'account1', 'All', 'api', '[]', 0, NULL);
-- Seed Personal Access Tokens (API Keys) table
INSERT INTO personal_access_tokens (id, user_id, name, hashed_token, expiration_date, created_by, created_at, last_used)
VALUES ('1', 'user1', 'Test API Key', 'smJvzexPcQ3NRezrVDUmF++0XqvFvXzx8Rsn2y9r1z0=', '2124-08-12 00:00:00', 'user1', '2024-08-12 00:00:00', NULL);
test/utils/management/setup_mgmt.sh
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/bin/bash
set -e
# Install sqlite3
apt-get update && apt-get install -y sqlite3 curl jq
while true
do
# Check if the schema is ready
if sqlite3 /var/lib/netbird/store.db "SELECT name FROM sqlite_master WHERE type='table' AND name='accounts';" | grep -q accounts; then
echo "Database schema detected."
# Check if the database is empty
if [ $(sqlite3 /var/lib/netbird/store.db "SELECT COUNT(*) FROM accounts;") -eq 0 ]; then
echo "Seeding the database..."
sqlite3 /var/lib/netbird/store.db < /app/seed_database.sql
echo "Database seeded successfully."
exit 0
else
echo "Database already contains data, skipping seeding."
exit 1
fi
else
echo "Database schema not found. Waiting..."
fi
done
test/utils/values.yaml
+15
View File
@@ -0,0 +1,15 @@
netbirdAPI:
key: "nbp_apTmlmUXHSC4PKmHwtIZNaGr8eqcVI2gMURp"
ingress:
enabled: true
allowAutomaticPolicyCreation: true
kubernetesAPI:
enabled: false
router:
enabled: true
policies:
default:
name: Kubernetes Default Group
sourceGroups:
- All