From 5e2bc5659e60a23f49793bb1274e44da93c5a57f Mon Sep 17 00:00:00 2001 From: Pascal Fischer Date: Wed, 28 May 2025 23:05:52 +0200 Subject: [PATCH] add metrics + network map --- .../controllers/network_map/controller.go | 31 ++++ internal/controllers/network_map/metrics.go | 31 ++++ .../controllers/network_map/repository.go | 65 +++++++++ .../controllers/network_map/updatechannel.go | 2 +- internal/modules/accounts/manager.go | 9 -- .../modules/accounts/{ => manager}/api.go | 2 +- internal/modules/accounts/manager/manager.go | 10 ++ .../accounts/{ => manager}/repository.go | 2 +- internal/modules/accounts/network.go | 73 ++++++++++ .../accounts/settings/{ => manager}/api.go | 2 +- .../settings/{ => manager}/manager.go | 2 +- .../settings/{ => manager}/repository.go | 2 +- .../accounts/settings/{types => }/settings.go | 2 +- internal/modules/groups/group.go | 25 ++++ internal/modules/groups/{ => manager}/api.go | 2 +- .../modules/groups/{ => manager}/manager.go | 2 +- .../groups/{ => manager}/repository.go | 2 +- internal/modules/groups/resource.go | 28 ++++ .../resources/{interface.go => manager.go} | 9 +- .../networks/resources/manager/manager.go | 1 - internal/modules/policies/manager/api.go | 1 + internal/modules/policies/manager/manager.go | 1 + .../modules/policies/manager/repository.go | 1 + internal/modules/policies/policy.go | 136 ++++++++++++++++++ internal/modules/policies/policyrule.go | 110 ++++++++++++++ internal/modules/template/interface.go | 8 ++ internal/modules/template/manager/api.go | 20 +++ internal/modules/template/manager/manager.go | 28 ++++ internal/modules/template/manager/metrics.go | 27 ++++ .../modules/template/manager/repository.go | 4 + internal/modules/template/type.go | 4 + internal/modules/users/types/user.go | 3 +- internal/server/boot.go | 46 ++++++ .../api/rest/middleware/logging_middleware.go | 11 ++ .../rest/middleware/recovery_middleware.go | 15 ++ internal/shared/api/rest/router.go | 33 +---- internal/shared/metrics/metrics.go | 83 +++++++++++ pkg/logging/init.go | 16 ++- 38 files changed, 796 insertions(+), 53 deletions(-) create mode 100644 internal/controllers/network_map/metrics.go create mode 100644 internal/controllers/network_map/repository.go rename internal/modules/accounts/{ => manager}/api.go (99%) create mode 100644 internal/modules/accounts/manager/manager.go rename internal/modules/accounts/{ => manager}/repository.go (95%) create mode 100644 internal/modules/accounts/network.go rename internal/modules/accounts/settings/{ => manager}/api.go (99%) rename internal/modules/accounts/settings/{ => manager}/manager.go (99%) rename internal/modules/accounts/settings/{ => manager}/repository.go (98%) rename internal/modules/accounts/settings/{types => }/settings.go (99%) create mode 100644 internal/modules/groups/group.go rename internal/modules/groups/{ => manager}/api.go (99%) rename internal/modules/groups/{ => manager}/manager.go (99%) rename internal/modules/groups/{ => manager}/repository.go (95%) create mode 100644 internal/modules/groups/resource.go rename internal/modules/networks/resources/{interface.go => manager.go} (91%) create mode 100644 internal/modules/policies/manager/api.go create mode 100644 internal/modules/policies/manager/manager.go create mode 100644 internal/modules/policies/manager/repository.go create mode 100644 internal/modules/policies/policy.go create mode 100644 internal/modules/policies/policyrule.go create mode 100644 internal/modules/template/interface.go create mode 100644 internal/modules/template/manager/api.go create mode 100644 internal/modules/template/manager/manager.go create mode 100644 internal/modules/template/manager/metrics.go create mode 100644 internal/modules/template/manager/repository.go create mode 100644 internal/modules/template/type.go create mode 100644 internal/shared/api/rest/middleware/recovery_middleware.go create mode 100644 internal/shared/metrics/metrics.go diff --git a/internal/controllers/network_map/controller.go b/internal/controllers/network_map/controller.go index 327053e..58649e7 100644 --- a/internal/controllers/network_map/controller.go +++ b/internal/controllers/network_map/controller.go @@ -1,4 +1,35 @@ package network_map +import ( + "fmt" + + log "github.com/sirupsen/logrus" + + "management/internal/shared/db" + appmetrics "management/internal/shared/metrics" +) + type Controller struct { + repo Repository + metrics *metrics +} + +func NewController(store *db.Store, metrics *appmetrics.AppMetrics) *Controller { + cMetrics, err := appmetrics.RegisterMetrics(metrics, newMetrics) + if err != nil { + log.Fatalf("Failed to register app metrics: %v", err) + } + return &Controller{ + repo: newRepository(store, cMetrics), + metrics: cMetrics, + } +} + +func (c *Controller) UpdatePeers(accountID string) error { + data, err := c.repo.GetNetworkMapData(accountID) + if err != nil { + return fmt.Errorf("get network map data: %w", err) + } + + return nil } diff --git a/internal/controllers/network_map/metrics.go b/internal/controllers/network_map/metrics.go new file mode 100644 index 0000000..1143540 --- /dev/null +++ b/internal/controllers/network_map/metrics.go @@ -0,0 +1,31 @@ +package network_map + +import ( + "context" + "time" + + "go.opentelemetry.io/otel/metric" +) + +type metrics struct { + dbAccessDuration metric.Int64Histogram +} + +func newMetrics(meter metric.Meter) (*metrics, error) { + dbAccessDuration, err := meter.Int64Histogram( + "sync_request_duration_seconds", + metric.WithDescription("Duration of sync requests in seconds"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + return &metrics{ + dbAccessDuration: dbAccessDuration, + }, nil +} + +func (m *metrics) RecordDBAccessDuration(duration time.Duration) { + m.dbAccessDuration.Record(context.Background(), duration.Milliseconds(), metric.WithAttributes()) +} diff --git a/internal/controllers/network_map/repository.go b/internal/controllers/network_map/repository.go new file mode 100644 index 0000000..0ecbaac --- /dev/null +++ b/internal/controllers/network_map/repository.go @@ -0,0 +1,65 @@ +package network_map + +import ( + "time" + + nbpeer "github.com/netbirdio/netbird/management/server/peer" + + "management/internal/modules/accounts" + "management/internal/modules/groups" + "management/internal/modules/networks" + "management/internal/modules/networks/resources" + "management/internal/modules/networks/routers" + "management/internal/modules/policies" + "management/internal/shared/db" +) + +type NetworkMapData struct { + // we have to name column to aid as it collides with Network.Id when work with associations + Id string `gorm:"primaryKey"` + + Domain string `gorm:"index"` + DomainCategory string + IsDomainPrimaryAccount bool + Network *accounts.Network `gorm:"embedded;embeddedPrefix:network_"` + Peers []nbpeer.Peer `json:"-" gorm:"foreignKey:AccountID;references:id"` + Groups []groups.Group `json:"-" gorm:"foreignKey:AccountID;references:id"` + Policies []*policies.Policy `gorm:"foreignKey:AccountID;references:id"` + + Networks []*networks.Network `gorm:"foreignKey:AccountID;references:id"` + NetworkRouters []*routers.NetworkRouter `gorm:"foreignKey:AccountID;references:id"` + NetworkResources []*resources.NetworkResource `gorm:"foreignKey:AccountID;references:id"` +} + +type Repository interface { + GetNetworkMapData(accountID string) (*NetworkMapData, error) +} + +type repository struct { + store *db.Store + metrics *metrics +} + +func newRepository(s *db.Store, metrics *metrics) Repository { + return &repository{ + store: s, + metrics: metrics, + } +} + +func (r *repository) GetNetworkMapData(accountID string) (*NetworkMapData, error) { + start := time.Now() + var networkMapData NetworkMapData + err := r.store.GetOne(nil, db.LockingStrengthShare, &networkMapData, "id = ?", accountID) + if err != nil { + return nil, err + } + + if err := r.store.Load(&networkMapData, "Peers", "Groups", "Policies", "Networks", "NetworkRouters", "NetworkResources"); err != nil { + return nil, err + } + + r.metrics.RecordDBAccessDuration(time.Since(start)) + + return &networkMapData, nil +} diff --git a/internal/controllers/network_map/updatechannel.go b/internal/controllers/network_map/updatechannel.go index 7789d1c..e804a01 100644 --- a/internal/controllers/network_map/updatechannel.go +++ b/internal/controllers/network_map/updatechannel.go @@ -1,4 +1,4 @@ -package server +package network_map import ( "context" diff --git a/internal/modules/accounts/manager.go b/internal/modules/accounts/manager.go index 12ef7cc..ec6c5a1 100644 --- a/internal/modules/accounts/manager.go +++ b/internal/modules/accounts/manager.go @@ -1,10 +1 @@ package accounts - -import "management/pkg/logging" - -var log = logging.LoggerForThisPackage() - -type Manager struct { - repo Repository - handler *handler -} diff --git a/internal/modules/accounts/api.go b/internal/modules/accounts/manager/api.go similarity index 99% rename from internal/modules/accounts/api.go rename to internal/modules/accounts/manager/api.go index 4acfaac..c8e1dd3 100644 --- a/internal/modules/accounts/api.go +++ b/internal/modules/accounts/manager/api.go @@ -1,4 +1,4 @@ -package accounts +package manager import ( "encoding/json" diff --git a/internal/modules/accounts/manager/manager.go b/internal/modules/accounts/manager/manager.go new file mode 100644 index 0000000..c07f9df --- /dev/null +++ b/internal/modules/accounts/manager/manager.go @@ -0,0 +1,10 @@ +package manager + +import "management/pkg/logging" + +var log = logging.LoggerForThisPackage() + +type Manager struct { + repo Repository + handler *handler +} diff --git a/internal/modules/accounts/repository.go b/internal/modules/accounts/manager/repository.go similarity index 95% rename from internal/modules/accounts/repository.go rename to internal/modules/accounts/manager/repository.go index 5057b8e..4f401a3 100644 --- a/internal/modules/accounts/repository.go +++ b/internal/modules/accounts/manager/repository.go @@ -1,4 +1,4 @@ -package accounts +package manager import "management/internal/shared/db" diff --git a/internal/modules/accounts/network.go b/internal/modules/accounts/network.go new file mode 100644 index 0000000..456e0a3 --- /dev/null +++ b/internal/modules/accounts/network.go @@ -0,0 +1,73 @@ +package accounts + +import ( + "math/rand" + "net" + "sync" + "time" + + "github.com/c-robinson/iplib" + "github.com/rs/xid" +) + +const ( + // SubnetSize is a size of the subnet of the global network, e.g. 100.77.0.0/16 + SubnetSize = 16 + // NetSize is a global network size 100.64.0.0/10 + NetSize = 10 + + // AllowedIPsFormat generates Wireguard AllowedIPs format (e.g. 100.64.30.1/32) + AllowedIPsFormat = "%s/32" +) + +type Network struct { + Identifier string `json:"id"` + Net net.IPNet `gorm:"serializer:json"` + Dns string + // Serial is an ID that increments by 1 when any change to the network happened (e.g. new peer has been added). + // Used to synchronize state to the client apps. + Serial uint64 + + Mu sync.Mutex `json:"-" gorm:"-"` +} + +// NewNetwork creates a new Network initializing it with a Serial=0 +// It takes a random /16 subnet from 100.64.0.0/10 (64 different subnets) +func NewNetwork() *Network { + + n := iplib.NewNet4(net.ParseIP("100.64.0.0"), NetSize) + sub, _ := n.Subnet(SubnetSize) + + s := rand.NewSource(time.Now().Unix()) + r := rand.New(s) + intn := r.Intn(len(sub)) + + return &Network{ + Identifier: xid.New().String(), + Net: sub[intn].IPNet, + Dns: "", + Serial: 0} +} + +// IncSerial increments Serial by 1 reflecting that the network state has been changed +func (n *Network) IncSerial() { + n.Mu.Lock() + defer n.Mu.Unlock() + n.Serial++ +} + +// CurrentSerial returns the Network.Serial of the network (latest state id) +func (n *Network) CurrentSerial() uint64 { + n.Mu.Lock() + defer n.Mu.Unlock() + return n.Serial +} + +func (n *Network) Copy() *Network { + return &Network{ + Identifier: n.Identifier, + Net: n.Net, + Dns: n.Dns, + Serial: n.Serial, + } +} diff --git a/internal/modules/accounts/settings/api.go b/internal/modules/accounts/settings/manager/api.go similarity index 99% rename from internal/modules/accounts/settings/api.go rename to internal/modules/accounts/settings/manager/api.go index 900c663..3c0050e 100644 --- a/internal/modules/accounts/settings/api.go +++ b/internal/modules/accounts/settings/manager/api.go @@ -1,4 +1,4 @@ -package settings +package manager import ( "encoding/json" diff --git a/internal/modules/accounts/settings/manager.go b/internal/modules/accounts/settings/manager/manager.go similarity index 99% rename from internal/modules/accounts/settings/manager.go rename to internal/modules/accounts/settings/manager/manager.go index aae981e..1527c3b 100644 --- a/internal/modules/accounts/settings/manager.go +++ b/internal/modules/accounts/settings/manager/manager.go @@ -1,4 +1,4 @@ -package settings +package manager import ( "context" diff --git a/internal/modules/accounts/settings/repository.go b/internal/modules/accounts/settings/manager/repository.go similarity index 98% rename from internal/modules/accounts/settings/repository.go rename to internal/modules/accounts/settings/manager/repository.go index 7d6d033..8320f9b 100644 --- a/internal/modules/accounts/settings/repository.go +++ b/internal/modules/accounts/settings/manager/repository.go @@ -1,4 +1,4 @@ -package settings +package manager import ( "management/internal/modules/accounts/settings/types" diff --git a/internal/modules/accounts/settings/types/settings.go b/internal/modules/accounts/settings/settings.go similarity index 99% rename from internal/modules/accounts/settings/types/settings.go rename to internal/modules/accounts/settings/settings.go index 7054ede..736be8f 100644 --- a/internal/modules/accounts/settings/types/settings.go +++ b/internal/modules/accounts/settings/settings.go @@ -1,4 +1,4 @@ -package types +package settings import ( "time" diff --git a/internal/modules/groups/group.go b/internal/modules/groups/group.go new file mode 100644 index 0000000..e2b31c9 --- /dev/null +++ b/internal/modules/groups/group.go @@ -0,0 +1,25 @@ +package groups + +import "github.com/netbirdio/netbird/management/server/integration_reference" + +type Group struct { + // ID of the group + ID string `gorm:"primaryKey"` + + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` + + // Name visible in the UI + Name string + + // Issued defines how this group was created (enum of "api", "integration" or "jwt") + Issued string + + // Peers list of the group + Peers []string `gorm:"serializer:json"` + + // Resources contains a list of resources in that group + Resources []Resource `gorm:"serializer:json"` + + IntegrationReference integration_reference.IntegrationReference `gorm:"embedded;embeddedPrefix:integration_ref_"` +} diff --git a/internal/modules/groups/api.go b/internal/modules/groups/manager/api.go similarity index 99% rename from internal/modules/groups/api.go rename to internal/modules/groups/manager/api.go index 992d699..7b46e7c 100644 --- a/internal/modules/groups/api.go +++ b/internal/modules/groups/manager/api.go @@ -1,4 +1,4 @@ -package groups +package manager import ( "encoding/json" diff --git a/internal/modules/groups/manager.go b/internal/modules/groups/manager/manager.go similarity index 99% rename from internal/modules/groups/manager.go rename to internal/modules/groups/manager/manager.go index f50710f..0ed2f58 100644 --- a/internal/modules/groups/manager.go +++ b/internal/modules/groups/manager/manager.go @@ -1,4 +1,4 @@ -package groups +package manager import ( "context" diff --git a/internal/modules/groups/repository.go b/internal/modules/groups/manager/repository.go similarity index 95% rename from internal/modules/groups/repository.go rename to internal/modules/groups/manager/repository.go index c4dc24f..f8e342f 100644 --- a/internal/modules/groups/repository.go +++ b/internal/modules/groups/manager/repository.go @@ -1,4 +1,4 @@ -package groups +package manager import ( "management/internal/shared/db" diff --git a/internal/modules/groups/resource.go b/internal/modules/groups/resource.go new file mode 100644 index 0000000..a43e94d --- /dev/null +++ b/internal/modules/groups/resource.go @@ -0,0 +1,28 @@ +package groups + +import "github.com/netbirdio/netbird/management/server/http/api" + +type Resource struct { + ID string + Type string +} + +func (r *Resource) ToAPIResponse() *api.Resource { + if r.ID == "" && r.Type == "" { + return nil + } + + return &api.Resource{ + Id: r.ID, + Type: api.ResourceType(r.Type), + } +} + +func (r *Resource) FromAPIRequest(req *api.Resource) { + if req == nil { + return + } + + r.ID = req.Id + r.Type = string(req.Type) +} diff --git a/internal/modules/networks/resources/interface.go b/internal/modules/networks/resources/manager.go similarity index 91% rename from internal/modules/networks/resources/interface.go rename to internal/modules/networks/resources/manager.go index 59cc9f6..8abe1aa 100644 --- a/internal/modules/networks/resources/interface.go +++ b/internal/modules/networks/resources/manager.go @@ -7,7 +7,14 @@ import ( ) type Manager interface { - DeleteResource(ctx context.Context, tx db.Transaction, accountID, userID, networkID, resourceID string) error + // Create + + // Read GetNetworkResourcesByNetID(ctx context.Context, tx db.Transaction, lockingStrength db.LockingStrength, accountID, userID, networkID string) ([]*NetworkResource, error) + + // Update + + // Delete + DeleteResource(ctx context.Context, tx db.Transaction, accountID, userID, networkID, resourceID string) error DeleteResourcesInNetwork(ctx context.Context, tx db.Transaction, accountID, userID, networkID string) error } diff --git a/internal/modules/networks/resources/manager/manager.go b/internal/modules/networks/resources/manager/manager.go index ed0af4b..1ffa6b9 100644 --- a/internal/modules/networks/resources/manager/manager.go +++ b/internal/modules/networks/resources/manager/manager.go @@ -5,7 +5,6 @@ import ( "github.com/gorilla/mux" - "management/internal/modules/networks" "management/internal/modules/networks/resources" "management/internal/modules/networks/routers" "management/internal/shared/db" diff --git a/internal/modules/policies/manager/api.go b/internal/modules/policies/manager/api.go new file mode 100644 index 0000000..5d04392 --- /dev/null +++ b/internal/modules/policies/manager/api.go @@ -0,0 +1 @@ +package manager diff --git a/internal/modules/policies/manager/manager.go b/internal/modules/policies/manager/manager.go new file mode 100644 index 0000000..5d04392 --- /dev/null +++ b/internal/modules/policies/manager/manager.go @@ -0,0 +1 @@ +package manager diff --git a/internal/modules/policies/manager/repository.go b/internal/modules/policies/manager/repository.go new file mode 100644 index 0000000..5d04392 --- /dev/null +++ b/internal/modules/policies/manager/repository.go @@ -0,0 +1 @@ +package manager diff --git a/internal/modules/policies/policy.go b/internal/modules/policies/policy.go new file mode 100644 index 0000000..95a4e99 --- /dev/null +++ b/internal/modules/policies/policy.go @@ -0,0 +1,136 @@ +package policies + +const ( + // PolicyTrafficActionAccept indicates that the traffic is accepted + PolicyTrafficActionAccept = PolicyTrafficActionType("accept") + // PolicyTrafficActionDrop indicates that the traffic is dropped + PolicyTrafficActionDrop = PolicyTrafficActionType("drop") +) + +const ( + // PolicyRuleProtocolALL type of traffic + PolicyRuleProtocolALL = PolicyRuleProtocolType("all") + // PolicyRuleProtocolTCP type of traffic + PolicyRuleProtocolTCP = PolicyRuleProtocolType("tcp") + // PolicyRuleProtocolUDP type of traffic + PolicyRuleProtocolUDP = PolicyRuleProtocolType("udp") + // PolicyRuleProtocolICMP type of traffic + PolicyRuleProtocolICMP = PolicyRuleProtocolType("icmp") +) + +const ( + // PolicyRuleFlowDirect allows traffic from source to destination + PolicyRuleFlowDirect = PolicyRuleDirection("direct") + // PolicyRuleFlowBidirect allows traffic to both directions + PolicyRuleFlowBidirect = PolicyRuleDirection("bidirect") +) + +const ( + // DefaultRuleName is a name for the Default rule that is created for every account + DefaultRuleName = "Default" + // DefaultRuleDescription is a description for the Default rule that is created for every account + DefaultRuleDescription = "This is a default rule that allows connections between all the resources" + // DefaultPolicyName is a name for the Default policy that is created for every account + DefaultPolicyName = "Default" + // DefaultPolicyDescription is a description for the Default policy that is created for every account + DefaultPolicyDescription = "This is a default policy that allows connections between all the resources" +) + +// PolicyUpdateOperation operation object with type and values to be applied +type PolicyUpdateOperation struct { + Type PolicyUpdateOperationType + Values []string +} + +// Policy of the Rego query +type Policy struct { + // ID of the policy' + ID string `gorm:"primaryKey"` + + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` + + // Name of the Policy + Name string + + // Description of the policy visible in the UI + Description string + + // Enabled status of the policy + Enabled bool + + // Rules of the policy + Rules []*PolicyRule `gorm:"foreignKey:PolicyID;references:id;constraint:OnDelete:CASCADE;"` + + // SourcePostureChecks are ID references to Posture checks for policy source groups + SourcePostureChecks []string `gorm:"serializer:json"` +} + +// Copy returns a copy of the policy. +func (p *Policy) Copy() *Policy { + c := &Policy{ + ID: p.ID, + AccountID: p.AccountID, + Name: p.Name, + Description: p.Description, + Enabled: p.Enabled, + Rules: make([]*PolicyRule, len(p.Rules)), + SourcePostureChecks: make([]string, len(p.SourcePostureChecks)), + } + for i, r := range p.Rules { + c.Rules[i] = r.Copy() + } + copy(c.SourcePostureChecks, p.SourcePostureChecks) + return c +} + +// EventMeta returns activity event meta related to this policy +func (p *Policy) EventMeta() map[string]any { + return map[string]any{"name": p.Name} +} + +// UpgradeAndFix different version of policies to latest version +func (p *Policy) UpgradeAndFix() { + for _, r := range p.Rules { + // start migrate from version v0.20.3 + if r.Protocol == "" { + r.Protocol = PolicyRuleProtocolALL + } + if r.Protocol == PolicyRuleProtocolALL && !r.Bidirectional { + r.Bidirectional = true + } + // -- v0.20.4 + } +} + +// RuleGroups returns a list of all groups referenced in the policy's rules, +// including sources and destinations. +func (p *Policy) RuleGroups() []string { + groups := make([]string, 0) + for _, rule := range p.Rules { + groups = append(groups, rule.Sources...) + groups = append(groups, rule.Destinations...) + } + + return groups +} + +// SourceGroups returns a slice of all unique source groups referenced in the policy's rules. +func (p *Policy) SourceGroups() []string { + if len(p.Rules) == 1 { + return p.Rules[0].Sources + } + groups := make(map[string]struct{}, len(p.Rules)) + for _, rule := range p.Rules { + for _, source := range rule.Sources { + groups[source] = struct{}{} + } + } + + groupIDs := make([]string, 0, len(groups)) + for groupID := range groups { + groupIDs = append(groupIDs, groupID) + } + + return groupIDs +} diff --git a/internal/modules/policies/policyrule.go b/internal/modules/policies/policyrule.go new file mode 100644 index 0000000..c3193e1 --- /dev/null +++ b/internal/modules/policies/policyrule.go @@ -0,0 +1,110 @@ +package policies + +import ( + "github.com/netbirdio/netbird/management/proto" + + "management/internal/modules/groups" +) + +// PolicyUpdateOperationType operation type +type PolicyUpdateOperationType int + +// PolicyTrafficActionType action type for the firewall +type PolicyTrafficActionType string + +// PolicyRuleProtocolType type of traffic +type PolicyRuleProtocolType string + +// PolicyRuleDirection direction of traffic +type PolicyRuleDirection string + +// RulePortRange represents a range of ports for a firewall rule. +type RulePortRange struct { + Start uint16 + End uint16 +} + +func (r *RulePortRange) ToProto() *proto.PortInfo { + return &proto.PortInfo{ + PortSelection: &proto.PortInfo_Range_{ + Range: &proto.PortInfo_Range{ + Start: uint32(r.Start), + End: uint32(r.End), + }, + }, + } +} + +func (r *RulePortRange) Equal(other *RulePortRange) bool { + return r.Start == other.Start && r.End == other.End +} + +// PolicyRule is the metadata of the policy +type PolicyRule struct { + // ID of the policy rule + ID string `gorm:"primaryKey"` + + // PolicyID is a reference to Policy that this object belongs + PolicyID string `json:"-" gorm:"index"` + + // Name of the rule visible in the UI + Name string + + // Description of the rule visible in the UI + Description string + + // Enabled status of rule in the system + Enabled bool + + // Action policy accept or drops packets + Action PolicyTrafficActionType + + // Destinations policy destination groups + Destinations []string `gorm:"serializer:json"` + + // DestinationResource policy destination resource that the rule is applied to + DestinationResource groups.Resource `gorm:"serializer:json"` + + // Sources policy source groups + Sources []string `gorm:"serializer:json"` + + // SourceResource policy source resource that the rule is applied to + SourceResource groups.Resource `gorm:"serializer:json"` + + // Bidirectional define if the rule is applicable in both directions, sources, and destinations + Bidirectional bool + + // Protocol type of the traffic + Protocol PolicyRuleProtocolType + + // Ports or it ranges list + Ports []string `gorm:"serializer:json"` + + // PortRanges a list of port ranges. + PortRanges []RulePortRange `gorm:"serializer:json"` +} + +// Copy returns a copy of a policy rule +func (pm *PolicyRule) Copy() *PolicyRule { + rule := &PolicyRule{ + ID: pm.ID, + PolicyID: pm.PolicyID, + Name: pm.Name, + Description: pm.Description, + Enabled: pm.Enabled, + Action: pm.Action, + Destinations: make([]string, len(pm.Destinations)), + DestinationResource: pm.DestinationResource, + Sources: make([]string, len(pm.Sources)), + SourceResource: pm.SourceResource, + Bidirectional: pm.Bidirectional, + Protocol: pm.Protocol, + Ports: make([]string, len(pm.Ports)), + PortRanges: make([]RulePortRange, len(pm.PortRanges)), + } + copy(rule.Destinations, pm.Destinations) + copy(rule.Sources, pm.Sources) + copy(rule.Ports, pm.Ports) + copy(rule.PortRanges, pm.PortRanges) + return rule +} diff --git a/internal/modules/template/interface.go b/internal/modules/template/interface.go new file mode 100644 index 0000000..7261413 --- /dev/null +++ b/internal/modules/template/interface.go @@ -0,0 +1,8 @@ +//go:build ignore +// +build ignore + +package template + +type Manager interface { + // Add all exported methods that the manager should implement +} diff --git a/internal/modules/template/manager/api.go b/internal/modules/template/manager/api.go new file mode 100644 index 0000000..9c2f04f --- /dev/null +++ b/internal/modules/template/manager/api.go @@ -0,0 +1,20 @@ +//go:build ignore +// +build ignore + +package manager + +type handler struct { + manager template.Manager + permissionsManager permissions.Manager +} + +func newHandler(manager template.Manager, permissionsManager permissions.Manager) *handler { + return &handler{ + manager: manager, + permissionsManager: permissionsManager, + } +} + +func (h *handler) RegisterEndpoints(router *mux.Router) { + // Register the API endpoints for the module +} diff --git a/internal/modules/template/manager/manager.go b/internal/modules/template/manager/manager.go new file mode 100644 index 0000000..a6aeb1f --- /dev/null +++ b/internal/modules/template/manager/manager.go @@ -0,0 +1,28 @@ +//go:build ignore +// +build ignore + +package manager + +import ( + "github.com/gorilla/mux" + + "management/internal/modules/template" + "management/internal/shared/db" + appmetrics "management/internal/shared/metrics" + "management/internal/shared/permissions" +) + +type managerImpl struct { + repo Repository +} + +func NewManager(store *db.Store, router *mux.Router, metrics appmetrics.AppMetrics, permissionsManager permissions.Manager) template.Manager { + repo := newRepository(store) + m := &managerImpl{ + repo: repo, + } + + api := newHandler(m, permissionsManager) + api.RegisterEndpoints(router) + return m +} diff --git a/internal/modules/template/manager/metrics.go b/internal/modules/template/manager/metrics.go new file mode 100644 index 0000000..0bc2399 --- /dev/null +++ b/internal/modules/template/manager/metrics.go @@ -0,0 +1,27 @@ +//go:build ignore +// +build ignore + +package manager + +type metrics struct { + dbAccessDuration metric.Int64Histogram +} + +func newMetrics(meter metric.Meter) (*metrics, error) { + dbAccessDuration, err := meter.Int64Histogram( + "sync_request_duration_seconds", + metric.WithDescription("Duration of sync requests in seconds"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + return &metrics{ + dbAccessDuration: dbAccessDuration, + }, nil +} + +func (m *metrics) RecordDBAccessDuration(duration time.Duration) { + m.dbAccessDuration.Record(context.Background(), duration.Milliseconds(), metric.WithAttributes()) +} diff --git a/internal/modules/template/manager/repository.go b/internal/modules/template/manager/repository.go new file mode 100644 index 0000000..7495f8b --- /dev/null +++ b/internal/modules/template/manager/repository.go @@ -0,0 +1,4 @@ +//go:build ignore +// +build ignore + +package manager diff --git a/internal/modules/template/type.go b/internal/modules/template/type.go new file mode 100644 index 0000000..72f9c77 --- /dev/null +++ b/internal/modules/template/type.go @@ -0,0 +1,4 @@ +//go:build ignore +// +build ignore + +package template diff --git a/internal/modules/users/types/user.go b/internal/modules/users/types/user.go index 9abbffe..dcbd2f7 100644 --- a/internal/modules/users/types/user.go +++ b/internal/modules/users/types/user.go @@ -8,6 +8,7 @@ import ( "github.com/netbirdio/netbird/management/server/idp" "github.com/netbirdio/netbird/management/server/integration_reference" + "management/internal/modules/accounts/settings" "management/internal/modules/users/pats/types" ) @@ -129,7 +130,7 @@ func (u *User) IsRegularUser() bool { } // ToUserInfo converts a User object to a UserInfo object. -func (u *User) ToUserInfo(userData *idp.UserData, settings *Settings) (*UserInfo, error) { +func (u *User) ToUserInfo(userData *idp.UserData, settings *settings.Settings) (*UserInfo, error) { autoGroups := u.AutoGroups if autoGroups == nil { autoGroups = []string{} diff --git a/internal/server/boot.go b/internal/server/boot.go index de21449..d143e73 100644 --- a/internal/server/boot.go +++ b/internal/server/boot.go @@ -6,10 +6,16 @@ import ( "context" "net/http" + "github.com/gorilla/mux" + + "management/internal/controllers/network_map" + "management/internal/modules/peers" "management/internal/shared/activity" "management/internal/shared/activity/sqlite" "management/internal/shared/api/rest" "management/internal/shared/db" + "management/internal/shared/metrics" + "management/internal/shared/permissions" ) func (s *Server) Store() *db.Store { @@ -35,6 +41,22 @@ func (s *Server) HttpServer() *http.Server { }) } +func (s *Server) Metrics() *metrics.AppMetrics { + return Create(s, func() *metrics.AppMetrics { + appMetrics, err := metrics.NewAppMetrics() + if err != nil { + log.Fatalf("error while creating app metrics: %s", err) + } + return appMetrics + }) +} + +func (s *Server) Router() *mux.Router { + return Create(s, func() *mux.Router { + return mux.NewRouter() + }) +} + func (s *Server) EventStore() activity.Store { return Create(s, func() activity.Store { ctx := context.Background() @@ -45,3 +67,27 @@ func (s *Server) EventStore() activity.Store { return store }) } + +func (s *Server) NetworkMapController() *network_map.Controller { + return Create(s, func() *network_map.Controller { + store := s.Store() + metrics := s.Metrics() + return network_map.NewController(store, metrics) + }) +} + +func (s *Server) PermissionsManager() permissions.Manager { + return Create(s, func() permissions.Manager { + return permissions.NewManager() + }) +} + +func (s *Server) PeersManager() *peers.Manager { + return Create(s, func() *peers.Manager { + store := s.Store() + router := s.Router() + permissionsManager := s.PermissionsManager() + + return peers.NewManager(store, router, permissionsManager) + }) +} diff --git a/internal/shared/api/rest/middleware/logging_middleware.go b/internal/shared/api/rest/middleware/logging_middleware.go index c870d7c..a5447b3 100644 --- a/internal/shared/api/rest/middleware/logging_middleware.go +++ b/internal/shared/api/rest/middleware/logging_middleware.go @@ -1 +1,12 @@ package middleware + +import "net/http" + +// loggingMiddleware is an example that logs each incoming request. +// Replace with your logger of choice. +func LoggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // log.Printf("[%s] %s", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/shared/api/rest/middleware/recovery_middleware.go b/internal/shared/api/rest/middleware/recovery_middleware.go new file mode 100644 index 0000000..7184e6d --- /dev/null +++ b/internal/shared/api/rest/middleware/recovery_middleware.go @@ -0,0 +1,15 @@ +package middleware + +import "net/http" + +// RecoveryMiddleware recovers from panics and returns a 500 Internal Server Error. +func RecoveryMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + } + }() + next.ServeHTTP(w, r) + }) +} diff --git a/internal/shared/api/rest/router.go b/internal/shared/api/rest/router.go index 931328b..55b7d9d 100644 --- a/internal/shared/api/rest/router.go +++ b/internal/shared/api/rest/router.go @@ -1,9 +1,9 @@ package rest import ( - "net/http" - "github.com/gorilla/mux" + + "management/internal/shared/api/rest/middleware" ) // NewRouter creates and returns a mux.Router configured with default middleware @@ -11,33 +11,8 @@ import ( func NewRouter() *mux.Router { r := mux.NewRouter() - // Attach middlewares - r.Use(loggingMiddleware) - r.Use(recoveryMiddleware) - - // Example endpoint - // r.HandleFunc("/health", healthCheckHandler).Methods("GET") + r.Use(middleware.LoggingMiddleware) + r.Use(middleware.RecoveryMiddleware) return r } - -// loggingMiddleware is an example that logs each incoming request. -// Replace with your logger of choice. -func loggingMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // log.Printf("[%s] %s", r.Method, r.URL.Path) - next.ServeHTTP(w, r) - }) -} - -// recoveryMiddleware recovers from panics and returns a 500 Internal Server Error. -func recoveryMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer func() { - if rec := recover(); rec != nil { - http.Error(w, "Internal Server Error", http.StatusInternalServerError) - } - }() - next.ServeHTTP(w, r) - }) -} diff --git a/internal/shared/metrics/metrics.go b/internal/shared/metrics/metrics.go new file mode 100644 index 0000000..9f7d99c --- /dev/null +++ b/internal/shared/metrics/metrics.go @@ -0,0 +1,83 @@ +package metrics + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + + "github.com/gorilla/mux" + prometheus2 "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/exporters/prometheus" + metric2 "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk/metric" +) + +const defaultEndpoint = "/metrics" + +type AppMetrics struct { + meter metric2.Meter + listener net.Listener + ctx context.Context +} + +func NewAppMetrics() (*AppMetrics, error) { + exporter, err := prometheus.New() + if err != nil { + return nil, err + } + + provider := metric.NewMeterProvider(metric.WithReader(exporter)) + pkg := reflect.TypeOf(defaultEndpoint).PkgPath() + meter := provider.Meter(pkg) + + return &AppMetrics{ + meter: meter, + }, nil +} + +// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used. +// Exposes metrics in the Prometheus format https://prometheus.io/ +func (appMetrics *AppMetrics) Expose(ctx context.Context, port int, endpoint string) error { + if endpoint == "" { + endpoint = defaultEndpoint + } + rootRouter := mux.NewRouter() + rootRouter.Handle(endpoint, promhttp.HandlerFor( + prometheus2.DefaultGatherer, + promhttp.HandlerOpts{EnableOpenMetrics: true})) + listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port)) + if err != nil { + return err + } + appMetrics.listener = listener + go func() { + err := http.Serve(listener, rootRouter) + if err != nil { + return + } + }() + + log.WithContext(ctx).Infof("enabled application metrics and exposing on http://%s", listener.Addr().String()) + + return nil +} + +// Close stop application metrics HTTP handler and closes listener. +func (appMetrics *AppMetrics) Close() error { + if appMetrics.listener == nil { + return nil + } + return appMetrics.listener.Close() +} + +// func (appMetrics *AppMetrics) RegisterMetrics(fn func(meter metric2.Meter) error) error { +// return fn(appMetrics.meter) +// } + +func RegisterMetrics[T any](app *AppMetrics, fn func(metric2.Meter) (T, error)) (T, error) { + return fn(app.meter) +} diff --git a/pkg/logging/init.go b/pkg/logging/init.go index 349d266..b5aaa21 100644 --- a/pkg/logging/init.go +++ b/pkg/logging/init.go @@ -3,7 +3,7 @@ package logging import ( "fmt" "io" - "log" + "log/syslog" "os" "path/filepath" "runtime" @@ -13,11 +13,14 @@ import ( "sync" "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" "github.com/spf13/viper" "google.golang.org/grpc/grpclog" "gopkg.in/natefinch/lumberjack.v2" ) +const defaultLogSize = 5 + // global map of package paths to *logrus.Logger var ( mu sync.RWMutex @@ -162,7 +165,7 @@ func InitLog(logLevel string, logPath string) error { } logrus.SetOutput(io.Writer(lumberjackLogger)) } else if logPath == "syslog" { - AddSyslogHook() + addSyslogHook() } //nolint:gocritic @@ -210,3 +213,12 @@ func getLogMaxSize() int { } return defaultLogSize } + +func addSyslogHook() { + hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO, "") + + if err != nil { + logrus.Errorf("Failed creating syslog hook: %s", err) + } + logrus.AddHook(hook) +}