Files
laptops-kernel/include/net/fib_rules.h
Kuniyuki IwashimaandPaolo Abeni eef9bddc33 net: fib_rules: Only hold RTNL for the first IPv4 RTM_NEWRULE.
Now, RTM_DELRULE no longer needs RTNL, and the only RTNL dependant
in RTM_NEWRULE is fib_unmerge(), which is called for the first
IPv4 rule.

Let's add fib_rules_ops.need_rtnl() and hold RTNL only for the
first IPv4 rule.

Tested:
The script below creates 1K rules in parallel in 4K netns, and
it got 20x/30x faster for IPv4/IPv6.

  #!/bin/bash
  N=4096
  F=rules.txt

  for i in $(seq $N); do ip netns add ns-$i; done
  printf 'rule add from all table %d\n' {1..1024} > $F

  for v in 4 6; do
  	echo "=== IPv${v} ==="
  	time { for i in $(seq $N); do nsenter \
  	--net=/var/run/netns/ns-$i ip -$v -batch $F & done; wait; }
  done

  for i in $(seq $N); do ip netns del ns-$i; done
  rm -f $F

Without this series:

  # ./test.sh
  === IPv4 ===

  real	0m22.752s
  user	0m7.834s
  sys	92m46.721s
  === IPv6 ===

  real	0m35.181s
  user	0m8.635s
  sys	142m30.479s

With this series:

  # ./test.sh
  === IPv4 ===

  real	0m0.918s
  user	0m5.675s
  sys	2m7.024s
  === IPv6 ===

  real	0m1.214s
  user	0m7.917s
  sys	4m19.489s

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20260629181226.1929658-10-kuniyu@google.com
Reviewed-by: Ido Schimmel <idosch@nvidia.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2026-07-01 18:42:34 +02:00

233 lines
6.2 KiB
C

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_FIB_RULES_H
#define __NET_FIB_RULES_H
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/netdevice.h>
#include <linux/fib_rules.h>
#include <linux/refcount.h>
#include <net/flow.h>
#include <net/rtnetlink.h>
#include <net/fib_notifier.h>
#include <linux/indirect_call_wrapper.h>
struct fib_kuid_range {
kuid_t start;
kuid_t end;
};
struct fib_rule {
struct list_head list;
int iifindex;
int oifindex;
u32 mark;
u32 mark_mask;
u32 flags;
u32 table;
u8 action;
u8 l3mdev;
u8 proto;
u8 ip_proto;
u32 target;
__be64 tun_id;
struct fib_rule __rcu *ctarget;
struct net *fr_net;
refcount_t refcnt;
u32 pref;
int suppress_ifgroup;
int suppress_prefixlen;
char iifname[IFNAMSIZ];
char oifname[IFNAMSIZ];
struct fib_kuid_range uid_range;
struct fib_rule_port_range sport_range;
struct fib_rule_port_range dport_range;
u16 sport_mask;
u16 dport_mask;
u8 iif_is_l3_master;
u8 oif_is_l3_master;
struct rcu_head rcu;
};
struct fib_lookup_arg {
void *lookup_ptr;
const void *lookup_data;
void *result;
struct fib_rule *rule;
u32 table;
int flags;
#define FIB_LOOKUP_NOREF 1
#define FIB_LOOKUP_IGNORE_LINKSTATE 2
};
struct fib_rules_ops {
int family;
struct list_head list;
int rule_size;
int addr_size;
int unresolved_rules;
int nr_goto_rules;
unsigned int fib_rules_seq;
int (*action)(struct fib_rule *,
struct flowi *, int,
struct fib_lookup_arg *);
bool (*suppress)(struct fib_rule *, int,
struct fib_lookup_arg *);
int (*match)(struct fib_rule *,
struct flowi *, int);
int (*configure)(struct fib_rule *,
struct sk_buff *,
struct fib_rule_hdr *,
struct nlattr **,
struct netlink_ext_ack *);
void (*delete)(struct fib_rule *);
int (*compare)(struct fib_rule *,
struct fib_rule_hdr *,
struct nlattr **);
int (*fill)(struct fib_rule *, struct sk_buff *,
struct fib_rule_hdr *);
size_t (*nlmsg_payload)(struct fib_rule *);
/* Called after modifications to the rules set, must flush
* the route cache if one exists. */
void (*flush_cache)(struct fib_rules_ops *ops);
bool (*need_rtnl)(struct net *net);
int nlgroup;
struct list_head rules_list;
struct module *owner;
struct net *fro_net;
struct mutex lock;
struct rcu_head rcu;
};
struct fib_rule_notifier_info {
struct fib_notifier_info info; /* must be first */
struct fib_rule *rule;
};
static inline void fib_rule_get(struct fib_rule *rule)
{
refcount_inc(&rule->refcnt);
}
static inline bool fib_rule_get_safe(struct fib_rule *rule)
{
return refcount_inc_not_zero(&rule->refcnt);
}
static inline void fib_rule_put(struct fib_rule *rule)
{
if (refcount_dec_and_test(&rule->refcnt))
kfree_rcu(rule, rcu);
}
#ifdef CONFIG_NET_L3_MASTER_DEV
static inline u32 fib_rule_get_table(struct fib_rule *rule,
struct fib_lookup_arg *arg)
{
return rule->l3mdev ? arg->table : rule->table;
}
#else
static inline u32 fib_rule_get_table(struct fib_rule *rule,
struct fib_lookup_arg *arg)
{
return rule->table;
}
#endif
static inline u32 frh_get_table(struct fib_rule_hdr *frh, struct nlattr **nla)
{
if (nla[FRA_TABLE])
return nla_get_u32(nla[FRA_TABLE]);
return frh->table;
}
static inline bool fib_rule_port_range_set(const struct fib_rule_port_range *range)
{
return range->start != 0 && range->end != 0;
}
static inline bool fib_rule_port_inrange(const struct fib_rule_port_range *a,
__be16 port)
{
return ntohs(port) >= a->start &&
ntohs(port) <= a->end;
}
static inline bool fib_rule_port_match(const struct fib_rule_port_range *range,
u16 port_mask, __be16 port)
{
if ((range->start ^ ntohs(port)) & port_mask)
return false;
if (!port_mask && fib_rule_port_range_set(range) &&
!fib_rule_port_inrange(range, port))
return false;
return true;
}
static inline bool fib_rule_port_range_valid(const struct fib_rule_port_range *a)
{
return a->start != 0 && a->end != 0 && a->end < 0xffff &&
a->start <= a->end;
}
static inline bool fib_rule_port_range_compare(struct fib_rule_port_range *a,
struct fib_rule_port_range *b)
{
return a->start == b->start &&
a->end == b->end;
}
static inline bool
fib_rule_port_is_range(const struct fib_rule_port_range *range)
{
return range->start != range->end;
}
static inline bool fib_rule_requires_fldissect(struct fib_rule *rule)
{
return rule->iifindex != LOOPBACK_IFINDEX && (rule->ip_proto ||
fib_rule_port_range_set(&rule->sport_range) ||
fib_rule_port_range_set(&rule->dport_range));
}
struct fib_rules_ops *fib_rules_register(const struct fib_rules_ops *,
struct net *);
void fib_rules_unregister(struct fib_rules_ops *);
int fib_rules_lookup(struct fib_rules_ops *, struct flowi *, int flags,
struct fib_lookup_arg *);
int fib_default_rule_add(struct fib_rules_ops *, u32 pref, u32 table);
bool fib_rule_matchall(const struct fib_rule *rule);
int fib_rules_dump(struct net *net, struct notifier_block *nb, int family,
struct netlink_ext_ack *extack);
unsigned int fib_rules_seq_read(const struct net *net, int family);
int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack, bool rtnl_held);
int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack, bool rtnl_held);
INDIRECT_CALLABLE_DECLARE(int fib6_rule_match(struct fib_rule *rule,
struct flowi *fl, int flags));
INDIRECT_CALLABLE_DECLARE(int fib4_rule_match(struct fib_rule *rule,
struct flowi *fl, int flags));
INDIRECT_CALLABLE_DECLARE(int fib6_rule_action(struct fib_rule *rule,
struct flowi *flp, int flags,
struct fib_lookup_arg *arg));
INDIRECT_CALLABLE_DECLARE(int fib4_rule_action(struct fib_rule *rule,
struct flowi *flp, int flags,
struct fib_lookup_arg *arg));
INDIRECT_CALLABLE_DECLARE(bool fib6_rule_suppress(struct fib_rule *rule,
int flags,
struct fib_lookup_arg *arg));
INDIRECT_CALLABLE_DECLARE(bool fib4_rule_suppress(struct fib_rule *rule,
int flags,
struct fib_lookup_arg *arg));
#endif