mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next
Pull bpf updates from Alexei Starovoitov:
"For this merge window we're splitting BPF pull request into three for
higher visibility: main changes, res_spin_lock, try_alloc_pages.
These are the main BPF changes:
- Add DFA-based live registers analysis to improve verification of
programs with loops (Eduard Zingerman)
- Introduce load_acquire and store_release BPF instructions and add
x86, arm64 JIT support (Peilin Ye)
- Fix loop detection logic in the verifier (Eduard Zingerman)
- Drop unnecesary lock in bpf_map_inc_not_zero() (Eric Dumazet)
- Add kfunc for populating cpumask bits (Emil Tsalapatis)
- Convert various shell based tests to selftests/bpf/test_progs
format (Bastien Curutchet)
- Allow passing referenced kptrs into struct_ops callbacks (Amery
Hung)
- Add a flag to LSM bpf hook to facilitate bpf program signing
(Blaise Boscaccy)
- Track arena arguments in kfuncs (Ihor Solodrai)
- Add copy_remote_vm_str() helper for reading strings from remote VM
and bpf_copy_from_user_task_str() kfunc (Jordan Rome)
- Add support for timed may_goto instruction (Kumar Kartikeya
Dwivedi)
- Allow bpf_get_netns_cookie() int cgroup_skb programs (Mahe Tardy)
- Reduce bpf_cgrp_storage_busy false positives when accessing cgroup
local storage (Martin KaFai Lau)
- Introduce bpf_dynptr_copy() kfunc (Mykyta Yatsenko)
- Allow retrieving BTF data with BTF token (Mykyta Yatsenko)
- Add BPF kfuncs to set and get xattrs with 'security.bpf.' prefix
(Song Liu)
- Reject attaching programs to noreturn functions (Yafang Shao)
- Introduce pre-order traversal of cgroup bpf programs (Yonghong
Song)"
* tag 'bpf-next-6.15' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (186 commits)
selftests/bpf: Add selftests for load-acquire/store-release when register number is invalid
bpf: Fix out-of-bounds read in check_atomic_load/store()
libbpf: Add namespace for errstr making it libbpf_errstr
bpf: Add struct_ops context information to struct bpf_prog_aux
selftests/bpf: Sanitize pointer prior fclose()
selftests/bpf: Migrate test_xdp_vlan.sh into test_progs
selftests/bpf: test_xdp_vlan: Rename BPF sections
bpf: clarify a misleading verifier error message
selftests/bpf: Add selftest for attaching fexit to __noreturn functions
bpf: Reject attaching fexit/fmod_ret to __noreturn functions
bpf: Only fails the busy counter check in bpf_cgrp_storage_get if it creates storage
bpf: Make perf_event_read_output accessible in all program types.
bpftool: Using the right format specifiers
bpftool: Add -Wformat-signedness flag to detect format errors
selftests/bpf: Test freplace from user namespace
libbpf: Pass BPF token from find_prog_btf_id to BPF_BTF_GET_FD_BY_ID
bpf: Return prog btf_id without capable check
bpf: BPF token support for BPF_BTF_GET_FD_BY_ID
bpf, x86: Fix objtool warning for timed may_goto
bpf: Check map->record at the beginning of check_and_free_fields()
...
This commit is contained in:
Linus Torvalds
@@ -86,7 +86,7 @@ following steps:
|
||||
The following are a few examples of selftest BPF iterator programs:
|
||||
|
||||
* `bpf_iter_tcp4.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c>`_
|
||||
* `bpf_iter_task_vma.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c>`_
|
||||
* `bpf_iter_task_vmas.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c>`_
|
||||
* `bpf_iter_task_file.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c>`_
|
||||
|
||||
Let us look at ``bpf_iter_task_file.c``, which runs in kernel space:
|
||||
|
||||
@@ -102,7 +102,8 @@ Each type contains the following common data::
|
||||
* bits 24-28: kind (e.g. int, ptr, array...etc)
|
||||
* bits 29-30: unused
|
||||
* bit 31: kind_flag, currently used by
|
||||
* struct, union, fwd, enum and enum64.
|
||||
* struct, union, enum, fwd, enum64,
|
||||
* decl_tag and type_tag
|
||||
*/
|
||||
__u32 info;
|
||||
/* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
|
||||
@@ -478,7 +479,7 @@ No additional type data follow ``btf_type``.
|
||||
|
||||
``struct btf_type`` encoding requirement:
|
||||
* ``name_off``: offset to a non-empty string
|
||||
* ``info.kind_flag``: 0
|
||||
* ``info.kind_flag``: 0 or 1
|
||||
* ``info.kind``: BTF_KIND_DECL_TAG
|
||||
* ``info.vlen``: 0
|
||||
* ``type``: ``struct``, ``union``, ``func``, ``var`` or ``typedef``
|
||||
@@ -489,7 +490,6 @@ No additional type data follow ``btf_type``.
|
||||
__u32 component_idx;
|
||||
};
|
||||
|
||||
The ``name_off`` encodes btf_decl_tag attribute string.
|
||||
The ``type`` should be ``struct``, ``union``, ``func``, ``var`` or ``typedef``.
|
||||
For ``var`` or ``typedef`` type, ``btf_decl_tag.component_idx`` must be ``-1``.
|
||||
For the other three types, if the btf_decl_tag attribute is
|
||||
@@ -499,12 +499,21 @@ the attribute is applied to a ``struct``/``union`` member or
|
||||
a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a
|
||||
valid index (starting from 0) pointing to a member or an argument.
|
||||
|
||||
If ``info.kind_flag`` is 0, then this is a normal decl tag, and the
|
||||
``name_off`` encodes btf_decl_tag attribute string.
|
||||
|
||||
If ``info.kind_flag`` is 1, then the decl tag represents an arbitrary
|
||||
__attribute__. In this case, ``name_off`` encodes a string
|
||||
representing the attribute-list of the attribute specifier. For
|
||||
example, for an ``__attribute__((aligned(4)))`` the string's contents
|
||||
is ``aligned(4)``.
|
||||
|
||||
2.2.18 BTF_KIND_TYPE_TAG
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
``struct btf_type`` encoding requirement:
|
||||
* ``name_off``: offset to a non-empty string
|
||||
* ``info.kind_flag``: 0
|
||||
* ``info.kind_flag``: 0 or 1
|
||||
* ``info.kind``: BTF_KIND_TYPE_TAG
|
||||
* ``info.vlen``: 0
|
||||
* ``type``: the type with ``btf_type_tag`` attribute
|
||||
@@ -522,6 +531,14 @@ type_tag, then zero or more const/volatile/restrict/typedef
|
||||
and finally the base type. The base type is one of
|
||||
int, ptr, array, struct, union, enum, func_proto and float types.
|
||||
|
||||
Similarly to decl tags, if the ``info.kind_flag`` is 0, then this is a
|
||||
normal type tag, and the ``name_off`` encodes btf_type_tag attribute
|
||||
string.
|
||||
|
||||
If ``info.kind_flag`` is 1, then the type tag represents an arbitrary
|
||||
__attribute__, and the ``name_off`` encodes a string representing the
|
||||
attribute-list of the attribute specifier.
|
||||
|
||||
2.2.19 BTF_KIND_ENUM64
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@@ -324,34 +324,42 @@ register.
|
||||
|
||||
.. table:: Arithmetic instructions
|
||||
|
||||
===== ===== ======= ==========================================================
|
||||
===== ===== ======= ===================================================================================
|
||||
name code offset description
|
||||
===== ===== ======= ==========================================================
|
||||
===== ===== ======= ===================================================================================
|
||||
ADD 0x0 0 dst += src
|
||||
SUB 0x1 0 dst -= src
|
||||
MUL 0x2 0 dst \*= src
|
||||
DIV 0x3 0 dst = (src != 0) ? (dst / src) : 0
|
||||
SDIV 0x3 1 dst = (src != 0) ? (dst s/ src) : 0
|
||||
SDIV 0x3 1 dst = (src == 0) ? 0 : ((src == -1 && dst == LLONG_MIN) ? LLONG_MIN : (dst s/ src))
|
||||
OR 0x4 0 dst \|= src
|
||||
AND 0x5 0 dst &= src
|
||||
LSH 0x6 0 dst <<= (src & mask)
|
||||
RSH 0x7 0 dst >>= (src & mask)
|
||||
NEG 0x8 0 dst = -dst
|
||||
MOD 0x9 0 dst = (src != 0) ? (dst % src) : dst
|
||||
SMOD 0x9 1 dst = (src != 0) ? (dst s% src) : dst
|
||||
SMOD 0x9 1 dst = (src == 0) ? dst : ((src == -1 && dst == LLONG_MIN) ? 0: (dst s% src))
|
||||
XOR 0xa 0 dst ^= src
|
||||
MOV 0xb 0 dst = src
|
||||
MOVSX 0xb 8/16/32 dst = (s8,s16,s32)src
|
||||
ARSH 0xc 0 :term:`sign extending<Sign Extend>` dst >>= (src & mask)
|
||||
END 0xd 0 byte swap operations (see `Byte swap instructions`_ below)
|
||||
===== ===== ======= ==========================================================
|
||||
===== ===== ======= ===================================================================================
|
||||
|
||||
Underflow and overflow are allowed during arithmetic operations, meaning
|
||||
the 64-bit or 32-bit value will wrap. If BPF program execution would
|
||||
result in division by zero, the destination register is instead set to zero.
|
||||
Otherwise, for ``ALU64``, if execution would result in ``LLONG_MIN``
|
||||
dividing -1, the desination register is instead set to ``LLONG_MIN``. For
|
||||
``ALU``, if execution would result in ``INT_MIN`` dividing -1, the
|
||||
desination register is instead set to ``INT_MIN``.
|
||||
|
||||
If execution would result in modulo by zero, for ``ALU64`` the value of
|
||||
the destination register is unchanged whereas for ``ALU`` the upper
|
||||
32 bits of the destination register are zeroed.
|
||||
32 bits of the destination register are zeroed. Otherwise, for ``ALU64``,
|
||||
if execution would resuslt in ``LLONG_MIN`` modulo -1, the destination
|
||||
register is instead set to 0. For ``ALU``, if execution would result in
|
||||
``INT_MIN`` modulo -1, the destination register is instead set to 0.
|
||||
|
||||
``{ADD, X, ALU}``, where 'code' = ``ADD``, 'source' = ``X``, and 'class' = ``ALU``, means::
|
||||
|
||||
|
||||
@@ -188,8 +188,10 @@ enum aarch64_insn_ldst_type {
|
||||
AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX,
|
||||
AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
|
||||
AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
|
||||
AARCH64_INSN_LDST_LOAD_ACQ,
|
||||
AARCH64_INSN_LDST_LOAD_EX,
|
||||
AARCH64_INSN_LDST_LOAD_ACQ_EX,
|
||||
AARCH64_INSN_LDST_STORE_REL,
|
||||
AARCH64_INSN_LDST_STORE_EX,
|
||||
AARCH64_INSN_LDST_STORE_REL_EX,
|
||||
AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET,
|
||||
@@ -351,8 +353,10 @@ __AARCH64_INSN_FUNCS(ldr_imm, 0x3FC00000, 0x39400000)
|
||||
__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000)
|
||||
__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
|
||||
__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000)
|
||||
__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000)
|
||||
__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000)
|
||||
__AARCH64_INSN_FUNCS(load_acq, 0x3FDFFC00, 0x08DFFC00)
|
||||
__AARCH64_INSN_FUNCS(store_rel, 0x3FDFFC00, 0x089FFC00)
|
||||
__AARCH64_INSN_FUNCS(load_ex, 0x3FC00000, 0x08400000)
|
||||
__AARCH64_INSN_FUNCS(store_ex, 0x3FC00000, 0x08000000)
|
||||
__AARCH64_INSN_FUNCS(mops, 0x3B200C00, 0x19000400)
|
||||
__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000)
|
||||
__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000)
|
||||
@@ -602,6 +606,10 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
|
||||
int offset,
|
||||
enum aarch64_insn_variant variant,
|
||||
enum aarch64_insn_ldst_type type);
|
||||
u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
|
||||
enum aarch64_insn_register base,
|
||||
enum aarch64_insn_size_type size,
|
||||
enum aarch64_insn_ldst_type type);
|
||||
u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
|
||||
enum aarch64_insn_register base,
|
||||
enum aarch64_insn_register state,
|
||||
|
||||
@@ -540,6 +540,35 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
|
||||
offset >> shift);
|
||||
}
|
||||
|
||||
u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
|
||||
enum aarch64_insn_register base,
|
||||
enum aarch64_insn_size_type size,
|
||||
enum aarch64_insn_ldst_type type)
|
||||
{
|
||||
u32 insn;
|
||||
|
||||
switch (type) {
|
||||
case AARCH64_INSN_LDST_LOAD_ACQ:
|
||||
insn = aarch64_insn_get_load_acq_value();
|
||||
break;
|
||||
case AARCH64_INSN_LDST_STORE_REL:
|
||||
insn = aarch64_insn_get_store_rel_value();
|
||||
break;
|
||||
default:
|
||||
pr_err("%s: unknown load-acquire/store-release encoding %d\n",
|
||||
__func__, type);
|
||||
return AARCH64_BREAK_FAULT;
|
||||
}
|
||||
|
||||
insn = aarch64_insn_encode_ldst_size(size, insn);
|
||||
|
||||
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
|
||||
reg);
|
||||
|
||||
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
|
||||
base);
|
||||
}
|
||||
|
||||
u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
|
||||
enum aarch64_insn_register base,
|
||||
enum aarch64_insn_register state,
|
||||
|
||||
@@ -119,6 +119,26 @@
|
||||
aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
|
||||
AARCH64_INSN_LDST_STORE_REL_EX)
|
||||
|
||||
/* Load-acquire & store-release */
|
||||
#define A64_LDAR(Rt, Rn, size) \
|
||||
aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
|
||||
AARCH64_INSN_LDST_LOAD_ACQ)
|
||||
#define A64_STLR(Rt, Rn, size) \
|
||||
aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
|
||||
AARCH64_INSN_LDST_STORE_REL)
|
||||
|
||||
/* Rt = [Rn] (load acquire) */
|
||||
#define A64_LDARB(Wt, Xn) A64_LDAR(Wt, Xn, 8)
|
||||
#define A64_LDARH(Wt, Xn) A64_LDAR(Wt, Xn, 16)
|
||||
#define A64_LDAR32(Wt, Xn) A64_LDAR(Wt, Xn, 32)
|
||||
#define A64_LDAR64(Xt, Xn) A64_LDAR(Xt, Xn, 64)
|
||||
|
||||
/* [Rn] = Rt (store release) */
|
||||
#define A64_STLRB(Wt, Xn) A64_STLR(Wt, Xn, 8)
|
||||
#define A64_STLRH(Wt, Xn) A64_STLR(Wt, Xn, 16)
|
||||
#define A64_STLR32(Wt, Xn) A64_STLR(Wt, Xn, 32)
|
||||
#define A64_STLR64(Xt, Xn) A64_STLR(Xt, Xn, 64)
|
||||
|
||||
/*
|
||||
* LSE atomics
|
||||
*
|
||||
|
||||
@@ -272,7 +272,7 @@ static inline void emit_a64_add_i(const bool is64, const int dst, const int src,
|
||||
{
|
||||
if (is_addsub_imm(imm)) {
|
||||
emit(A64_ADD_I(is64, dst, src, imm), ctx);
|
||||
} else if (is_addsub_imm(-imm)) {
|
||||
} else if (is_addsub_imm(-(u32)imm)) {
|
||||
emit(A64_SUB_I(is64, dst, src, -imm), ctx);
|
||||
} else {
|
||||
emit_a64_mov_i(is64, tmp, imm, ctx);
|
||||
@@ -647,6 +647,81 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
|
||||
{
|
||||
const s32 imm = insn->imm;
|
||||
const s16 off = insn->off;
|
||||
const u8 code = insn->code;
|
||||
const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
|
||||
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
|
||||
const u8 dst = bpf2a64[insn->dst_reg];
|
||||
const u8 src = bpf2a64[insn->src_reg];
|
||||
const u8 tmp = bpf2a64[TMP_REG_1];
|
||||
u8 reg;
|
||||
|
||||
switch (imm) {
|
||||
case BPF_LOAD_ACQ:
|
||||
reg = src;
|
||||
break;
|
||||
case BPF_STORE_REL:
|
||||
reg = dst;
|
||||
break;
|
||||
default:
|
||||
pr_err_once("unknown atomic load/store op code %02x\n", imm);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (off) {
|
||||
emit_a64_add_i(1, tmp, reg, tmp, off, ctx);
|
||||
reg = tmp;
|
||||
}
|
||||
if (arena) {
|
||||
emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
|
||||
reg = tmp;
|
||||
}
|
||||
|
||||
switch (imm) {
|
||||
case BPF_LOAD_ACQ:
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_B:
|
||||
emit(A64_LDARB(dst, reg), ctx);
|
||||
break;
|
||||
case BPF_H:
|
||||
emit(A64_LDARH(dst, reg), ctx);
|
||||
break;
|
||||
case BPF_W:
|
||||
emit(A64_LDAR32(dst, reg), ctx);
|
||||
break;
|
||||
case BPF_DW:
|
||||
emit(A64_LDAR64(dst, reg), ctx);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case BPF_STORE_REL:
|
||||
switch (BPF_SIZE(code)) {
|
||||
case BPF_B:
|
||||
emit(A64_STLRB(src, reg), ctx);
|
||||
break;
|
||||
case BPF_H:
|
||||
emit(A64_STLRH(src, reg), ctx);
|
||||
break;
|
||||
case BPF_W:
|
||||
emit(A64_STLR32(src, reg), ctx);
|
||||
break;
|
||||
case BPF_DW:
|
||||
emit(A64_STLR64(src, reg), ctx);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
pr_err_once("unexpected atomic load/store op code %02x\n",
|
||||
imm);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_LSE_ATOMICS
|
||||
static int emit_lse_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
|
||||
{
|
||||
@@ -1159,7 +1234,7 @@ emit_bswap_uxt:
|
||||
case BPF_ALU64 | BPF_SUB | BPF_K:
|
||||
if (is_addsub_imm(imm)) {
|
||||
emit(A64_SUB_I(is64, dst, dst, imm), ctx);
|
||||
} else if (is_addsub_imm(-imm)) {
|
||||
} else if (is_addsub_imm(-(u32)imm)) {
|
||||
emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
|
||||
} else {
|
||||
emit_a64_mov_i(is64, tmp, imm, ctx);
|
||||
@@ -1330,7 +1405,7 @@ emit_cond_jmp:
|
||||
case BPF_JMP32 | BPF_JSLE | BPF_K:
|
||||
if (is_addsub_imm(imm)) {
|
||||
emit(A64_CMP_I(is64, dst, imm), ctx);
|
||||
} else if (is_addsub_imm(-imm)) {
|
||||
} else if (is_addsub_imm(-(u32)imm)) {
|
||||
emit(A64_CMN_I(is64, dst, -imm), ctx);
|
||||
} else {
|
||||
emit_a64_mov_i(is64, tmp, imm, ctx);
|
||||
@@ -1641,11 +1716,17 @@ emit_cond_jmp:
|
||||
return ret;
|
||||
break;
|
||||
|
||||
case BPF_STX | BPF_ATOMIC | BPF_B:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_H:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_DW:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
|
||||
if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
|
||||
if (bpf_atomic_is_load_store(insn))
|
||||
ret = emit_atomic_ld_st(insn, ctx);
|
||||
else if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
|
||||
ret = emit_lse_atomic(insn, ctx);
|
||||
else
|
||||
ret = emit_ll_sc_atomic(insn, ctx);
|
||||
@@ -2669,7 +2750,8 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
|
||||
switch (insn->code) {
|
||||
case BPF_STX | BPF_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_DW:
|
||||
if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
|
||||
if (!bpf_atomic_is_load_store(insn) &&
|
||||
!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void)
|
||||
|
||||
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
|
||||
{
|
||||
/*
|
||||
* Currently the verifier uses this function only to check which
|
||||
* atomic stores to arena are supported, and they all are.
|
||||
*/
|
||||
if (!in_arena)
|
||||
return true;
|
||||
switch (insn->code) {
|
||||
case BPF_STX | BPF_ATOMIC | BPF_B:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_H:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_DW:
|
||||
if (bpf_atomic_is_load_store(insn))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,5 +6,5 @@
|
||||
ifeq ($(CONFIG_X86_32),y)
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
|
||||
else
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
|
||||
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
|
||||
endif
|
||||
|
||||
+87
-13
@@ -1250,8 +1250,8 @@ static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)
|
||||
emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
|
||||
}
|
||||
|
||||
static int emit_atomic(u8 **pprog, u8 atomic_op,
|
||||
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
|
||||
static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,
|
||||
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
|
||||
{
|
||||
u8 *prog = *pprog;
|
||||
|
||||
@@ -1291,8 +1291,9 @@ static int emit_atomic(u8 **pprog, u8 atomic_op,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
|
||||
u32 dst_reg, u32 src_reg, u32 index_reg, int off)
|
||||
static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
|
||||
u32 dst_reg, u32 src_reg, u32 index_reg,
|
||||
int off)
|
||||
{
|
||||
u8 *prog = *pprog;
|
||||
|
||||
@@ -1305,7 +1306,7 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
|
||||
EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
|
||||
break;
|
||||
default:
|
||||
pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");
|
||||
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
@@ -1339,6 +1340,49 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
|
||||
u32 src_reg, s16 off, u8 bpf_size)
|
||||
{
|
||||
switch (atomic_op) {
|
||||
case BPF_LOAD_ACQ:
|
||||
/* dst_reg = smp_load_acquire(src_reg + off16) */
|
||||
emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
|
||||
break;
|
||||
case BPF_STORE_REL:
|
||||
/* smp_store_release(dst_reg + off16, src_reg) */
|
||||
emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
|
||||
break;
|
||||
default:
|
||||
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
|
||||
atomic_op);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,
|
||||
u32 dst_reg, u32 src_reg, u32 index_reg,
|
||||
int off)
|
||||
{
|
||||
switch (atomic_op) {
|
||||
case BPF_LOAD_ACQ:
|
||||
/* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */
|
||||
emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);
|
||||
break;
|
||||
case BPF_STORE_REL:
|
||||
/* smp_store_release(dst_reg + idx_reg + off16, src_reg) */
|
||||
emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);
|
||||
break;
|
||||
default:
|
||||
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
|
||||
atomic_op);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define DONT_CLEAR 1
|
||||
|
||||
bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)
|
||||
@@ -2121,6 +2165,13 @@ populate_extable:
|
||||
}
|
||||
break;
|
||||
|
||||
case BPF_STX | BPF_ATOMIC | BPF_B:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_H:
|
||||
if (!bpf_atomic_is_load_store(insn)) {
|
||||
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
|
||||
return -EFAULT;
|
||||
}
|
||||
fallthrough;
|
||||
case BPF_STX | BPF_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_ATOMIC | BPF_DW:
|
||||
if (insn->imm == (BPF_AND | BPF_FETCH) ||
|
||||
@@ -2156,10 +2207,10 @@ populate_extable:
|
||||
EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
|
||||
add_2reg(0xC0, AUX_REG, real_src_reg));
|
||||
/* Attempt to swap in new value */
|
||||
err = emit_atomic(&prog, BPF_CMPXCHG,
|
||||
real_dst_reg, AUX_REG,
|
||||
insn->off,
|
||||
BPF_SIZE(insn->code));
|
||||
err = emit_atomic_rmw(&prog, BPF_CMPXCHG,
|
||||
real_dst_reg, AUX_REG,
|
||||
insn->off,
|
||||
BPF_SIZE(insn->code));
|
||||
if (WARN_ON(err))
|
||||
return err;
|
||||
/*
|
||||
@@ -2174,17 +2225,35 @@ populate_extable:
|
||||
break;
|
||||
}
|
||||
|
||||
err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
|
||||
insn->off, BPF_SIZE(insn->code));
|
||||
if (bpf_atomic_is_load_store(insn))
|
||||
err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
|
||||
insn->off, BPF_SIZE(insn->code));
|
||||
else
|
||||
err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
|
||||
insn->off, BPF_SIZE(insn->code));
|
||||
if (err)
|
||||
return err;
|
||||
break;
|
||||
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
|
||||
if (!bpf_atomic_is_load_store(insn)) {
|
||||
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
|
||||
return -EFAULT;
|
||||
}
|
||||
fallthrough;
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
|
||||
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
|
||||
start_of_ldx = prog;
|
||||
err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),
|
||||
dst_reg, src_reg, X86_REG_R12, insn->off);
|
||||
|
||||
if (bpf_atomic_is_load_store(insn))
|
||||
err = emit_atomic_ld_st_index(&prog, insn->imm,
|
||||
BPF_SIZE(insn->code), dst_reg,
|
||||
src_reg, X86_REG_R12, insn->off);
|
||||
else
|
||||
err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code),
|
||||
dst_reg, src_reg, X86_REG_R12,
|
||||
insn->off);
|
||||
if (err)
|
||||
return err;
|
||||
goto populate_extable;
|
||||
@@ -3801,3 +3870,8 @@ u64 bpf_arch_uaddress_limit(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool bpf_jit_supports_timed_may_goto(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <linux/export.h>
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/nospec-branch.h>
|
||||
|
||||
.code64
|
||||
.section .text, "ax"
|
||||
|
||||
SYM_FUNC_START(arch_bpf_timed_may_goto)
|
||||
ANNOTATE_NOENDBR
|
||||
|
||||
/*
|
||||
* r10 passes us stack depth, load the pointer to count and timestamp
|
||||
* into r10 by adding it to BPF frame pointer.
|
||||
*/
|
||||
leaq (%rbp, %r10, 1), %r10
|
||||
|
||||
/* Setup frame. */
|
||||
pushq %rbp
|
||||
movq %rsp, %rbp
|
||||
|
||||
/* Save r0-r5. */
|
||||
pushq %rax
|
||||
pushq %rdi
|
||||
pushq %rsi
|
||||
pushq %rdx
|
||||
pushq %rcx
|
||||
pushq %r8
|
||||
|
||||
/*
|
||||
* r10 has the pointer to count and timestamp, pass it as first
|
||||
* argument.
|
||||
*/
|
||||
movq %r10, %rdi
|
||||
|
||||
/* Emit call depth accounting for call below. */
|
||||
CALL_DEPTH_ACCOUNT
|
||||
call bpf_check_timed_may_goto
|
||||
|
||||
/* BPF_REG_AX=r10 will be stored into count, so move return value to it. */
|
||||
movq %rax, %r10
|
||||
|
||||
/* Restore r5-r0. */
|
||||
popq %r8
|
||||
popq %rcx
|
||||
popq %rdx
|
||||
popq %rsi
|
||||
popq %rdi
|
||||
popq %rax
|
||||
|
||||
leave
|
||||
RET
|
||||
SYM_FUNC_END(arch_bpf_timed_may_goto)
|
||||
+214
-11
@@ -2,10 +2,12 @@
|
||||
/* Copyright (c) 2024 Google LLC. */
|
||||
|
||||
#include <linux/bpf.h>
|
||||
#include <linux/bpf_lsm.h>
|
||||
#include <linux/btf.h>
|
||||
#include <linux/btf_ids.h>
|
||||
#include <linux/dcache.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/fsnotify.h>
|
||||
#include <linux/file.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/xattr.h>
|
||||
@@ -93,6 +95,24 @@ __bpf_kfunc int bpf_path_d_path(struct path *path, char *buf, size_t buf__sz)
|
||||
return len;
|
||||
}
|
||||
|
||||
static bool match_security_bpf_prefix(const char *name__str)
|
||||
{
|
||||
return !strncmp(name__str, XATTR_NAME_BPF_LSM, XATTR_NAME_BPF_LSM_LEN);
|
||||
}
|
||||
|
||||
static int bpf_xattr_read_permission(const char *name, struct inode *inode)
|
||||
{
|
||||
if (WARN_ON(!inode))
|
||||
return -EINVAL;
|
||||
|
||||
/* Allow reading xattr with user. and security.bpf. prefix */
|
||||
if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) &&
|
||||
!match_security_bpf_prefix(name))
|
||||
return -EPERM;
|
||||
|
||||
return inode_permission(&nop_mnt_idmap, inode, MAY_READ);
|
||||
}
|
||||
|
||||
/**
|
||||
* bpf_get_dentry_xattr - get xattr of a dentry
|
||||
* @dentry: dentry to get xattr from
|
||||
@@ -101,9 +121,10 @@ __bpf_kfunc int bpf_path_d_path(struct path *path, char *buf, size_t buf__sz)
|
||||
*
|
||||
* Get xattr *name__str* of *dentry* and store the output in *value_ptr*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "user." is allowed.
|
||||
* For security reasons, only *name__str* with prefixes "user." or
|
||||
* "security.bpf." are allowed.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
* Return: length of the xattr value on success, a negative value on error.
|
||||
*/
|
||||
__bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__str,
|
||||
struct bpf_dynptr *value_p)
|
||||
@@ -114,18 +135,12 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__st
|
||||
void *value;
|
||||
int ret;
|
||||
|
||||
if (WARN_ON(!inode))
|
||||
return -EINVAL;
|
||||
|
||||
if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
|
||||
return -EPERM;
|
||||
|
||||
value_len = __bpf_dynptr_size(value_ptr);
|
||||
value = __bpf_dynptr_data_rw(value_ptr, value_len);
|
||||
if (!value)
|
||||
return -EINVAL;
|
||||
|
||||
ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ);
|
||||
ret = bpf_xattr_read_permission(name__str, inode);
|
||||
if (ret)
|
||||
return ret;
|
||||
return __vfs_getxattr(dentry, inode, name__str, value, value_len);
|
||||
@@ -139,9 +154,10 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__st
|
||||
*
|
||||
* Get xattr *name__str* of *file* and store the output in *value_ptr*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "user." is allowed.
|
||||
* For security reasons, only *name__str* with prefixes "user." or
|
||||
* "security.bpf." are allowed.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
* Return: length of the xattr value on success, a negative value on error.
|
||||
*/
|
||||
__bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str,
|
||||
struct bpf_dynptr *value_p)
|
||||
@@ -154,6 +170,160 @@ __bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str,
|
||||
|
||||
__bpf_kfunc_end_defs();
|
||||
|
||||
static int bpf_xattr_write_permission(const char *name, struct inode *inode)
|
||||
{
|
||||
if (WARN_ON(!inode))
|
||||
return -EINVAL;
|
||||
|
||||
/* Only allow setting and removing security.bpf. xattrs */
|
||||
if (!match_security_bpf_prefix(name))
|
||||
return -EPERM;
|
||||
|
||||
return inode_permission(&nop_mnt_idmap, inode, MAY_WRITE);
|
||||
}
|
||||
|
||||
/**
|
||||
* bpf_set_dentry_xattr_locked - set a xattr of a dentry
|
||||
* @dentry: dentry to get xattr from
|
||||
* @name__str: name of the xattr
|
||||
* @value_p: xattr value
|
||||
* @flags: flags to pass into filesystem operations
|
||||
*
|
||||
* Set xattr *name__str* of *dentry* to the value in *value_ptr*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "security.bpf."
|
||||
* is allowed.
|
||||
*
|
||||
* The caller already locked dentry->d_inode.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
*/
|
||||
int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
|
||||
const struct bpf_dynptr *value_p, int flags)
|
||||
{
|
||||
|
||||
struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
|
||||
struct inode *inode = d_inode(dentry);
|
||||
const void *value;
|
||||
u32 value_len;
|
||||
int ret;
|
||||
|
||||
value_len = __bpf_dynptr_size(value_ptr);
|
||||
value = __bpf_dynptr_data(value_ptr, value_len);
|
||||
if (!value)
|
||||
return -EINVAL;
|
||||
|
||||
ret = bpf_xattr_write_permission(name__str, inode);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = __vfs_setxattr(&nop_mnt_idmap, dentry, inode, name__str,
|
||||
value, value_len, flags);
|
||||
if (!ret) {
|
||||
fsnotify_xattr(dentry);
|
||||
|
||||
/* This xattr is set by BPF LSM, so we do not call
|
||||
* security_inode_post_setxattr. Otherwise, we would
|
||||
* risk deadlocks by calling back to the same kfunc.
|
||||
*
|
||||
* This is the same as security_inode_setsecurity().
|
||||
*/
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* bpf_remove_dentry_xattr_locked - remove a xattr of a dentry
|
||||
* @dentry: dentry to get xattr from
|
||||
* @name__str: name of the xattr
|
||||
*
|
||||
* Rmove xattr *name__str* of *dentry*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "security.bpf."
|
||||
* is allowed.
|
||||
*
|
||||
* The caller already locked dentry->d_inode.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
*/
|
||||
int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str)
|
||||
{
|
||||
struct inode *inode = d_inode(dentry);
|
||||
int ret;
|
||||
|
||||
ret = bpf_xattr_write_permission(name__str, inode);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = __vfs_removexattr(&nop_mnt_idmap, dentry, name__str);
|
||||
if (!ret) {
|
||||
fsnotify_xattr(dentry);
|
||||
|
||||
/* This xattr is removed by BPF LSM, so we do not call
|
||||
* security_inode_post_removexattr. Otherwise, we would
|
||||
* risk deadlocks by calling back to the same kfunc.
|
||||
*/
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
__bpf_kfunc_start_defs();
|
||||
|
||||
/**
|
||||
* bpf_set_dentry_xattr - set a xattr of a dentry
|
||||
* @dentry: dentry to get xattr from
|
||||
* @name__str: name of the xattr
|
||||
* @value_p: xattr value
|
||||
* @flags: flags to pass into filesystem operations
|
||||
*
|
||||
* Set xattr *name__str* of *dentry* to the value in *value_ptr*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "security.bpf."
|
||||
* is allowed.
|
||||
*
|
||||
* The caller has not locked dentry->d_inode.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
*/
|
||||
__bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str,
|
||||
const struct bpf_dynptr *value_p, int flags)
|
||||
{
|
||||
struct inode *inode = d_inode(dentry);
|
||||
int ret;
|
||||
|
||||
inode_lock(inode);
|
||||
ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags);
|
||||
inode_unlock(inode);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* bpf_remove_dentry_xattr - remove a xattr of a dentry
|
||||
* @dentry: dentry to get xattr from
|
||||
* @name__str: name of the xattr
|
||||
*
|
||||
* Rmove xattr *name__str* of *dentry*.
|
||||
*
|
||||
* For security reasons, only *name__str* with prefix "security.bpf."
|
||||
* is allowed.
|
||||
*
|
||||
* The caller has not locked dentry->d_inode.
|
||||
*
|
||||
* Return: 0 on success, a negative value on error.
|
||||
*/
|
||||
__bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str)
|
||||
{
|
||||
struct inode *inode = d_inode(dentry);
|
||||
int ret;
|
||||
|
||||
inode_lock(inode);
|
||||
ret = bpf_remove_dentry_xattr_locked(dentry, name__str);
|
||||
inode_unlock(inode);
|
||||
return ret;
|
||||
}
|
||||
|
||||
__bpf_kfunc_end_defs();
|
||||
|
||||
BTF_KFUNCS_START(bpf_fs_kfunc_set_ids)
|
||||
BTF_ID_FLAGS(func, bpf_get_task_exe_file,
|
||||
KF_ACQUIRE | KF_TRUSTED_ARGS | KF_RET_NULL)
|
||||
@@ -161,6 +331,8 @@ BTF_ID_FLAGS(func, bpf_put_file, KF_RELEASE)
|
||||
BTF_ID_FLAGS(func, bpf_path_d_path, KF_TRUSTED_ARGS)
|
||||
BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
|
||||
BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
|
||||
BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
|
||||
BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS)
|
||||
BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)
|
||||
|
||||
static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
|
||||
@@ -171,6 +343,37 @@ static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
/* bpf_[set|remove]_dentry_xattr.* hooks have KF_TRUSTED_ARGS and
|
||||
* KF_SLEEPABLE, so they are only available to sleepable hooks with
|
||||
* dentry arguments.
|
||||
*
|
||||
* Setting and removing xattr requires exclusive lock on dentry->d_inode.
|
||||
* Some hooks already locked d_inode, while some hooks have not locked
|
||||
* d_inode. Therefore, we need different kfuncs for different hooks.
|
||||
* Specifically, hooks in the following list (d_inode_locked_hooks)
|
||||
* should call bpf_[set|remove]_dentry_xattr_locked; while other hooks
|
||||
* should call bpf_[set|remove]_dentry_xattr.
|
||||
*/
|
||||
BTF_SET_START(d_inode_locked_hooks)
|
||||
BTF_ID(func, bpf_lsm_inode_post_removexattr)
|
||||
BTF_ID(func, bpf_lsm_inode_post_setattr)
|
||||
BTF_ID(func, bpf_lsm_inode_post_setxattr)
|
||||
BTF_ID(func, bpf_lsm_inode_removexattr)
|
||||
BTF_ID(func, bpf_lsm_inode_rmdir)
|
||||
BTF_ID(func, bpf_lsm_inode_setattr)
|
||||
BTF_ID(func, bpf_lsm_inode_setxattr)
|
||||
BTF_ID(func, bpf_lsm_inode_unlink)
|
||||
#ifdef CONFIG_SECURITY_PATH
|
||||
BTF_ID(func, bpf_lsm_path_unlink)
|
||||
BTF_ID(func, bpf_lsm_path_rmdir)
|
||||
#endif /* CONFIG_SECURITY_PATH */
|
||||
BTF_SET_END(d_inode_locked_hooks)
|
||||
|
||||
bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
|
||||
{
|
||||
return btf_id_set_contains(&d_inode_locked_hooks, prog->aux->attach_btf_id);
|
||||
}
|
||||
|
||||
static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {
|
||||
.owner = THIS_MODULE,
|
||||
.set = &bpf_fs_kfunc_set_ids,
|
||||
|
||||
@@ -111,6 +111,7 @@ struct bpf_prog_list {
|
||||
struct bpf_prog *prog;
|
||||
struct bpf_cgroup_link *link;
|
||||
struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
|
||||
u32 flags;
|
||||
};
|
||||
|
||||
int cgroup_bpf_inherit(struct cgroup *cgrp);
|
||||
|
||||
+29
-2
@@ -968,6 +968,7 @@ struct bpf_insn_access_aux {
|
||||
struct {
|
||||
struct btf *btf;
|
||||
u32 btf_id;
|
||||
u32 ref_obj_id;
|
||||
};
|
||||
};
|
||||
struct bpf_verifier_log *log; /* for verbose logs */
|
||||
@@ -990,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
|
||||
return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
|
||||
}
|
||||
|
||||
/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
|
||||
* atomic load or store, and false if it is a read-modify-write instruction.
|
||||
*/
|
||||
static inline bool
|
||||
bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
|
||||
{
|
||||
switch (atomic_insn->imm) {
|
||||
case BPF_LOAD_ACQ:
|
||||
case BPF_STORE_REL:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct bpf_prog_ops {
|
||||
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
|
||||
union bpf_attr __user *uattr);
|
||||
@@ -1481,6 +1497,8 @@ struct bpf_ctx_arg_aux {
|
||||
enum bpf_reg_type reg_type;
|
||||
struct btf *btf;
|
||||
u32 btf_id;
|
||||
u32 ref_obj_id;
|
||||
bool refcounted;
|
||||
};
|
||||
|
||||
struct btf_mod_pair {
|
||||
@@ -1503,11 +1521,12 @@ struct bpf_prog_aux {
|
||||
u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
|
||||
u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
|
||||
u32 attach_btf_id; /* in-kernel BTF type id to attach to */
|
||||
u32 attach_st_ops_member_off;
|
||||
u32 ctx_arg_info_size;
|
||||
u32 max_rdonly_access;
|
||||
u32 max_rdwr_access;
|
||||
struct btf *attach_btf;
|
||||
const struct bpf_ctx_arg_aux *ctx_arg_info;
|
||||
struct bpf_ctx_arg_aux *ctx_arg_info;
|
||||
void __percpu *priv_stack_ptr;
|
||||
struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
|
||||
struct bpf_prog *dst_prog;
|
||||
@@ -1528,6 +1547,7 @@ struct bpf_prog_aux {
|
||||
bool jits_use_priv_stack;
|
||||
bool priv_stack_requested;
|
||||
bool changes_pkt_data;
|
||||
bool might_sleep;
|
||||
u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
|
||||
struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
|
||||
struct bpf_arena *arena;
|
||||
@@ -1547,6 +1567,7 @@ struct bpf_prog_aux {
|
||||
#endif
|
||||
struct bpf_ksym ksym;
|
||||
const struct bpf_prog_ops *ops;
|
||||
const struct bpf_struct_ops *st_ops;
|
||||
struct bpf_map **used_maps;
|
||||
struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
|
||||
struct btf_mod_pair *used_btfs;
|
||||
@@ -1945,6 +1966,9 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op
|
||||
|
||||
#endif
|
||||
|
||||
int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
|
||||
const struct bpf_ctx_arg_aux *info, u32 cnt);
|
||||
|
||||
#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
|
||||
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
|
||||
int cgroup_atype);
|
||||
@@ -1980,6 +2004,7 @@ struct bpf_array {
|
||||
*/
|
||||
enum {
|
||||
BPF_MAX_LOOPS = 8 * 1024 * 1024,
|
||||
BPF_MAX_TIMED_LOOPS = 0xffff,
|
||||
};
|
||||
|
||||
#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
|
||||
@@ -2036,6 +2061,8 @@ int bpf_prog_calc_tag(struct bpf_prog *fp);
|
||||
const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
|
||||
const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
|
||||
|
||||
const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
|
||||
|
||||
typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
|
||||
unsigned long off, unsigned long len);
|
||||
typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
|
||||
@@ -2546,7 +2573,7 @@ struct bpf_iter__bpf_map_elem {
|
||||
|
||||
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
|
||||
void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
|
||||
bool bpf_iter_prog_supported(struct bpf_prog *prog);
|
||||
int bpf_iter_prog_supported(struct bpf_prog *prog);
|
||||
const struct bpf_func_proto *
|
||||
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
|
||||
int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
|
||||
|
||||
@@ -48,6 +48,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func)
|
||||
|
||||
int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
|
||||
struct bpf_retval_range *range);
|
||||
int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
|
||||
const struct bpf_dynptr *value_p, int flags);
|
||||
int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str);
|
||||
bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog);
|
||||
|
||||
#else /* !CONFIG_BPF_LSM */
|
||||
|
||||
static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
|
||||
@@ -86,6 +91,19 @@ static inline int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static inline int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
|
||||
const struct bpf_dynptr *value_p, int flags)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static inline int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif /* CONFIG_BPF_LSM */
|
||||
|
||||
#endif /* _LINUX_BPF_LSM_H */
|
||||
|
||||
@@ -427,11 +427,6 @@ struct bpf_verifier_state {
|
||||
bool active_rcu_lock;
|
||||
|
||||
bool speculative;
|
||||
/* If this state was ever pointed-to by other state's loop_entry field
|
||||
* this flag would be set to true. Used to avoid freeing such states
|
||||
* while they are still in use.
|
||||
*/
|
||||
bool used_as_loop_entry;
|
||||
bool in_sleepable;
|
||||
|
||||
/* first and last insn idx of this verifier state */
|
||||
@@ -458,6 +453,11 @@ struct bpf_verifier_state {
|
||||
u32 dfs_depth;
|
||||
u32 callback_unroll_depth;
|
||||
u32 may_goto_depth;
|
||||
/* If this state was ever pointed-to by other state's loop_entry field
|
||||
* this flag would be set to true. Used to avoid freeing such states
|
||||
* while they are still in use.
|
||||
*/
|
||||
u32 used_as_loop_entry;
|
||||
};
|
||||
|
||||
#define bpf_get_spilled_reg(slot, frame, mask) \
|
||||
@@ -498,8 +498,10 @@ struct bpf_verifier_state {
|
||||
/* linked list of verifier states used to prune search */
|
||||
struct bpf_verifier_state_list {
|
||||
struct bpf_verifier_state state;
|
||||
struct bpf_verifier_state_list *next;
|
||||
int miss_cnt, hit_cnt;
|
||||
struct list_head node;
|
||||
u32 miss_cnt;
|
||||
u32 hit_cnt:31;
|
||||
u32 in_free_list:1;
|
||||
};
|
||||
|
||||
struct bpf_loop_inline_state {
|
||||
@@ -589,6 +591,8 @@ struct bpf_insn_aux_data {
|
||||
* accepts callback function as a parameter.
|
||||
*/
|
||||
bool calls_callback;
|
||||
/* registers alive before this instruction. */
|
||||
u16 live_regs_before;
|
||||
};
|
||||
|
||||
#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
|
||||
@@ -665,6 +669,7 @@ struct bpf_subprog_info {
|
||||
/* true if bpf_fastcall stack region is used by functions that can't be inlined */
|
||||
bool keep_fastcall_stack: 1;
|
||||
bool changes_pkt_data: 1;
|
||||
bool might_sleep: 1;
|
||||
|
||||
enum priv_stack_mode priv_stack_mode;
|
||||
u8 arg_cnt;
|
||||
@@ -710,8 +715,11 @@ struct bpf_verifier_env {
|
||||
bool test_state_freq; /* test verifier with different pruning frequency */
|
||||
bool test_reg_invariants; /* fail verification on register invariants violations */
|
||||
struct bpf_verifier_state *cur_state; /* current verifier state */
|
||||
struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
|
||||
struct bpf_verifier_state_list *free_list;
|
||||
/* Search pruning optimization, array of list_heads for
|
||||
* lists of struct bpf_verifier_state_list.
|
||||
*/
|
||||
struct list_head *explored_states;
|
||||
struct list_head free_list; /* list of struct bpf_verifier_state_list */
|
||||
struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */
|
||||
struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */
|
||||
u32 used_map_cnt; /* number of used maps */
|
||||
@@ -742,7 +750,11 @@ struct bpf_verifier_env {
|
||||
struct {
|
||||
int *insn_state;
|
||||
int *insn_stack;
|
||||
/* vector of instruction indexes sorted in post-order */
|
||||
int *insn_postorder;
|
||||
int cur_stack;
|
||||
/* current position in the insn_postorder vector */
|
||||
int cur_postorder;
|
||||
} cfg;
|
||||
struct backtrack_state bt;
|
||||
struct bpf_insn_hist_entry *insn_hist;
|
||||
@@ -767,6 +779,8 @@ struct bpf_verifier_env {
|
||||
u32 peak_states;
|
||||
/* longest register parentage chain walked for liveness marking */
|
||||
u32 longest_mark_read_walk;
|
||||
u32 free_list_size;
|
||||
u32 explored_states_size;
|
||||
bpfptr_t fd_array;
|
||||
|
||||
/* bit mask to keep track of whether a register has been accessed
|
||||
|
||||
@@ -76,6 +76,9 @@
|
||||
#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
|
||||
#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
|
||||
#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
|
||||
#define KF_ARENA_RET (1 << 13) /* kfunc returns an arena pointer */
|
||||
#define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */
|
||||
#define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */
|
||||
|
||||
/*
|
||||
* Tag marking a kernel function as a kfunc. This is meant to minimize the
|
||||
|
||||
@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
|
||||
* BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
|
||||
* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
|
||||
* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
|
||||
* BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
|
||||
* BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
|
||||
*/
|
||||
|
||||
#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \
|
||||
@@ -469,6 +471,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
|
||||
.off = 0, \
|
||||
.imm = BPF_CALL_IMM(FUNC) })
|
||||
|
||||
/* Kfunc call */
|
||||
|
||||
#define BPF_CALL_KFUNC(OFF, IMM) \
|
||||
((struct bpf_insn) { \
|
||||
.code = BPF_JMP | BPF_CALL, \
|
||||
.dst_reg = 0, \
|
||||
.src_reg = BPF_PSEUDO_KFUNC_CALL, \
|
||||
.off = OFF, \
|
||||
.imm = IMM })
|
||||
|
||||
/* Raw code statement block */
|
||||
|
||||
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \
|
||||
@@ -659,6 +671,11 @@ struct bpf_prog_stats {
|
||||
struct u64_stats_sync syncp;
|
||||
} __aligned(2 * sizeof(u64));
|
||||
|
||||
struct bpf_timed_may_goto {
|
||||
u64 count;
|
||||
u64 timestamp;
|
||||
};
|
||||
|
||||
struct sk_filter {
|
||||
refcount_t refcnt;
|
||||
struct rcu_head rcu;
|
||||
@@ -1120,8 +1137,11 @@ bool bpf_jit_supports_ptr_xchg(void);
|
||||
bool bpf_jit_supports_arena(void);
|
||||
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
|
||||
bool bpf_jit_supports_private_stack(void);
|
||||
bool bpf_jit_supports_timed_may_goto(void);
|
||||
u64 bpf_arch_uaddress_limit(void);
|
||||
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
||||
u64 arch_bpf_timed_may_goto(void);
|
||||
u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
|
||||
bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
|
||||
|
||||
static inline bool bpf_dump_raw_ok(const struct cred *cred)
|
||||
|
||||
@@ -426,14 +426,14 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule)
|
||||
#endif /* CONFIG_AUDIT */
|
||||
|
||||
#ifdef CONFIG_BPF_SYSCALL
|
||||
LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size)
|
||||
LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
|
||||
LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode)
|
||||
LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
|
||||
LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr,
|
||||
struct bpf_token *token)
|
||||
struct bpf_token *token, bool kernel)
|
||||
LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)
|
||||
LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr,
|
||||
struct bpf_token *token)
|
||||
struct bpf_token *token, bool kernel)
|
||||
LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)
|
||||
LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr,
|
||||
const struct path *path)
|
||||
|
||||
@@ -2477,6 +2477,11 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
|
||||
extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
|
||||
void *buf, int len, unsigned int gup_flags);
|
||||
|
||||
#ifdef CONFIG_BPF_SYSCALL
|
||||
extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
|
||||
void *buf, int len, unsigned int gup_flags);
|
||||
#endif
|
||||
|
||||
long get_user_pages_remote(struct mm_struct *mm,
|
||||
unsigned long start, unsigned long nr_pages,
|
||||
unsigned int gup_flags, struct page **pages,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user