diff --git a/MAINTAINERS b/MAINTAINERS index 15011f5752a9..a3ed337e827d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14334,6 +14334,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git F: Documentation/admin-guide/mm/kho.rst F: Documentation/core-api/kho/* +F: include/asm-generic/kexec_handover.h F: include/linux/kexec_handover.h F: include/linux/kho/ F: include/linux/kho_block.h diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 8e4bf5365ac6..22267a83e064 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -32,6 +32,8 @@ #include /* For COMMAND_LINE_SIZE */ #undef _SETUP +#include + extern unsigned long get_cmd_line_ptr(void); /* Simplified build-specific string for starting entropy. */ diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild index 078fd2c0d69d..47ef8cb482e3 100644 --- a/arch/x86/include/asm/Kbuild +++ b/arch/x86/include/asm/Kbuild @@ -15,3 +15,4 @@ generic-y += fprobe.h generic-y += mcs_spinlock.h generic-y += mmzone.h generic-y += ring_buffer.h +generic-y += kexec_handover.h diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index 914eb32581c7..895d09faaf83 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -69,8 +69,6 @@ extern void x86_ce4100_early_setup(void); static inline void x86_ce4100_early_setup(void) { } #endif -#include - #ifndef _SETUP #include diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 46882ce79c3a..5ebb521e136d 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/include/asm-generic/kexec_handover.h b/include/asm-generic/kexec_handover.h new file mode 100644 index 000000000000..50839fb5ee8e --- /dev/null +++ b/include/asm-generic/kexec_handover.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_KEXEC_HANDOVER_H +#define __ASM_GENERIC_KEXEC_HANDOVER_H + +#include + +struct kho_scratch { + phys_addr_t addr; + phys_addr_t size; +}; + +#endif /* __ASM_GENERIC_KEXEC_HANDOVER_H */ diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h index 8968c56d2d73..c83ec4a15fe7 100644 --- a/include/linux/kexec_handover.h +++ b/include/linux/kexec_handover.h @@ -5,11 +5,8 @@ #include #include #include - -struct kho_scratch { - phys_addr_t addr; - phys_addr_t size; -}; +#include +#include struct kho_vmalloc; @@ -37,9 +34,20 @@ void kho_remove_subtree(void *blob); int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size); void kho_memory_init(void); +void kho_memory_init_early(void); void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, u64 scratch_len); + +bool kho_scratch_overlap(phys_addr_t phys, size_t size); + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + if (kho_scratch_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT)) + return MIGRATE_CMA; + return mt; +} #else static inline bool kho_is_enabled(void) { @@ -112,10 +120,23 @@ static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys, static inline void kho_memory_init(void) { } +static inline void kho_memory_init_early(void) { } + static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, u64 scratch_len) { } + +static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) +{ + return false; +} + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + return mt; +} #endif /* CONFIG_KEXEC_HANDOVER */ #endif /* LINUX_KEXEC_HANDOVER_H */ diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h index 5e2eb8519bda..2f4fb9c63942 100644 --- a/include/linux/kho/abi/kexec_handover.h +++ b/include/linux/kho/abi/kexec_handover.h @@ -257,11 +257,8 @@ struct kho_vmalloc { * memory. These constants govern the indexing, sizing, and depth of the tree. */ enum kho_radix_consts { - /* - * The bit position of the order bit (and also the length of the - * shifted physical address) for an order-0 page. - */ - KHO_ORDER_0_LOG2 = 64 - PAGE_SHIFT, + /* Need to store the PFN, plus one bit for order. */ + KHO_RADIX_KEY_WIDTH = 64 - PAGE_SHIFT + 1, /* Size of the table in kho_radix_node, in log2 */ KHO_TABLE_SIZE_LOG2 = const_ilog2(PAGE_SIZE / sizeof(phys_addr_t)), @@ -274,7 +271,7 @@ enum kho_radix_consts { * and 1 bitmap level. */ KHO_TREE_MAX_DEPTH = - DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2 + 1, + DIV_ROUND_UP(KHO_RADIX_KEY_WIDTH - KHO_BITMAP_SIZE_LOG2, KHO_TABLE_SIZE_LOG2) + 1, }; diff --git a/include/linux/kho_radix_tree.h b/include/linux/kho_radix_tree.h index 84e918b96e53..5d6ae2893684 100644 --- a/include/linux/kho_radix_tree.h +++ b/include/linux/kho_radix_tree.h @@ -34,37 +34,53 @@ struct kho_radix_tree { struct mutex lock; /* protects the tree's structure and root pointer */ }; -typedef int (*kho_radix_tree_walk_callback_t)(phys_addr_t phys, - unsigned int order); +/** + * struct kho_radix_walk_cb - Callbacks for KHO radix tree walk. + * @leaf: Called on each present key in the radix tree. + * @node: Called on each node of the radix tree itself. Receives the + * physical address of the page containing the node. + * + * For each callback, a return value of 0 continues the walk and a non-zero + * return value is directly returned to the caller. + */ +struct kho_radix_walk_cb { + int (*leaf)(unsigned long key, void *data); + int (*node)(phys_addr_t phys, void *data); +}; #ifdef CONFIG_KEXEC_HANDOVER -int kho_radix_add_page(struct kho_radix_tree *tree, unsigned long pfn, - unsigned int order); - -void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn, - unsigned int order); - +int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key); +void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key); int kho_radix_walk_tree(struct kho_radix_tree *tree, - kho_radix_tree_walk_callback_t cb); + const struct kho_radix_walk_cb *cb, void *data); +int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root); +void kho_radix_destroy_tree(struct kho_radix_tree *tree); #else /* #ifdef CONFIG_KEXEC_HANDOVER */ -static inline int kho_radix_add_page(struct kho_radix_tree *tree, long pfn, - unsigned int order) +static inline int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key) { return -EOPNOTSUPP; } -static inline void kho_radix_del_page(struct kho_radix_tree *tree, - unsigned long pfn, unsigned int order) { } +static inline void kho_radix_del_key(struct kho_radix_tree *tree, + unsigned long key) { } static inline int kho_radix_walk_tree(struct kho_radix_tree *tree, - kho_radix_tree_walk_callback_t cb) + const struct kho_radix_walk_cb *cb, void *data) { return -EOPNOTSUPP; } +static inline int kho_radix_init_tree(struct kho_radix_tree *tree, + struct kho_radix_node *root) +{ + return 0; +} + +static inline void kho_radix_destroy_tree(struct kho_radix_tree *tree) { } + #endif /* #ifdef CONFIG_KEXEC_HANDOVER */ #endif /* _LINUX_KHO_RADIX_TREE_H */ diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 5afcd99aa8c1..d62db9e776cf 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -51,6 +51,7 @@ extern unsigned long long max_possible_pfn; * memory reservations yet, so we get scratch memory from the previous * kernel that we know is good to use. It is the only memory that * allocations may happen from in this phase. + * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages */ enum memblock_flags { MEMBLOCK_NONE = 0x0, /* No special request */ @@ -61,6 +62,7 @@ enum memblock_flags { MEMBLOCK_RSRV_NOINIT = 0x10, /* don't initialize struct pages */ MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */ MEMBLOCK_KHO_SCRATCH = 0x40, /* scratch memory for kexec handover */ + MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */ }; /** @@ -420,6 +422,7 @@ void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, void *memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); +void *memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid); static __always_inline void *memblock_alloc(phys_addr_t size, phys_addr_t align) { @@ -484,6 +487,7 @@ static inline __init_memblock bool memblock_bottom_up(void) phys_addr_t memblock_phys_mem_size(void); phys_addr_t memblock_reserved_size(void); phys_addr_t memblock_reserved_kern_size(phys_addr_t limit, int nid); +phys_addr_t memblock_reserved_hugetlb_size(phys_addr_t limit, int nid); unsigned long memblock_estimated_nr_free_pages(void); phys_addr_t memblock_start_of_DRAM(void); phys_addr_t memblock_end_of_DRAM(void); @@ -613,28 +617,9 @@ static inline void memtest_report_meminfo(struct seq_file *m) { } #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH void memblock_set_kho_scratch_only(void); void memblock_clear_kho_scratch_only(void); -bool memblock_is_kho_scratch_memory(phys_addr_t addr); - -static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, - enum migratetype mt) -{ - if (memblock_is_kho_scratch_memory(PFN_PHYS(pfn))) - return MIGRATE_CMA; - return mt; -} #else static inline void memblock_set_kho_scratch_only(void) { } static inline void memblock_clear_kho_scratch_only(void) { } -static inline bool memblock_is_kho_scratch_memory(phys_addr_t addr) -{ - return false; -} - -static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, - enum migratetype mt) -{ - return mt; -} #endif #endif /* _LINUX_MEMBLOCK_H */ diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile index eec9d3ae07eb..d9f469462556 100644 --- a/kernel/liveupdate/Makefile +++ b/kernel/liveupdate/Makefile @@ -8,7 +8,6 @@ luo-y := \ luo_session.o obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o -obj-$(CONFIG_KEXEC_HANDOVER_DEBUG) += kexec_handover_debug.o obj-$(CONFIG_KEXEC_HANDOVER_DEBUGFS) += kexec_handover_debugfs.o obj-$(CONFIG_LIVEUPDATE) += luo.o diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 6fad9152387a..d14ae5d1e218 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -94,8 +94,25 @@ static struct kho_out kho_out = { }, }; +struct kho_in { + phys_addr_t fdt_phys; + phys_addr_t scratch_phys; + char previous_release[__NEW_UTS_LEN + 1]; + u32 kexec_count; + struct kho_debugfs dbg; + struct kho_radix_tree radix_tree; +}; + +static struct kho_in kho_in = { +}; + +static const void *kho_get_fdt(void) +{ + return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL; +} + /** - * kho_radix_encode_key - Encodes a physical address and order into a radix key. + * kho_encode_radix_key - Encodes a physical address and order into a radix key. * @phys: The physical address of the page. * @order: The order of the page. * @@ -105,35 +122,38 @@ static struct kho_out kho_out = { * * Return: The encoded unsigned long radix key. */ -static unsigned long kho_radix_encode_key(phys_addr_t phys, unsigned int order) +static unsigned long kho_encode_radix_key(phys_addr_t phys, unsigned int order) { - /* Order bits part */ - unsigned long h = 1UL << (KHO_ORDER_0_LOG2 - order); - /* Shifted physical address part */ - unsigned long l = phys >> (PAGE_SHIFT + order); + /* The physical address is encoded by shifting the PFN by its order. */ + unsigned long shift = PAGE_SHIFT + order; + /* Order bit goes right before the shifted PFN. */ + unsigned long h = 1UL << (64 - shift); + /* Shifted PFN. */ + unsigned long l = phys >> shift; return h | l; } /** - * kho_radix_decode_key - Decodes a radix key back into a physical address and order. + * kho_decode_radix_key - Decodes a radix key back into a physical address and order. * @key: The unsigned long key to decode. * @order: An output parameter, a pointer to an unsigned int where the decoded * page order will be stored. * - * This function reverses the encoding performed by kho_radix_encode_key(), + * This function reverses the encoding performed by kho_encode_radix_key(), * extracting the original physical address and page order from a given key. * * Return: The decoded physical address. */ -static phys_addr_t kho_radix_decode_key(unsigned long key, unsigned int *order) +static phys_addr_t kho_decode_radix_key(unsigned long key, unsigned int *order) { - unsigned int order_bit = fls64(key); + /* fls64() indexes starting from 1. */ + unsigned int order_bit = fls64(key) - 1; phys_addr_t phys; - /* order_bit is numbered starting at 1 from fls64 */ - *order = KHO_ORDER_0_LOG2 - order_bit + 1; - /* The order is discarded by the shift */ + /* order bit goes right before the shifted PFN. */ + *order = 64 - (PAGE_SHIFT + order_bit); + /* The order bit is discarded by the shift */ phys = key << (PAGE_SHIFT + *order); return phys; @@ -153,25 +173,47 @@ static unsigned long kho_radix_get_table_index(unsigned long key, return (key >> s) % (1 << KHO_TABLE_SIZE_LOG2); } +static void __ref *kho_radix_alloc_node(void) +{ + struct kho_radix_node *node; + + if (slab_is_available()) + node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL); + else + node = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + + return node; +} + +static void __ref kho_radix_free_node(struct kho_radix_node *node) +{ + if (slab_is_available()) + free_page((unsigned long)node); + else + memblock_free(node, PAGE_SIZE); +} + /** - * kho_radix_add_page - Marks a page as preserved in the radix tree. + * kho_radix_add_key - Add a key to the radix tree. * @tree: The KHO radix tree. - * @pfn: The page frame number of the page to preserve. - * @order: The order of the page. + * @key: The key to add. * - * This function traverses the radix tree based on the key derived from @pfn - * and @order. It sets the corresponding bit in the leaf bitmap to mark the - * page for preservation. If intermediate nodes do not exist along the path, - * they are allocated and added to the tree. + * This function traverses the radix tree based on the @key provided. It sets the + * corresponding bit in the leaf bitmap to mark the @key as present. If + * intermediate nodes do not exist along the path, they are allocated and added + * to the tree. + * + * NOTE: Currently only keys of width up to %KHO_RADIX_KEY_WIDTH are supported. + * This limit only exists because current users of the radix tree don't use more + * than that. Changing the maximum width requires changing the tree depth, which + * needs bumping the ABI version. * * Return: 0 on success, or a negative error code on failure. */ -int kho_radix_add_page(struct kho_radix_tree *tree, - unsigned long pfn, unsigned int order) +int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key) { /* Newly allocated nodes for error cleanup */ struct kho_radix_node *intermediate_nodes[KHO_TREE_MAX_DEPTH] = { 0 }; - unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order); struct kho_radix_node *anchor_node = NULL; struct kho_radix_node *node = tree->root; struct kho_radix_node *new_node; @@ -182,6 +224,9 @@ int kho_radix_add_page(struct kho_radix_tree *tree, if (WARN_ON_ONCE(!tree->root)) return -EINVAL; + if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH)) + return -ERANGE; + might_sleep(); guard(mutex)(&tree->lock); @@ -196,7 +241,7 @@ int kho_radix_add_page(struct kho_radix_tree *tree, } /* Next node is empty, create a new node for it */ - new_node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL); + new_node = kho_radix_alloc_node(); if (!new_node) { err = -ENOMEM; goto err_free_nodes; @@ -227,29 +272,26 @@ int kho_radix_add_page(struct kho_radix_tree *tree, err_free_nodes: for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) { if (intermediate_nodes[i]) - free_page((unsigned long)intermediate_nodes[i]); + kho_radix_free_node(intermediate_nodes[i]); } if (anchor_node) anchor_node->table[anchor_idx] = 0; return err; } -EXPORT_SYMBOL_GPL(kho_radix_add_page); +EXPORT_SYMBOL_GPL(kho_radix_add_key); /** - * kho_radix_del_page - Removes a page's preservation status from the radix tree. + * kho_radix_del_key - Removes the key from the radix tree. * @tree: The KHO radix tree. - * @pfn: The page frame number of the page to unpreserve. - * @order: The order of the page. + * @key: The key to remove. * * This function traverses the radix tree and clears the bit corresponding to - * the page, effectively removing its "preserved" status. It does not free - * the tree's intermediate nodes, even if they become empty. + * the @key, effectively removing it from the tree. It does not free the tree's + * intermediate nodes, even if they become empty. */ -void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn, - unsigned int order) +void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key) { - unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order); struct kho_radix_node *node = tree->root; struct kho_radix_leaf *leaf; unsigned int i, idx; @@ -257,6 +299,10 @@ void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn, if (WARN_ON_ONCE(!tree->root)) return; + /* Keys wider than KHO_RADIX_KEY_WIDTH are not allowed to be added. */ + if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH)) + return; + might_sleep(); guard(mutex)(&tree->lock); @@ -280,21 +326,85 @@ void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn, idx = kho_radix_get_bitmap_index(key); __clear_bit(idx, leaf->bitmap); } -EXPORT_SYMBOL_GPL(kho_radix_del_page); +EXPORT_SYMBOL_GPL(kho_radix_del_key); -static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf, - unsigned long key, - kho_radix_tree_walk_callback_t cb) +static void __kho_radix_destroy_tree(struct kho_radix_node *root, + unsigned int level) +{ + unsigned long i; + + if (level == 0) { + kho_radix_free_node(root); + return; + } + + for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) { + if (root->table[i]) + __kho_radix_destroy_tree(phys_to_virt(root->table[i]), + level - 1); + } + + kho_radix_free_node(root); +} + +/** + * kho_radix_init_tree - initialize the radix tree. + * @tree: the tree to initialize. + * @root: root table of the radix tree. + * + * Initialize the radix tree with the given root node. If root is %NULL, an + * empty root table is allocated. If root is not %NULL, it is the caller's + * responsibility to make sure the root is valid and in the correct format. + * + * Return: 0 on success, -errno on failure. + */ +int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root) +{ + if (!root) + root = kho_radix_alloc_node(); + if (!root) + return -ENOMEM; + + tree->root = root; + mutex_init(&tree->lock); + return 0; +} +EXPORT_SYMBOL_GPL(kho_radix_init_tree); + +/** + * kho_radix_destroy_tree - Destroy the radix tree + * @tree: The radix tree to destroy + * + * Walk @tree and free all its nodes. + */ +void kho_radix_destroy_tree(struct kho_radix_tree *tree) +{ + if (!tree->root) + return; + + __kho_radix_destroy_tree(tree->root, KHO_TREE_MAX_DEPTH - 1); + tree->root = NULL; +} +EXPORT_SYMBOL_GPL(kho_radix_destroy_tree); + +static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf, unsigned long key, + const struct kho_radix_walk_cb *cb, void *data) { unsigned long *bitmap = (unsigned long *)leaf; - unsigned int order; - phys_addr_t phys; unsigned int i; int err; + if (cb->node) { + err = cb->node(virt_to_phys(leaf), data); + if (err) + return err; + } + + if (!cb->leaf) + return 0; + for_each_set_bit(i, bitmap, PAGE_SIZE * BITS_PER_BYTE) { - phys = kho_radix_decode_key(key | i, &order); - err = cb(phys, order); + err = cb->leaf(key | i, data); if (err) return err; } @@ -304,7 +414,7 @@ static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf, static int __kho_radix_walk_tree(struct kho_radix_node *root, unsigned int level, unsigned long start, - kho_radix_tree_walk_callback_t cb) + const struct kho_radix_walk_cb *cb, void *data) { struct kho_radix_node *node; struct kho_radix_leaf *leaf; @@ -312,6 +422,12 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root, unsigned int shift; int err; + if (cb->node) { + err = cb->node(virt_to_phys(root), data); + if (err) + return err; + } + for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) { if (!root->table[i]) continue; @@ -328,10 +444,10 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root, * node is pointing to the level 0 bitmap. */ leaf = (struct kho_radix_leaf *)node; - err = kho_radix_walk_leaf(leaf, key, cb); + err = kho_radix_walk_leaf(leaf, key, cb, data); } else { err = __kho_radix_walk_tree(node, level - 1, - key, cb); + key, cb, data); } if (err) @@ -342,28 +458,27 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root, } /** - * kho_radix_walk_tree - Traverses the radix tree and calls a callback for each preserved page. + * kho_radix_walk_tree - Traverses the radix tree and calls a callback for each key. * @tree: A pointer to the KHO radix tree to walk. - * @cb: A callback function of type kho_radix_tree_walk_callback_t that will be - * invoked for each preserved page found in the tree. The callback receives - * the physical address and order of the preserved page. + * @cb: Set of callbacks to be invoked during the tree walk. + * @data: Opaque data pointer passed to each callback in @cb. * - * This function walks the radix tree, searching from the specified top level - * down to the lowest level (level 0). For each preserved page found, it invokes - * the provided callback, passing the page's physical address and order. + * This function walks the radix tree, searching from the top level down to the + * lowest level (level 0), invoking the appropriate callbacks. * * Return: 0 if the walk completed the specified tree, or the non-zero return * value from the callback that stopped the walk. */ int kho_radix_walk_tree(struct kho_radix_tree *tree, - kho_radix_tree_walk_callback_t cb) + const struct kho_radix_walk_cb *cb, void *data) { if (WARN_ON_ONCE(!tree->root)) return -EINVAL; guard(mutex)(&tree->lock); - return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb); + return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb, + data); } EXPORT_SYMBOL_GPL(kho_radix_walk_tree); @@ -494,13 +609,16 @@ static struct page *__init kho_get_preserved_page(phys_addr_t phys, return pfn_to_page(pfn); } -static int __init kho_preserved_memory_reserve(phys_addr_t phys, - unsigned int order) +static int __init kho_preserved_memory_reserve(unsigned long key, void *data) { union kho_page_info info; struct page *page; + unsigned int order; + phys_addr_t phys; u64 sz; + phys = kho_decode_radix_key(key, &order); + sz = 1UL << (order + PAGE_SHIFT); page = kho_get_preserved_page(phys, order); @@ -514,19 +632,24 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, return 0; } -/* Returns physical address of the preserved memory map from FDT */ -static phys_addr_t __init kho_get_mem_map_phys(const void *fdt) +/* Returns virtual address of the preserved memory map from FDT */ +static __init void *kho_get_mem_map(const void *fdt) { const void *mem_ptr; + phys_addr_t mem_map_phys; int len; mem_ptr = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len); if (!mem_ptr || len != sizeof(u64)) { pr_err("failed to get preserved memory map\n"); - return 0; + return NULL; } - return get_unaligned((const u64 *)mem_ptr); + mem_map_phys = get_unaligned((const u64 *)mem_ptr); + if (!mem_map_phys) + return NULL; + + return phys_to_virt(mem_map_phys); } /* @@ -629,19 +752,24 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { /* - * If fixed sizes are not provided via command line, calculate them - * now. + * If fixed sizes are not provided via command line, calculate them now. + * Remove HugeTLB allocations from it because they never get allocated + * from scratch. */ if (scratch_scale) { phys_addr_t size; size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, NUMA_NO_NODE); + size -= memblock_reserved_hugetlb_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); size = size * scratch_scale / 100; scratch_size_lowmem = size; size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, NUMA_NO_NODE); + size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); size = size * scratch_scale / 100 - scratch_size_lowmem; scratch_size_global = size; } @@ -661,6 +789,9 @@ static phys_addr_t __init scratch_size_node(int nid) if (scratch_scale) { size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, nid); + /* Do not count HugeTLB pages. */ + size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE, + nid); size = size * scratch_scale / 100; } else { size = scratch_size_pernode; @@ -669,6 +800,22 @@ static phys_addr_t __init scratch_size_node(int nid) return round_up(size, SCRATCH_ALIGNMENT_BYTES); } +bool kho_scratch_overlap(phys_addr_t phys, size_t size) +{ + phys_addr_t scratch_start, scratch_end; + unsigned int i; + + for (i = 0; i < kho_scratch_cnt; i++) { + scratch_start = kho_scratch[i].addr; + scratch_end = kho_scratch[i].addr + kho_scratch[i].size; + + if (phys < scratch_end && (phys + size) > scratch_start) + return true; + } + + return false; +} + /** * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec * @@ -756,6 +903,132 @@ err_disable_kho: kho_enable = false; } +/* + * Look for free blocks of 1G. This is a heuristic chosen to work efficiently + * with large systems with hundreds of gigabytes of memory. It will work poorly + * on smaller systems. The algorithm itself doesn't depend on the actual value, + * so it can be changed to a different heuristic later if needed. + */ +#define KHO_SCRATCH_EXT_BLKSIZE SZ_1G +#define KHO_SCRATCH_EXT_BLKSHIFT const_ilog2(KHO_SCRATCH_EXT_BLKSIZE) + +/* Called for the KHO preserved memory radix tree. */ +static int __init kho_ext_walk_leaf(unsigned long key, void *data) +{ + /* Radix tree tracking free blocks. */ + struct kho_radix_tree *tree = data; + phys_addr_t start, end; + unsigned int order; + int err; + + /* + * The key is from the KHO preserved memory radix tree. It is decoded to + * a physical address of a preservation and its order. + */ + start = kho_decode_radix_key(key, &order); + end = start + (1UL << (order + PAGE_SHIFT)); + + while (start < end) { + err = kho_radix_add_key(tree, start >> KHO_SCRATCH_EXT_BLKSHIFT); + if (err) + return err; + + start += (1UL << KHO_SCRATCH_EXT_BLKSHIFT); + } + + return 0; +} + +/* Called for the KHO preserved memory radix tree. */ +static int __init kho_ext_walk_node(phys_addr_t phys, void *data) +{ + /* Radix tree tracking free blocks. */ + struct kho_radix_tree *tree = data; + + return kho_radix_add_key(tree, phys >> KHO_SCRATCH_EXT_BLKSHIFT); +} + +/* Called for the free block radix tree. */ +static int __init kho_ext_mark_scratch(unsigned long key, void *data) +{ + phys_addr_t *prev_end = data; + phys_addr_t start = key << KHO_SCRATCH_EXT_BLKSHIFT; + int err; + + if (start > *prev_end) { + err = memblock_mark_kho_scratch(*prev_end, start - *prev_end); + if (err) + return err; + } + + *prev_end = start + (1UL << KHO_SCRATCH_EXT_BLKSHIFT); + return 0; +} + +/* + * kho_extend_scratch - Extend the scratch regions + * + * The KHO preserved memory radix tree mixes both physical address and order + * into a single key. This makes it hard to look for free ranges directly. This + * function first walks the radix tree and digests it down into another radix + * tree, whose keys identify blocks of size KHO_SCRATCH_EXT_BLKSIZE which + * contain preserved memory. + * + * Then it walks the digested radix tree and marks everything that doesn't have + * preserved memory as scratch. + * + * NOTE: This function allocates memory so it should be called when scratch has + * available space. + * + * NOTE: The pages of the KHO preserved memory radix tree tables are not marked + * as preserved in the preserved memory tree. But they are expected to remain + * untouched until the tree is fully parsed. So this function also considers + * them to be "preserved memory" and marks their blocks as busy. + * + * NOTE: efi_init()::reserve_regions() removes all regions except + * MEMBLOCK_KHO_SCRATCH. This function adds such regions but they are not KHO + * scratch memory, so they should not be removed. This function should always be + * called after reserve_regions(). + */ +static void __init kho_extend_scratch(void) +{ + const struct kho_radix_walk_cb kho_cb = { + .leaf = kho_ext_walk_leaf, + .node = kho_ext_walk_node, + }; + const struct kho_radix_walk_cb ext_cb = { + .leaf = kho_ext_mark_scratch, + }; + struct kho_radix_tree radix; + phys_addr_t prev_end = 0; + int err = 0; + + err = kho_radix_init_tree(&radix, NULL); + if (err) + goto print; + + /* Walk the KHO radix tree to find busy blocks. */ + err = kho_radix_walk_tree(&kho_in.radix_tree, &kho_cb, &radix); + if (err) + goto out; + + /* Walk the blocks and mark everything between keys as scratch. */ + err = kho_radix_walk_tree(&radix, &ext_cb, &prev_end); + if (err) + goto out; + + /* Mark everything from last busy block to end of DRAM. */ + if (prev_end < memblock_end_of_DRAM()) + err = memblock_mark_kho_scratch(prev_end, memblock_end_of_DRAM() - prev_end); + + /* fallthrough */ +out: + kho_radix_destroy_tree(&radix); +print: + if (err) + pr_err("Failed to extend scratch: %pe\n", ERR_PTR(err)); +} + /** * kho_add_subtree - record the physical address of a sub blob in KHO root tree. * @name: name of the sub tree. @@ -866,10 +1139,12 @@ int kho_preserve_folio(struct folio *folio) const unsigned long pfn = folio_pfn(folio); const unsigned int order = folio_order(folio); - if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order))) + if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) && + WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order))) return -EINVAL; - return kho_radix_add_page(tree, pfn, order); + return kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), + order)); } EXPORT_SYMBOL_GPL(kho_preserve_folio); @@ -887,7 +1162,7 @@ void kho_unpreserve_folio(struct folio *folio) const unsigned long pfn = folio_pfn(folio); const unsigned int order = folio_order(folio); - kho_radix_del_page(tree, pfn, order); + kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), order)); } EXPORT_SYMBOL_GPL(kho_unpreserve_folio); @@ -916,7 +1191,8 @@ static void __kho_unpreserve(struct kho_radix_tree *tree, while (pfn < end_pfn) { order = __kho_preserve_pages_order(pfn, end_pfn); - kho_radix_del_page(tree, pfn, order); + kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), + order)); pfn += 1 << order; } @@ -941,7 +1217,8 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages) unsigned long failed_pfn = 0; int err = 0; - if (WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT, + if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) && + WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT))) { return -EINVAL; } @@ -949,7 +1226,8 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages) while (pfn < end_pfn) { unsigned int order = __kho_preserve_pages_order(pfn, end_pfn); - err = kho_radix_add_page(tree, pfn, order); + err = kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), + order)); if (err) { failed_pfn = pfn; break; @@ -1325,22 +1603,6 @@ void kho_restore_free(void *mem) } EXPORT_SYMBOL_GPL(kho_restore_free); -struct kho_in { - phys_addr_t fdt_phys; - phys_addr_t scratch_phys; - char previous_release[__NEW_UTS_LEN + 1]; - u32 kexec_count; - struct kho_debugfs dbg; -}; - -static struct kho_in kho_in = { -}; - -static const void *kho_get_fdt(void) -{ - return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL; -} - /** * is_kho_boot - check if current kernel was booted via KHO-enabled * kexec @@ -1408,26 +1670,24 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size) } EXPORT_SYMBOL_GPL(kho_retrieve_subtree); -static int __init kho_mem_retrieve(const void *fdt) +static void __init kho_mem_retrieve(void) { - struct kho_radix_tree tree; - const phys_addr_t *mem; - int len; + const struct kho_radix_walk_cb cb = { + .leaf = kho_preserved_memory_reserve, + }; - /* Retrieve the KHO radix tree from passed-in FDT. */ - mem = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len); + if (kho_radix_walk_tree(&kho_in.radix_tree, &cb, NULL)) + goto err; - if (!mem || len != sizeof(*mem)) { - pr_err("failed to get preserved KHO memory tree\n"); - return -ENOENT; - } + return; - if (!*mem) - return -EINVAL; - - tree.root = phys_to_virt(*mem); - mutex_init(&tree.lock); - return kho_radix_walk_tree(&tree, kho_preserved_memory_reserve); +err: + /* + * Failed to initialize preserved memory. Clear FDT and radix so KHO + * users don't treat it as a KHO boot. + */ + kho_in.fdt_phys = 0; + kho_in.radix_tree.root = NULL; } static __init int kho_out_fdt_setup(void) @@ -1553,16 +1813,14 @@ static __init int kho_init(void) if (!kho_enable) return 0; - tree->root = kzalloc(PAGE_SIZE, GFP_KERNEL); - if (!tree->root) { - err = -ENOMEM; + err = kho_radix_init_tree(tree, NULL); + if (err) goto err_free_scratch; - } kho_out.fdt = kho_alloc_preserve(PAGE_SIZE); if (IS_ERR(kho_out.fdt)) { err = PTR_ERR(kho_out.fdt); - goto err_free_kho_radix_tree_root; + goto err_free_kho_radix_tree; } err = kho_debugfs_init(); @@ -1613,9 +1871,8 @@ static __init int kho_init(void) err_free_fdt: kho_unpreserve_free(kho_out.fdt); -err_free_kho_radix_tree_root: - kfree(tree->root); - tree->root = NULL; +err_free_kho_radix_tree: + kho_radix_destroy_tree(tree); err_free_scratch: kho_out.fdt = NULL; for (int i = 0; i < kho_scratch_cnt; i++) { @@ -1629,16 +1886,43 @@ err_free_scratch: } fs_initcall(kho_init); +void __init kho_memory_init_early(void) +{ + const void *fdt = kho_get_fdt(); + + if (!is_kho_boot()) + return; + + /* + * kho_scratch_overlap() needs kho_scratch to be initialized. It + * is used by free_area_init() on KHO boots, so initialize it + * early. + */ + kho_scratch = phys_to_virt(kho_in.scratch_phys); + + /* + * kho_get_mem_map() should always succeed. If it fails, kho_populate() + * catches that and never sets kho_in.fdt_phys. + */ + if (kho_radix_init_tree(&kho_in.radix_tree, kho_get_mem_map(fdt))) { + /* + * Failed to initialize preserved memory radix tree. Clear FDT + * and scratch so KHO users don't treat it as a KHO boot. + */ + kho_in.fdt_phys = 0; + kho_in.scratch_phys = 0; + return; + } + + kho_extend_scratch(); +} + void __init kho_memory_init(void) { - if (kho_in.scratch_phys) { - kho_scratch = phys_to_virt(kho_in.scratch_phys); - - if (kho_mem_retrieve(kho_get_fdt())) - kho_in.fdt_phys = 0; - } else { + if (kho_in.scratch_phys) + kho_mem_retrieve(); + else kho_reserve_scratch(); - } } void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len, @@ -1646,9 +1930,8 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len, { unsigned int scratch_cnt = scratch_len / sizeof(*kho_scratch); struct kho_scratch *scratch = NULL; - phys_addr_t mem_map_phys; - void *fdt = NULL; bool populated = false; + void *fdt = NULL; int err; /* Validate the input FDT */ @@ -1670,8 +1953,13 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len, goto unmap_fdt; } - mem_map_phys = kho_get_mem_map_phys(fdt); - if (!mem_map_phys) + /* + * At this point phys_to_virt() doesn't work properly and so + * kho_get_mem_map() can return a pre-KASLR virtual address. But here we + * only want to make sure the mem_map is valid so the actual value + * doesn't matter as long as it isn't NULL. + */ + if (!kho_get_mem_map(fdt)) goto unmap_fdt; scratch = early_memremap(scratch_phys, scratch_len); diff --git a/kernel/liveupdate/kexec_handover_debug.c b/kernel/liveupdate/kexec_handover_debug.c deleted file mode 100644 index 6efb696f5426..000000000000 --- a/kernel/liveupdate/kexec_handover_debug.c +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * kexec_handover_debug.c - kexec handover optional debug functionality - * Copyright (C) 2025 Google LLC, Pasha Tatashin - */ - -#define pr_fmt(fmt) "KHO: " fmt - -#include "kexec_handover_internal.h" - -bool kho_scratch_overlap(phys_addr_t phys, size_t size) -{ - phys_addr_t scratch_start, scratch_end; - unsigned int i; - - for (i = 0; i < kho_scratch_cnt; i++) { - scratch_start = kho_scratch[i].addr; - scratch_end = kho_scratch[i].addr + kho_scratch[i].size; - - if (phys < scratch_end && (phys + size) > scratch_start) - return true; - } - - return false; -} diff --git a/kernel/liveupdate/kexec_handover_internal.h b/kernel/liveupdate/kexec_handover_internal.h index 0399ff107775..805d2a76c388 100644 --- a/kernel/liveupdate/kexec_handover_internal.h +++ b/kernel/liveupdate/kexec_handover_internal.h @@ -41,13 +41,4 @@ static inline void kho_debugfs_blob_remove(struct kho_debugfs *dbg, void *blob) { } #endif /* CONFIG_KEXEC_HANDOVER_DEBUGFS */ -#ifdef CONFIG_KEXEC_HANDOVER_DEBUG -bool kho_scratch_overlap(phys_addr_t phys, size_t size); -#else -static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) -{ - return false; -} -#endif /* CONFIG_KEXEC_HANDOVER_DEBUG */ - #endif /* LINUX_KEXEC_HANDOVER_INTERNAL_H */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 571212b80835..ab4afc818e8c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3033,29 +3033,21 @@ static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) if (hugetlb_early_cma(h)) m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), - huge_page_size(h), 0, - MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), - huge_page_size(h), 0, - MEMBLOCK_ALLOC_ACCESSIBLE, nid); + m = memblock_alloc_hugetlb(huge_page_size(h), nid, node_exact); + if (m) { + m->flags = 0; + m->cma = NULL; + /* * For pre-HVO to work correctly, pages need to be on * the list for the node they were actually allocated * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, + * fallback by memblock_alloc_hugetlb_bootmem. So, * extract the actual node first. */ - if (m) + if (!node_exact) listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); } - - if (m) { - m->flags = 0; - m->cma = NULL; - } } if (m) { diff --git a/mm/memblock.c b/mm/memblock.c index 6349c48154f4..78112b3f00d0 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -19,11 +19,9 @@ #include #include -#ifdef CONFIG_KEXEC_HANDOVER #include #include #include -#endif /* CONFIG_KEXEC_HANDOVER */ #include #include @@ -1506,6 +1504,32 @@ int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, return 0; } +static void memblock_prep_allocation(phys_addr_t start, phys_addr_t size, + bool kmemleak_trace) +{ + /* + * Skip kmemleak for those places like kasan_init() and + * early_pgtable_alloc() due to high volume. + */ + if (kmemleak_trace) + /* + * Memblock allocated blocks are never reported as + * leaks. This is because many of these blocks are + * only referred via the physical address which is + * not looked up by kmemleak. + */ + kmemleak_alloc_phys(start, size, 0); + + /* + * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, + * require memory to be accepted before it can be used by the + * guest. + * + * Accept the memory of the allocated buffer. + */ + accept_memory(start, size); +} + /** * memblock_alloc_range_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes @@ -1580,28 +1604,7 @@ again: return 0; done: - /* - * Skip kmemleak for those places like kasan_init() and - * early_pgtable_alloc() due to high volume. - */ - if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) - /* - * Memblock allocated blocks are never reported as - * leaks. This is because many of these blocks are - * only referred via the physical address which is - * not looked up by kmemleak. - */ - kmemleak_alloc_phys(found, size, 0); - - /* - * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, - * require memory to be accepted before it can be used by the - * guest. - * - * Accept the memory of the allocated buffer. - */ - accept_memory(found, size); - + memblock_prep_allocation(found, size, end != MEMBLOCK_ALLOC_NOLEAKTRACE); return found; } @@ -1756,6 +1759,69 @@ void * __init memblock_alloc_try_nid_raw( false); } +/** + * memblock_alloc_hugetlb - allocate boot memory for HugeTLB pages + * @size: size of the memory to be allocated in bytes + * @nid: nid of the free memory to find, %NUMA_NO_NODE for any node + * @exact_nid: only allocate from the specified nid. If %false, the specified + * nid is tried first, and then all nodes are tried as fallback. + * + * HugeTLB pages are always aligned by their size, so the alignment matches + * @size. Since the memory is for userspace, mirrored memory is not used. The + * memory is not zeroed. Does not panic if request cannot be satisfied. + * + * Return: + * Virtual address of allocated memory block on success, %NULL on failure. + */ +void * __init memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid) +{ + enum memblock_flags flags = choose_memblock_flags(); + phys_addr_t addr, start = 0, end = MEMBLOCK_ALLOC_ACCESSIBLE; + + memblock_dbg("%s: %llu bytes, nid=%d, exact_nid=%d %pS\n", __func__, + (u64)size, nid, exact_nid, (void *)_RET_IP_); + + /* Don't waste mirrored memory on HugeTLB pages. */ + flags &= ~MEMBLOCK_MIRROR; +retry: + /* HugeTLB pages are always aligned by their size. */ + addr = memblock_find_in_range_node(size, size, start, end, nid, flags); + if (addr) + goto found; + + /* Try all nodes if allowed. */ + if (numa_valid_node(nid) && !exact_nid) { + nid = NUMA_NO_NODE; + goto retry; + } + + /* Found nothing... :-( */ + return NULL; + +found: + /* + * HugeTLB pages can be preserved with KHO and no preserved memory can + * be in scratch. So retry if found address overlaps with scratch. + * + * Scratch areas are normally not very large, so this shouldn't take too + * many retries. + */ + if (kho_scratch_overlap(addr, size)) { + if (memblock_bottom_up()) + start = addr + size; + else + start = addr - size; + + goto retry; + } + + if (__memblock_reserve(addr, size, nid, MEMBLOCK_RSRV_KERN | MEMBLOCK_RSRV_HUGETLB)) + return NULL; + + memblock_prep_allocation(addr, size, true); + return phys_to_virt(addr); +} + /** * memblock_alloc_try_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes @@ -1825,6 +1891,28 @@ phys_addr_t __init_memblock memblock_reserved_size(void) return memblock.reserved.total_size; } +phys_addr_t __init_memblock memblock_reserved_hugetlb_size(phys_addr_t limit, int nid) +{ + struct memblock_region *r; + phys_addr_t total = 0; + + for_each_reserved_mem_region(r) { + phys_addr_t size = r->size; + + if (r->base > limit) + break; + + if (r->base + r->size > limit) + size = limit - r->base; + + if (nid == memblock_get_region_node(r) || !numa_valid_node(nid)) + if (r->flags & MEMBLOCK_RSRV_HUGETLB) + total += size; + } + + return total; +} + phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid) { struct memblock_region *r; @@ -2511,16 +2599,6 @@ __init void memblock_clear_kho_scratch_only(void) { kho_scratch_only = false; } - -bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr) -{ - int i = memblock_search(&memblock.memory, addr); - - if (i == -1) - return false; - - return memblock_is_kho_scratch(&memblock.memory.regions[i]); -} #endif #ifdef CONFIG_KEXEC_HANDOVER diff --git a/mm/mm_init.c b/mm/mm_init.c index 0f64909e8d20..dc20d6814e48 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -674,15 +674,17 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#ifdef CONFIG_ZONE_DEVICE +#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype) + unsigned long nr_pages, int migratetype, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { - init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false); - if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); + + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); } } @@ -932,8 +934,9 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone * over the place during system boot. */ if (pageblock_aligned(pfn)) { - init_pageblock_migratetype(page, migratetype, - isolate_pageblock); + enum migratetype mt = kho_scratch_migratetype(pfn, migratetype); + + init_pageblock_migratetype(page, mt, isolate_pageblock); cond_resched(); } pfn++; @@ -943,8 +946,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone static void __init memmap_init_zone_range(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn, - unsigned long *hole_pfn, - enum migratetype mt) + unsigned long *hole_pfn) { unsigned long zone_start_pfn = zone->zone_start_pfn; unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; @@ -957,7 +959,8 @@ static void __init memmap_init_zone_range(struct zone *zone, return; memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, - zone_end_pfn, MEMINIT_EARLY, NULL, mt, false); + zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, + false); if (*hole_pfn < start_pfn) init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); @@ -973,8 +976,6 @@ static void __init memmap_init(void) for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { struct pglist_data *node = NODE_DATA(nid); - enum migratetype mt = - kho_scratch_migratetype(start_pfn, MIGRATE_MOVABLE); for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = node->node_zones + j; @@ -983,7 +984,7 @@ static void __init memmap_init(void) continue; memmap_init_zone_range(zone, start_pfn, end_pfn, - &hole_pfn, mt); + &hole_pfn); zone_id = j; } } @@ -1142,7 +1143,7 @@ void __ref memmap_init_zone_device(struct zone *zone, compound_nr_pages(pfn, altmap, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1973,7 +1974,7 @@ unsigned long __init node_map_pfn_alignment(void) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static void __init deferred_free_pages(unsigned long pfn, - unsigned long nr_pages, enum migratetype mt) + unsigned long nr_pages) { struct page *page; unsigned long i; @@ -1981,12 +1982,12 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; + pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, true); + page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { - for (i = 0; i < nr_pages; i += pageblock_nr_pages) - init_pageblock_migratetype(page + i, mt, false); __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); return; } @@ -1994,11 +1995,8 @@ static void __init deferred_free_pages(unsigned long pfn, /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); - for (i = 0; i < nr_pages; i++, page++, pfn++) { - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(page, mt, false); - __free_pages_core(page, 0, MEMINIT_EARLY); - } + for (i = 0; i < nr_pages; i++) + __free_pages_core(page + i, 0, MEMINIT_EARLY); } /* Completion tracking for deferred_init_memmap() threads */ @@ -2054,8 +2052,6 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { unsigned long spfn = PFN_UP(start); unsigned long epfn = PFN_DOWN(end); - enum migratetype mt = - kho_scratch_migratetype(spfn, MIGRATE_MOVABLE); if (spfn >= end_pfn) break; @@ -2068,7 +2064,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, unsigned long chunk_end = min(mo_pfn, epfn); nr_pages += deferred_init_pages(zone, spfn, chunk_end); - deferred_free_pages(spfn, chunk_end - spfn, mt); + deferred_free_pages(spfn, chunk_end - spfn); spfn = chunk_end; @@ -2687,6 +2683,7 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { + kho_memory_init_early(); hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); diff --git a/tools/testing/memblock/internal.h b/tools/testing/memblock/internal.h index b6b1d147fd75..e86bb9000b22 100644 --- a/tools/testing/memblock/internal.h +++ b/tools/testing/memblock/internal.h @@ -66,4 +66,9 @@ static inline void init_deferred_page(unsigned long pfn, int nid) #define __SetPageReserved(p) ((void)(p)) +static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) +{ + return false; +} + #endif