wifi-rtl8852bs/core/rtw_prealloc.c

/******************************************************************************
 *
 * Copyright(c) 2016 - 2022 Realtek Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 *****************************************************************************/

#include <linux/kernel.h>
#include <linux/gfp.h>
#include <drv_types.h>
#include <rtw_mem.h>

#if defined(CONFIG_STACKTRACE) &&                                              \
	(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
#define STACKTRACE 1
#define MAX_STACK_TRACE 4
#endif /* CONFIG_STACKTRACE */

#define TRACE_ORDER(a, b) ((a) && (((a)&BIT(b)) > 0))

#if (defined(CONFIG_RTKM) && defined(CONFIG_RTKM_STANDALONE))
#ifdef pr_fmt
#undef pr_fmt
#endif
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#undef RTW_PRINT
#define RTW_PRINT pr_info
#else /* !CONFIG_RTKM */
#ifdef pr_fmt
#undef pr_fmt
#endif
#define pr_fmt(fmt) "RTKM: " fmt
#endif /* CONFIG_RTKM */

struct phy_mem_list {
	_lock lock;
	_list list;
	struct rb_root rb_tree;
	unsigned short entries;
	unsigned short used;
	unsigned short peak;
	unsigned char order;
} rtkm_phy_list[MAX_ORDER];

struct mem_entry {
	_list list;
	struct rb_node rbn;
	void *data;
	size_t size;
#ifdef STACKTRACE
#ifndef CONFIG_ARCH_STACKWALK
	struct stack_trace trace;
#endif /* CONFIG_ARCH_STACKWALK */
	unsigned long stack_entries[MAX_STACK_TRACE];
#endif /* STACKTRACE */
	unsigned char order;
	unsigned char is_use;
};

#ifdef STACKTRACE
#ifdef CONFIG_ARCH_STACKWALK
static inline void stacktrace_print(const struct mem_entry *entries,
				    unsigned int nr_entries, int spaces)
{
	stack_trace_print(entries->stack_entries, nr_entries, spaces);
}

static inline int stacktrace_save(struct mem_entry *store)
{
	return stack_trace_save(store->stack_entries,
				ARRAY_SIZE(store->stack_entries), 1);
}
#else /* !CONFIG_ARCH_STACKWALK */
static inline void stacktrace_print(const struct mem_entry *entries,
				    unsigned int nr_entries, int spaces)
{
	stack_trace_print(entries->trace.entries, nr_entries, spaces);
}

static inline void stacktrace_save(struct mem_entry *store)
{
	store->trace.skip = 0;
	store->trace.nr_entries = 0;
	store->trace.entries = store->stack_entries;
	store->trace.max_entries = MAX_STACK_TRACE;
	save_stack_trace(&store->trace);
}
#endif /* CONFIG_ARCH_STACKWALK */
#else /* !STACKTRACE */
#define stacktrace_print(a, b, c)
#define stacktrace_save(a)
#endif /* STACKTRACE */

/* Trace mpool */
static unsigned int rtkm_trace = 0;
module_param(rtkm_trace, uint, 0644);
MODULE_PARM_DESC(rtkm_trace, "Trace memory pool");

/* Preallocated memory expansion */
static bool rtkm_mem_exp = 1;
module_param(rtkm_mem_exp, bool, 0644);
MODULE_PARM_DESC(rtkm_mem_exp, "Preallocated memory expansion");

#ifndef RTKM_MPOOL_0
#define RTKM_MPOOL_0 0
#endif
#ifndef RTKM_MPOOL_1
#define RTKM_MPOOL_1 0
#endif
#ifndef RTKM_MPOOL_2
#define RTKM_MPOOL_2 0
#endif
#ifndef RTKM_MPOOL_3
#define RTKM_MPOOL_3 0
#endif
#ifndef RTKM_MPOOL_4
#define RTKM_MPOOL_4 0
#endif
#ifndef RTKM_MPOOL_5
#define RTKM_MPOOL_5 0
#endif
#ifndef RTKM_MPOOL_6
#define RTKM_MPOOL_6 0
#endif
#ifndef RTKM_MPOOL_7
#define RTKM_MPOOL_7 0
#endif
#ifndef RTKM_MPOOL_8
#define RTKM_MPOOL_8 0
#endif

/* Preallocated memory pool  */
static int mpool[MAX_ORDER] = {
	[0] = RTKM_MPOOL_0, [1] = RTKM_MPOOL_1, [2] = RTKM_MPOOL_2,
	[3] = RTKM_MPOOL_3, [4] = RTKM_MPOOL_4, [5] = RTKM_MPOOL_5,
	[6] = RTKM_MPOOL_6, [7] = RTKM_MPOOL_7, [8] = RTKM_MPOOL_8
};
static int n_mpool = 1;
module_param_array(mpool, int, &n_mpool, 0644);
MODULE_PARM_DESC(mpool, "Preallocated memory pool");

static inline void dump_mstatus(void *sel)
{
	int i;
	unsigned int a, b, c;
	long unsigned int musage = 0;

	a = b = c = 0;

	RTW_PRINT_SEL(sel,
		      "====================== RTKM ======================\n");
	RTW_PRINT_SEL(sel, "%6s %10s %10s %10s %10s\n", "order", "use", "peak",
		      rtkm_mem_exp ? "alloc+" : "alloc", "size");
	RTW_PRINT_SEL(sel,
		      "--------------------------------------------------\n");
	for (i = 0; i < MAX_ORDER; i++) {
		if (rtkm_phy_list[i].entries) {
			RTW_PRINT_SEL(
				sel, "%6d %10d %10d %10d %10lu\n", i,
				rtkm_phy_list[i].used, rtkm_phy_list[i].peak,
				rtkm_phy_list[i].entries,
				(rtkm_phy_list[i].entries) * (PAGE_SIZE << i));
			a += rtkm_phy_list[i].used;
			b += rtkm_phy_list[i].peak;
			c += rtkm_phy_list[i].entries;
			musage += (rtkm_phy_list[i].entries) * (PAGE_SIZE << i);
		}
	}
	RTW_PRINT_SEL(sel, "%6s %10d %10d %10d %10lu\n", "sum", a, b, c,
		      musage);
}

void rtkm_dump_mstatus(void *sel)
{
	dump_mstatus(sel);
}
EXPORT_SYMBOL(rtkm_dump_mstatus);

void rtkm_set_trace(unsigned int mask)
{
	rtkm_trace = mask;
}
EXPORT_SYMBOL(rtkm_set_trace);

static void rb_insert_mem(struct phy_mem_list *mlist, struct mem_entry *entry)
{
	struct rb_node **p = &mlist->rb_tree.rb_node;
	struct rb_node *parent = NULL;
	struct mem_entry *tmp = NULL;

	while (*p) {
		parent = *p;
		tmp = rb_entry(parent, struct mem_entry, rbn);

		if (tmp->data < entry->data)
			p = &(*p)->rb_left;
		else
			p = &(*p)->rb_right;
	}

	rb_link_node(&entry->rbn, parent, p);
	rb_insert_color(&entry->rbn, &mlist->rb_tree);
}

static struct mem_entry *rb_find_mem(struct phy_mem_list *mlist,
				     const void *objp)
{
	struct rb_node *n = mlist->rb_tree.rb_node;
	struct mem_entry *entry = NULL;

	while (n) {
		entry = rb_entry(n, struct mem_entry, rbn);

		if (entry->data == objp)
			return entry;
		else if (entry->data < objp)
			n = n->rb_left;
		else
			n = n->rb_right;
	}

	return NULL;
}

static inline void *create_mem_entry(int order)
{
	struct mem_entry *entry;

	entry = _rtw_malloc(sizeof(struct mem_entry));
	if (entry == NULL) {
		pr_warn("%s: alloc memory entry fail!\n", __func__);
		return NULL;
	}
	entry->order = order;
	entry->is_use = _FALSE;
	/* get memory by pages */
	entry->data = (void *)__get_free_pages(
		in_interrupt() ? GFP_ATOMIC : GFP_KERNEL, entry->order);
	if (entry->data == NULL) {
		_rtw_mfree(entry, sizeof(struct mem_entry));
		pr_warn("%s: alloc memory oreder-%d fail!\n", __func__, order);
		return NULL;
	}

	return entry;
}

static inline void *_kmalloc(size_t size, gfp_t flags, int clear)
{
	int order = 0;
	int warn = _FALSE;
	struct mem_entry *entry = NULL;

	order = get_order(size);

	if (rtkm_phy_list[order].entries == rtkm_phy_list[order].used) {
		if (rtkm_mem_exp) {
			warn = _TRUE;
			pr_warn("%s: No enough order-%d pool\n", __func__,
				order);
			entry = create_mem_entry(order);
			if (entry) {
				_rtw_spinlock_bh(&rtkm_phy_list[order].lock);
				list_add_tail(
					&entry->list,
					&rtkm_phy_list[entry->order].list);
				rtkm_phy_list[entry->order].entries++;
				_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);
			}
		}

		if (entry == NULL) {
			pr_warn("%s: No more memory for size %ld\n", __func__,
				size);
			WARN_ON(1);
			return NULL;
		}
	}

	_rtw_spinlock_bh(&rtkm_phy_list[order].lock);
	list_for_each_entry (entry, &rtkm_phy_list[order].list, list) {
		if (entry->is_use == _FALSE) {
			list_del_init(&entry->list);
			entry->is_use = _TRUE;
			entry->size = size;
			if (clear == _TRUE)
				memset(entry->data, 0, size);
			stacktrace_save(entry);
			rtkm_phy_list[order].used++;
			list_add_tail(&entry->list, &rtkm_phy_list[order].list);
			rb_insert_mem(&rtkm_phy_list[order], entry);
			break;
		}
	}
	if (rtkm_phy_list[order].peak < rtkm_phy_list[order].used)
		rtkm_phy_list[order].peak = rtkm_phy_list[order].used;
	_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);

	if ((warn) || TRACE_ORDER(rtkm_trace, order)) {
		pr_info("%s: require(%p, %lu) usage(%d %u/%u)\n", __func__,
			entry->data, entry->size, order,
			rtkm_phy_list[order].used,
			rtkm_phy_list[order].entries);
		stacktrace_print(entry, MAX_STACK_TRACE, 0);
	}

	return entry->data;
}

static inline void _kfree(const void *objp, size_t size)
{
	int found = _FALSE, order = 0;
	struct mem_entry *entry;

	order = get_order(size);

	if (!list_empty(&rtkm_phy_list[order].list)) {
		_rtw_spinlock_bh(&rtkm_phy_list[order].lock);

		entry = rb_find_mem(&rtkm_phy_list[order], objp);
		if (entry && (entry->is_use == _TRUE) &&
		    (entry->data == objp)) {
			if (TRACE_ORDER(rtkm_trace, order)) {
				pr_info("%s: release(%p, %lu)\n", __func__,
					objp, size);
			}
			rtw_list_delete(&entry->list);
			rtkm_phy_list[order].used--;
			entry->is_use = _FALSE;
			entry->size = 0;
			rb_erase(&entry->rbn, &rtkm_phy_list[order].rb_tree);
			list_add(&entry->list, &rtkm_phy_list[order].list);
			found = _TRUE;
		}

		_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);
	}
	if (found == _FALSE) {
		pr_warn("%s: not found (%p, %lu)\n", __func__, objp, size);
	}
}

void *rtkm_kmalloc(size_t size, gfp_t flags)
{
	if (size > RTKM_MGMT_SIZE)
		return _kmalloc(size, flags, _FALSE);
	else
		return kmalloc(size, flags);
}
EXPORT_SYMBOL(rtkm_kmalloc);

void *rtkm_kzalloc(size_t size, gfp_t flags)
{
	if (size > RTKM_MGMT_SIZE)
		return _kmalloc(size, flags, _TRUE);
	else
		return kzalloc(size, flags);
}
EXPORT_SYMBOL(rtkm_kzalloc);

void rtkm_kfree(const void *objp, size_t size)
{
	if (size > RTKM_MGMT_SIZE)
		return _kfree(objp, size);
	else
		return kfree(objp);
}
EXPORT_SYMBOL(rtkm_kfree);

static inline int rtkm_init_phy(void)
{
	int ret = 0, i, j;
	struct mem_entry *entry;

	pr_info("%s", __func__);
	pr_info("%s: memory expansion:%d\n", __func__, rtkm_mem_exp);

	for (i = (MAX_ORDER - 1); i > -1; i--) {
		INIT_LIST_HEAD(&rtkm_phy_list[i].list);
		_rtw_spinlock_init(&rtkm_phy_list[i].lock);
		rtkm_phy_list[i].rb_tree = RB_ROOT;

		for (j = 0; (ret == 0) && (j < mpool[i]); j++) {
			entry = create_mem_entry(i);
			if (entry == NULL) {
				ret = -ENOMEM;
				break;
			}

			list_add_tail(&entry->list,
				      &rtkm_phy_list[entry->order].list);
			rtkm_phy_list[entry->order].entries++;
		}
	}
	if (ret == 0)
		dump_mstatus(RTW_DBGDUMP);

	return ret;
}

static inline void rtkm_destroy_phy(void)
{
	int i = 0;
	struct mem_entry *entry;

	pr_info("%s", __func__);
	dump_mstatus(RTW_DBGDUMP);

	for (i = 0; i < MAX_ORDER; i++) {
		if (rtkm_phy_list[i].used)
			pr_err("%s: memory leak! order=%d num=%d\n", __func__,
			       i, rtkm_phy_list[i].used);

		if (rtkm_phy_list[i].rb_tree.rb_node != NULL)
			pr_err("%s: rb tree leak! order=%d\n", __func__, i);

		while (!list_empty(&rtkm_phy_list[i].list)) {
			entry = list_entry(rtkm_phy_list[i].list.next,
					   struct mem_entry, list);
			list_del_init(&entry->list);
			if (entry->is_use == _TRUE) {
				rb_erase(&entry->rbn,
					 &rtkm_phy_list[i].rb_tree);
				pr_err("%s: memory leak! (%p, %lu)\n", __func__,
				       entry->data, entry->size);
				stacktrace_print(entry, MAX_STACK_TRACE, 0);
			}
			if (entry->data)
				free_pages((unsigned long)(entry->data),
					   entry->order);
			entry->data = NULL;
			entry->size = 0;
			entry->is_use = _FALSE;
			_rtw_mfree(entry, sizeof(struct mem_entry));
			entry = NULL;
			rtkm_phy_list[i].entries--;
		}
		_rtw_spinlock_free(&rtkm_phy_list[i].lock);
	}
}

int rtkm_prealloc_init(void)
{
	int ret = 0;

	pr_info("%s\n", __func__);

	ret = rtkm_init_phy();
	if (ret == -ENOMEM) {
		pr_err("No enough memory for phiscal.");
		rtkm_destroy_phy();
	}

	pr_info("%s: done ret=%d\n", __func__, ret);
	return ret;
}
EXPORT_SYMBOL(rtkm_prealloc_init);

void rtkm_prealloc_destroy(void)
{
	pr_info("%s\n", __func__);

	rtkm_destroy_phy();

	pr_info("%s: done\n", __func__);
}
EXPORT_SYMBOL(rtkm_prealloc_destroy);