You've already forked linux-apfs
mirror of
https://github.com/linux-apfs/linux-apfs.git
synced 2026-05-01 15:00:59 -07:00
drm: Add the TTM GPU memory manager subsystem.
TTM is a GPU memory manager subsystem designed for use with GPU devices with various memory types (On-card VRAM, AGP, PCI apertures etc.). It's essentially a helper library that assists the DRM driver in creating and managing persistent buffer objects. TTM manages placement of data and CPU map setup and teardown on data movement. It can also optionally manage synchronization of data on a per-buffer-object level. TTM takes care to provide an always valid virtual user-space address to a buffer object which makes user-space sub-allocation of big buffer objects feasible. TTM uses a fine-grained per buffer-object locking scheme, taking care to release all relevant locks when waiting for the GPU. Although this implies some locking overhead, it's probably a big win for devices with multiple command submission mechanisms, since the lock contention will be minimal. TTM can be used with whatever user-space interface the driver chooses, including GEM. It's used by the upcoming Radeon KMS DRM driver and is also the GPU memory management core of various new experimental DRM drivers. Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com> Signed-off-by: Jerome Glisse <jglisse@redhat.com> Signed-off-by: Dave Airlie <airlied@redhat.com>
This commit is contained in:
Thomas Hellstrom
committed by
Dave Airlie
Dave Airlie
parent
e6c03c5b40
commit
ba4e7d973d
@@ -0,0 +1,8 @@
|
||||
#
|
||||
# Makefile for the drm device driver. This driver provides support for the
|
||||
|
||||
ccflags-y := -Iinclude/drm
|
||||
ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \
|
||||
ttm_bo_util.o ttm_bo_vm.o ttm_module.o ttm_global.o
|
||||
|
||||
obj-$(CONFIG_DRM_TTM) += ttm.o
|
||||
@@ -0,0 +1,150 @@
|
||||
/**************************************************************************
|
||||
*
|
||||
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sub license, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the
|
||||
* next paragraph) shall be included in all copies or substantial portions
|
||||
* of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
**************************************************************************/
|
||||
/*
|
||||
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
|
||||
* Keith Packard.
|
||||
*/
|
||||
|
||||
#include "ttm/ttm_module.h"
|
||||
#include "ttm/ttm_bo_driver.h"
|
||||
#ifdef TTM_HAS_AGP
|
||||
#include "ttm/ttm_placement.h"
|
||||
#include <linux/agp_backend.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/io.h>
|
||||
#include <asm/agp.h>
|
||||
|
||||
struct ttm_agp_backend {
|
||||
struct ttm_backend backend;
|
||||
struct agp_memory *mem;
|
||||
struct agp_bridge_data *bridge;
|
||||
};
|
||||
|
||||
static int ttm_agp_populate(struct ttm_backend *backend,
|
||||
unsigned long num_pages, struct page **pages,
|
||||
struct page *dummy_read_page)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be =
|
||||
container_of(backend, struct ttm_agp_backend, backend);
|
||||
struct page **cur_page, **last_page = pages + num_pages;
|
||||
struct agp_memory *mem;
|
||||
|
||||
mem = agp_allocate_memory(agp_be->bridge, num_pages, AGP_USER_MEMORY);
|
||||
if (unlikely(mem == NULL))
|
||||
return -ENOMEM;
|
||||
|
||||
mem->page_count = 0;
|
||||
for (cur_page = pages; cur_page < last_page; ++cur_page) {
|
||||
struct page *page = *cur_page;
|
||||
if (!page)
|
||||
page = dummy_read_page;
|
||||
|
||||
mem->memory[mem->page_count++] =
|
||||
phys_to_gart(page_to_phys(page));
|
||||
}
|
||||
agp_be->mem = mem;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be =
|
||||
container_of(backend, struct ttm_agp_backend, backend);
|
||||
struct agp_memory *mem = agp_be->mem;
|
||||
int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
|
||||
int ret;
|
||||
|
||||
mem->is_flushed = 1;
|
||||
mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
|
||||
|
||||
ret = agp_bind_memory(mem, bo_mem->mm_node->start);
|
||||
if (ret)
|
||||
printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int ttm_agp_unbind(struct ttm_backend *backend)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be =
|
||||
container_of(backend, struct ttm_agp_backend, backend);
|
||||
|
||||
if (agp_be->mem->is_bound)
|
||||
return agp_unbind_memory(agp_be->mem);
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ttm_agp_clear(struct ttm_backend *backend)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be =
|
||||
container_of(backend, struct ttm_agp_backend, backend);
|
||||
struct agp_memory *mem = agp_be->mem;
|
||||
|
||||
if (mem) {
|
||||
ttm_agp_unbind(backend);
|
||||
agp_free_memory(mem);
|
||||
}
|
||||
agp_be->mem = NULL;
|
||||
}
|
||||
|
||||
static void ttm_agp_destroy(struct ttm_backend *backend)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be =
|
||||
container_of(backend, struct ttm_agp_backend, backend);
|
||||
|
||||
if (agp_be->mem)
|
||||
ttm_agp_clear(backend);
|
||||
kfree(agp_be);
|
||||
}
|
||||
|
||||
static struct ttm_backend_func ttm_agp_func = {
|
||||
.populate = ttm_agp_populate,
|
||||
.clear = ttm_agp_clear,
|
||||
.bind = ttm_agp_bind,
|
||||
.unbind = ttm_agp_unbind,
|
||||
.destroy = ttm_agp_destroy,
|
||||
};
|
||||
|
||||
struct ttm_backend *ttm_agp_backend_init(struct ttm_bo_device *bdev,
|
||||
struct agp_bridge_data *bridge)
|
||||
{
|
||||
struct ttm_agp_backend *agp_be;
|
||||
|
||||
agp_be = kmalloc(sizeof(*agp_be), GFP_KERNEL);
|
||||
if (!agp_be)
|
||||
return NULL;
|
||||
|
||||
agp_be->mem = NULL;
|
||||
agp_be->bridge = bridge;
|
||||
agp_be->backend.func = &ttm_agp_func;
|
||||
agp_be->backend.bdev = bdev;
|
||||
return &agp_be->backend;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_agp_backend_init);
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,454 @@
|
||||
/**************************************************************************
|
||||
*
|
||||
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sub license, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the
|
||||
* next paragraph) shall be included in all copies or substantial portions
|
||||
* of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
**************************************************************************/
|
||||
/*
|
||||
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
|
||||
*/
|
||||
|
||||
#include <ttm/ttm_module.h>
|
||||
#include <ttm/ttm_bo_driver.h>
|
||||
#include <ttm/ttm_placement.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/version.h>
|
||||
#include <linux/rbtree.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#define TTM_BO_VM_NUM_PREFAULT 16
|
||||
|
||||
static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
|
||||
unsigned long page_start,
|
||||
unsigned long num_pages)
|
||||
{
|
||||
struct rb_node *cur = bdev->addr_space_rb.rb_node;
|
||||
unsigned long cur_offset;
|
||||
struct ttm_buffer_object *bo;
|
||||
struct ttm_buffer_object *best_bo = NULL;
|
||||
|
||||
while (likely(cur != NULL)) {
|
||||
bo = rb_entry(cur, struct ttm_buffer_object, vm_rb);
|
||||
cur_offset = bo->vm_node->start;
|
||||
if (page_start >= cur_offset) {
|
||||
cur = cur->rb_right;
|
||||
best_bo = bo;
|
||||
if (page_start == cur_offset)
|
||||
break;
|
||||
} else
|
||||
cur = cur->rb_left;
|
||||
}
|
||||
|
||||
if (unlikely(best_bo == NULL))
|
||||
return NULL;
|
||||
|
||||
if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
|
||||
(page_start + num_pages)))
|
||||
return NULL;
|
||||
|
||||
return best_bo;
|
||||
}
|
||||
|
||||
static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
|
||||
{
|
||||
struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
|
||||
vma->vm_private_data;
|
||||
struct ttm_bo_device *bdev = bo->bdev;
|
||||
unsigned long bus_base;
|
||||
unsigned long bus_offset;
|
||||
unsigned long bus_size;
|
||||
unsigned long page_offset;
|
||||
unsigned long page_last;
|
||||
unsigned long pfn;
|
||||
struct ttm_tt *ttm = NULL;
|
||||
struct page *page;
|
||||
int ret;
|
||||
int i;
|
||||
bool is_iomem;
|
||||
unsigned long address = (unsigned long)vmf->virtual_address;
|
||||
int retval = VM_FAULT_NOPAGE;
|
||||
|
||||
/*
|
||||
* Work around locking order reversal in fault / nopfn
|
||||
* between mmap_sem and bo_reserve: Perform a trylock operation
|
||||
* for reserve, and if it fails, retry the fault after scheduling.
|
||||
*/
|
||||
|
||||
ret = ttm_bo_reserve(bo, true, true, false, 0);
|
||||
if (unlikely(ret != 0)) {
|
||||
if (ret == -EBUSY)
|
||||
set_need_resched();
|
||||
return VM_FAULT_NOPAGE;
|
||||
}
|
||||
|
||||
/*
|
||||
* Wait for buffer data in transit, due to a pipelined
|
||||
* move.
|
||||
*/
|
||||
|
||||
spin_lock(&bo->lock);
|
||||
if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
|
||||
ret = ttm_bo_wait(bo, false, true, false);
|
||||
spin_unlock(&bo->lock);
|
||||
if (unlikely(ret != 0)) {
|
||||
retval = (ret != -ERESTART) ?
|
||||
VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
|
||||
goto out_unlock;
|
||||
}
|
||||
} else
|
||||
spin_unlock(&bo->lock);
|
||||
|
||||
|
||||
ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset,
|
||||
&bus_size);
|
||||
if (unlikely(ret != 0)) {
|
||||
retval = VM_FAULT_SIGBUS;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
is_iomem = (bus_size != 0);
|
||||
|
||||
page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
|
||||
bo->vm_node->start - vma->vm_pgoff;
|
||||
page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) +
|
||||
bo->vm_node->start - vma->vm_pgoff;
|
||||
|
||||
if (unlikely(page_offset >= bo->num_pages)) {
|
||||
retval = VM_FAULT_SIGBUS;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
/*
|
||||
* Strictly, we're not allowed to modify vma->vm_page_prot here,
|
||||
* since the mmap_sem is only held in read mode. However, we
|
||||
* modify only the caching bits of vma->vm_page_prot and
|
||||
* consider those bits protected by
|
||||
* the bo->mutex, as we should be the only writers.
|
||||
* There shouldn't really be any readers of these bits except
|
||||
* within vm_insert_mixed()? fork?
|
||||
*
|
||||
* TODO: Add a list of vmas to the bo, and change the
|
||||
* vma->vm_page_prot when the object changes caching policy, with
|
||||
* the correct locks held.
|
||||
*/
|
||||
|
||||
if (is_iomem) {
|
||||
vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
|
||||
vma->vm_page_prot);
|
||||
} else {
|
||||
ttm = bo->ttm;
|
||||
vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
|
||||
vm_get_page_prot(vma->vm_flags) :
|
||||
ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
|
||||
}
|
||||
|
||||
/*
|
||||
* Speculatively prefault a number of pages. Only error on
|
||||
* first page.
|
||||
*/
|
||||
|
||||
for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
|
||||
|
||||
if (is_iomem)
|
||||
pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) +
|
||||
page_offset;
|
||||
else {
|
||||
page = ttm_tt_get_page(ttm, page_offset);
|
||||
if (unlikely(!page && i == 0)) {
|
||||
retval = VM_FAULT_OOM;
|
||||
goto out_unlock;
|
||||
} else if (unlikely(!page)) {
|
||||
break;
|
||||
}
|
||||
pfn = page_to_pfn(page);
|
||||
}
|
||||
|
||||
ret = vm_insert_mixed(vma, address, pfn);
|
||||
/*
|
||||
* Somebody beat us to this PTE or prefaulting to
|
||||
* an already populated PTE, or prefaulting error.
|
||||
*/
|
||||
|
||||
if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
|
||||
break;
|
||||
else if (unlikely(ret != 0)) {
|
||||
retval =
|
||||
(ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
|
||||
goto out_unlock;
|
||||
|
||||
}
|
||||
|
||||
address += PAGE_SIZE;
|
||||
if (unlikely(++page_offset >= page_last))
|
||||
break;
|
||||
}
|
||||
|
||||
out_unlock:
|
||||
ttm_bo_unreserve(bo);
|
||||
return retval;
|
||||
}
|
||||
|
||||
static void ttm_bo_vm_open(struct vm_area_struct *vma)
|
||||
{
|
||||
struct ttm_buffer_object *bo =
|
||||
(struct ttm_buffer_object *)vma->vm_private_data;
|
||||
|
||||
(void)ttm_bo_reference(bo);
|
||||
}
|
||||
|
||||
static void ttm_bo_vm_close(struct vm_area_struct *vma)
|
||||
{
|
||||
struct ttm_buffer_object *bo =
|
||||
(struct ttm_buffer_object *)vma->vm_private_data;
|
||||
|
||||
ttm_bo_unref(&bo);
|
||||
vma->vm_private_data = NULL;
|
||||
}
|
||||
|
||||
static struct vm_operations_struct ttm_bo_vm_ops = {
|
||||
.fault = ttm_bo_vm_fault,
|
||||
.open = ttm_bo_vm_open,
|
||||
.close = ttm_bo_vm_close
|
||||
};
|
||||
|
||||
int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
|
||||
struct ttm_bo_device *bdev)
|
||||
{
|
||||
struct ttm_bo_driver *driver;
|
||||
struct ttm_buffer_object *bo;
|
||||
int ret;
|
||||
|
||||
read_lock(&bdev->vm_lock);
|
||||
bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff,
|
||||
(vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
|
||||
if (likely(bo != NULL))
|
||||
ttm_bo_reference(bo);
|
||||
read_unlock(&bdev->vm_lock);
|
||||
|
||||
if (unlikely(bo == NULL)) {
|
||||
printk(KERN_ERR TTM_PFX
|
||||
"Could not find buffer object to map.\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
driver = bo->bdev->driver;
|
||||
if (unlikely(!driver->verify_access)) {
|
||||
ret = -EPERM;
|
||||
goto out_unref;
|
||||
}
|
||||
ret = driver->verify_access(bo, filp);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_unref;
|
||||
|
||||
vma->vm_ops = &ttm_bo_vm_ops;
|
||||
|
||||
/*
|
||||
* Note: We're transferring the bo reference to
|
||||
* vma->vm_private_data here.
|
||||
*/
|
||||
|
||||
vma->vm_private_data = bo;
|
||||
vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
|
||||
return 0;
|
||||
out_unref:
|
||||
ttm_bo_unref(&bo);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_bo_mmap);
|
||||
|
||||
int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
|
||||
{
|
||||
if (vma->vm_pgoff != 0)
|
||||
return -EACCES;
|
||||
|
||||
vma->vm_ops = &ttm_bo_vm_ops;
|
||||
vma->vm_private_data = ttm_bo_reference(bo);
|
||||
vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_fbdev_mmap);
|
||||
|
||||
|
||||
ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
|
||||
const char __user *wbuf, char __user *rbuf, size_t count,
|
||||
loff_t *f_pos, bool write)
|
||||
{
|
||||
struct ttm_buffer_object *bo;
|
||||
struct ttm_bo_driver *driver;
|
||||
struct ttm_bo_kmap_obj map;
|
||||
unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
|
||||
unsigned long kmap_offset;
|
||||
unsigned long kmap_end;
|
||||
unsigned long kmap_num;
|
||||
size_t io_size;
|
||||
unsigned int page_offset;
|
||||
char *virtual;
|
||||
int ret;
|
||||
bool no_wait = false;
|
||||
bool dummy;
|
||||
|
||||
read_lock(&bdev->vm_lock);
|
||||
bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
|
||||
if (likely(bo != NULL))
|
||||
ttm_bo_reference(bo);
|
||||
read_unlock(&bdev->vm_lock);
|
||||
|
||||
if (unlikely(bo == NULL))
|
||||
return -EFAULT;
|
||||
|
||||
driver = bo->bdev->driver;
|
||||
if (unlikely(driver->verify_access)) {
|
||||
ret = -EPERM;
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
ret = driver->verify_access(bo, filp);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_unref;
|
||||
|
||||
kmap_offset = dev_offset - bo->vm_node->start;
|
||||
if (unlikely(kmap_offset) >= bo->num_pages) {
|
||||
ret = -EFBIG;
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
page_offset = *f_pos & ~PAGE_MASK;
|
||||
io_size = bo->num_pages - kmap_offset;
|
||||
io_size = (io_size << PAGE_SHIFT) - page_offset;
|
||||
if (count < io_size)
|
||||
io_size = count;
|
||||
|
||||
kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
|
||||
kmap_num = kmap_end - kmap_offset + 1;
|
||||
|
||||
ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
|
||||
|
||||
switch (ret) {
|
||||
case 0:
|
||||
break;
|
||||
case -ERESTART:
|
||||
ret = -EINTR;
|
||||
goto out_unref;
|
||||
case -EBUSY:
|
||||
ret = -EAGAIN;
|
||||
goto out_unref;
|
||||
default:
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
|
||||
if (unlikely(ret != 0)) {
|
||||
ttm_bo_unreserve(bo);
|
||||
goto out_unref;
|
||||
}
|
||||
|
||||
virtual = ttm_kmap_obj_virtual(&map, &dummy);
|
||||
virtual += page_offset;
|
||||
|
||||
if (write)
|
||||
ret = copy_from_user(virtual, wbuf, io_size);
|
||||
else
|
||||
ret = copy_to_user(rbuf, virtual, io_size);
|
||||
|
||||
ttm_bo_kunmap(&map);
|
||||
ttm_bo_unreserve(bo);
|
||||
ttm_bo_unref(&bo);
|
||||
|
||||
if (unlikely(ret != 0))
|
||||
return -EFBIG;
|
||||
|
||||
*f_pos += io_size;
|
||||
|
||||
return io_size;
|
||||
out_unref:
|
||||
ttm_bo_unref(&bo);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
|
||||
char __user *rbuf, size_t count, loff_t *f_pos,
|
||||
bool write)
|
||||
{
|
||||
struct ttm_bo_kmap_obj map;
|
||||
unsigned long kmap_offset;
|
||||
unsigned long kmap_end;
|
||||
unsigned long kmap_num;
|
||||
size_t io_size;
|
||||
unsigned int page_offset;
|
||||
char *virtual;
|
||||
int ret;
|
||||
bool no_wait = false;
|
||||
bool dummy;
|
||||
|
||||
kmap_offset = (*f_pos >> PAGE_SHIFT);
|
||||
if (unlikely(kmap_offset) >= bo->num_pages)
|
||||
return -EFBIG;
|
||||
|
||||
page_offset = *f_pos & ~PAGE_MASK;
|
||||
io_size = bo->num_pages - kmap_offset;
|
||||
io_size = (io_size << PAGE_SHIFT) - page_offset;
|
||||
if (count < io_size)
|
||||
io_size = count;
|
||||
|
||||
kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
|
||||
kmap_num = kmap_end - kmap_offset + 1;
|
||||
|
||||
ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
|
||||
|
||||
switch (ret) {
|
||||
case 0:
|
||||
break;
|
||||
case -ERESTART:
|
||||
return -EINTR;
|
||||
case -EBUSY:
|
||||
return -EAGAIN;
|
||||
default:
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
|
||||
if (unlikely(ret != 0)) {
|
||||
ttm_bo_unreserve(bo);
|
||||
return ret;
|
||||
}
|
||||
|
||||
virtual = ttm_kmap_obj_virtual(&map, &dummy);
|
||||
virtual += page_offset;
|
||||
|
||||
if (write)
|
||||
ret = copy_from_user(virtual, wbuf, io_size);
|
||||
else
|
||||
ret = copy_to_user(rbuf, virtual, io_size);
|
||||
|
||||
ttm_bo_kunmap(&map);
|
||||
ttm_bo_unreserve(bo);
|
||||
ttm_bo_unref(&bo);
|
||||
|
||||
if (unlikely(ret != 0))
|
||||
return ret;
|
||||
|
||||
*f_pos += io_size;
|
||||
|
||||
return io_size;
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**************************************************************************
|
||||
*
|
||||
* Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sub license, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the
|
||||
* next paragraph) shall be included in all copies or substantial portions
|
||||
* of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
**************************************************************************/
|
||||
/*
|
||||
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
|
||||
*/
|
||||
|
||||
#include "ttm/ttm_module.h"
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
struct ttm_global_item {
|
||||
struct mutex mutex;
|
||||
void *object;
|
||||
int refcount;
|
||||
};
|
||||
|
||||
static struct ttm_global_item glob[TTM_GLOBAL_NUM];
|
||||
|
||||
void ttm_global_init(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < TTM_GLOBAL_NUM; ++i) {
|
||||
struct ttm_global_item *item = &glob[i];
|
||||
mutex_init(&item->mutex);
|
||||
item->object = NULL;
|
||||
item->refcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void ttm_global_release(void)
|
||||
{
|
||||
int i;
|
||||
for (i = 0; i < TTM_GLOBAL_NUM; ++i) {
|
||||
struct ttm_global_item *item = &glob[i];
|
||||
BUG_ON(item->object != NULL);
|
||||
BUG_ON(item->refcount != 0);
|
||||
}
|
||||
}
|
||||
|
||||
int ttm_global_item_ref(struct ttm_global_reference *ref)
|
||||
{
|
||||
int ret;
|
||||
struct ttm_global_item *item = &glob[ref->global_type];
|
||||
void *object;
|
||||
|
||||
mutex_lock(&item->mutex);
|
||||
if (item->refcount == 0) {
|
||||
item->object = kmalloc(ref->size, GFP_KERNEL);
|
||||
if (unlikely(item->object == NULL)) {
|
||||
ret = -ENOMEM;
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
ref->object = item->object;
|
||||
ret = ref->init(ref);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_err;
|
||||
|
||||
++item->refcount;
|
||||
}
|
||||
ref->object = item->object;
|
||||
object = item->object;
|
||||
mutex_unlock(&item->mutex);
|
||||
return 0;
|
||||
out_err:
|
||||
kfree(item->object);
|
||||
mutex_unlock(&item->mutex);
|
||||
item->object = NULL;
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_global_item_ref);
|
||||
|
||||
void ttm_global_item_unref(struct ttm_global_reference *ref)
|
||||
{
|
||||
struct ttm_global_item *item = &glob[ref->global_type];
|
||||
|
||||
mutex_lock(&item->mutex);
|
||||
BUG_ON(item->refcount == 0);
|
||||
BUG_ON(ref->object != item->object);
|
||||
if (--item->refcount == 0) {
|
||||
ref->release(ref);
|
||||
kfree(item->object);
|
||||
item->object = NULL;
|
||||
}
|
||||
mutex_unlock(&item->mutex);
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_global_item_unref);
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
/**************************************************************************
|
||||
*
|
||||
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sub license, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the
|
||||
* next paragraph) shall be included in all copies or substantial portions
|
||||
* of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
**************************************************************************/
|
||||
|
||||
#include "ttm/ttm_memory.h"
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
#define TTM_PFX "[TTM] "
|
||||
#define TTM_MEMORY_ALLOC_RETRIES 4
|
||||
|
||||
/**
|
||||
* At this point we only support a single shrink callback.
|
||||
* Extend this if needed, perhaps using a linked list of callbacks.
|
||||
* Note that this function is reentrant:
|
||||
* many threads may try to swap out at any given time.
|
||||
*/
|
||||
|
||||
static void ttm_shrink(struct ttm_mem_global *glob, bool from_workqueue,
|
||||
uint64_t extra)
|
||||
{
|
||||
int ret;
|
||||
struct ttm_mem_shrink *shrink;
|
||||
uint64_t target;
|
||||
uint64_t total_target;
|
||||
|
||||
spin_lock(&glob->lock);
|
||||
if (glob->shrink == NULL)
|
||||
goto out;
|
||||
|
||||
if (from_workqueue) {
|
||||
target = glob->swap_limit;
|
||||
total_target = glob->total_memory_swap_limit;
|
||||
} else if (capable(CAP_SYS_ADMIN)) {
|
||||
total_target = glob->emer_total_memory;
|
||||
target = glob->emer_memory;
|
||||
} else {
|
||||
total_target = glob->max_total_memory;
|
||||
target = glob->max_memory;
|
||||
}
|
||||
|
||||
total_target = (extra >= total_target) ? 0 : total_target - extra;
|
||||
target = (extra >= target) ? 0 : target - extra;
|
||||
|
||||
while (glob->used_memory > target ||
|
||||
glob->used_total_memory > total_target) {
|
||||
shrink = glob->shrink;
|
||||
spin_unlock(&glob->lock);
|
||||
ret = shrink->do_shrink(shrink);
|
||||
spin_lock(&glob->lock);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
}
|
||||
out:
|
||||
spin_unlock(&glob->lock);
|
||||
}
|
||||
|
||||
static void ttm_shrink_work(struct work_struct *work)
|
||||
{
|
||||
struct ttm_mem_global *glob =
|
||||
container_of(work, struct ttm_mem_global, work);
|
||||
|
||||
ttm_shrink(glob, true, 0ULL);
|
||||
}
|
||||
|
||||
int ttm_mem_global_init(struct ttm_mem_global *glob)
|
||||
{
|
||||
struct sysinfo si;
|
||||
uint64_t mem;
|
||||
|
||||
spin_lock_init(&glob->lock);
|
||||
glob->swap_queue = create_singlethread_workqueue("ttm_swap");
|
||||
INIT_WORK(&glob->work, ttm_shrink_work);
|
||||
init_waitqueue_head(&glob->queue);
|
||||
|
||||
si_meminfo(&si);
|
||||
|
||||
mem = si.totalram - si.totalhigh;
|
||||
mem *= si.mem_unit;
|
||||
|
||||
glob->max_memory = mem >> 1;
|
||||
glob->emer_memory = (mem >> 1) + (mem >> 2);
|
||||
glob->swap_limit = glob->max_memory - (mem >> 3);
|
||||
glob->used_memory = 0;
|
||||
glob->used_total_memory = 0;
|
||||
glob->shrink = NULL;
|
||||
|
||||
mem = si.totalram;
|
||||
mem *= si.mem_unit;
|
||||
|
||||
glob->max_total_memory = mem >> 1;
|
||||
glob->emer_total_memory = (mem >> 1) + (mem >> 2);
|
||||
|
||||
glob->total_memory_swap_limit = glob->max_total_memory - (mem >> 3);
|
||||
|
||||
printk(KERN_INFO TTM_PFX "TTM available graphics memory: %llu MiB\n",
|
||||
glob->max_total_memory >> 20);
|
||||
printk(KERN_INFO TTM_PFX "TTM available object memory: %llu MiB\n",
|
||||
glob->max_memory >> 20);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_mem_global_init);
|
||||
|
||||
void ttm_mem_global_release(struct ttm_mem_global *glob)
|
||||
{
|
||||
printk(KERN_INFO TTM_PFX "Used total memory is %llu bytes.\n",
|
||||
(unsigned long long)glob->used_total_memory);
|
||||
flush_workqueue(glob->swap_queue);
|
||||
destroy_workqueue(glob->swap_queue);
|
||||
glob->swap_queue = NULL;
|
||||
}
|
||||
EXPORT_SYMBOL(ttm_mem_global_release);
|
||||
|
||||
static inline void ttm_check_swapping(struct ttm_mem_global *glob)
|
||||
{
|
||||
bool needs_swapping;
|
||||
|
||||
spin_lock(&glob->lock);
|
||||
needs_swapping = (glob->used_memory > glob->swap_limit ||
|
||||
glob->used_total_memory >
|
||||
glob->total_memory_swap_limit);
|
||||
spin_unlock(&glob->lock);
|
||||
|
||||
if (unlikely(needs_swapping))
|
||||
(void)queue_work(glob->swap_queue, &glob->work);
|
||||
|
||||
}
|
||||
|
||||
void ttm_mem_global_free(struct ttm_mem_global *glob,
|
||||
uint64_t amount, bool himem)
|
||||
{
|
||||
spin_lock(&glob->lock);
|
||||
glob->used_total_memory -= amount;
|
||||
if (!himem)
|
||||
glob->used_memory -= amount;
|
||||
wake_up_all(&glob->queue);
|
||||
spin_unlock(&glob->lock);
|
||||
}
|
||||
|
||||
static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
|
||||
uint64_t amount, bool himem, bool reserve)
|
||||
{
|
||||
uint64_t limit;
|
||||
uint64_t lomem_limit;
|
||||
int ret = -ENOMEM;
|
||||
|
||||
spin_lock(&glob->lock);
|
||||
|
||||
if (capable(CAP_SYS_ADMIN)) {
|
||||
limit = glob->emer_total_memory;
|
||||
lomem_limit = glob->emer_memory;
|
||||
} else {
|
||||
limit = glob->max_total_memory;
|
||||
lomem_limit = glob->max_memory;
|
||||
}
|
||||
|
||||
if (unlikely(glob->used_total_memory + amount > limit))
|
||||
goto out_unlock;
|
||||
if (unlikely(!himem && glob->used_memory + amount > lomem_limit))
|
||||
goto out_unlock;
|
||||
|
||||
if (reserve) {
|
||||
glob->used_total_memory += amount;
|
||||
if (!himem)
|
||||
glob->used_memory += amount;
|
||||
}
|
||||
ret = 0;
|
||||
out_unlock:
|
||||
spin_unlock(&glob->lock);
|
||||
ttm_check_swapping(glob);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
|
||||
bool no_wait, bool interruptible, bool himem)
|
||||
{
|
||||
int count = TTM_MEMORY_ALLOC_RETRIES;
|
||||
|
||||
while (unlikely(ttm_mem_global_reserve(glob, memory, himem, true)
|
||||
!= 0)) {
|
||||
if (no_wait)
|
||||
return -ENOMEM;
|
||||
if (unlikely(count-- == 0))
|
||||
return -ENOMEM;
|
||||
ttm_shrink(glob, false, memory + (memory >> 2) + 16);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t ttm_round_pot(size_t size)
|
||||
{
|
||||
if ((size & (size - 1)) == 0)
|
||||
return size;
|
||||
else if (size > PAGE_SIZE)
|
||||
return PAGE_ALIGN(size);
|
||||
else {
|
||||
size_t tmp_size = 4;
|
||||
|
||||
while (tmp_size < size)
|
||||
tmp_size <<= 1;
|
||||
|
||||
return tmp_size;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/**************************************************************************
|
||||
*
|
||||
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sub license, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the
|
||||
* next paragraph) shall be included in all copies or substantial portions
|
||||
* of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*
|
||||
**************************************************************************/
|
||||
/*
|
||||
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
|
||||
* Jerome Glisse
|
||||
*/
|
||||
#include <linux/module.h>
|
||||
#include <ttm/ttm_module.h>
|
||||
|
||||
static int __init ttm_init(void)
|
||||
{
|
||||
ttm_global_init();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __exit ttm_exit(void)
|
||||
{
|
||||
ttm_global_release();
|
||||
}
|
||||
|
||||
module_init(ttm_init);
|
||||
module_exit(ttm_exit);
|
||||
|
||||
MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse");
|
||||
MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)");
|
||||
MODULE_LICENSE("GPL and additional rights");
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user