izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/juanquintela/tags/migration-20211031-pull-request' into staging

Browse Source 
Migration Pull request

Hi

this includes pending bits of migration patches.

- virtio-mem support by David Hildenbrand
- dirtyrate improvements by Hyman Huang
- fix rdma wrid by Li Zhijian
- dump-guest-memory fixes by Peter Xu

Pleas apply.

Thanks, Juan.

# gpg: Signature made Mon 01 Nov 2021 06:03:44 PM EDT
# gpg:                using RSA key 1899FF8EDEBF58CCEE034B82F487EF185872D723
# gpg: Good signature from "Juan Quintela <quintela@redhat.com>" [full]
# gpg:                 aka "Juan Quintela <quintela@trasno.org>" [full]

* remotes/juanquintela/tags/migration-20211031-pull-request:
  migration/dirtyrate: implement dirty-bitmap dirtyrate calculation
  memory: introduce total_dirty_pages to stat dirty pages
  migration/ram: Handle RAMBlocks with a RamDiscardManager on background snapshots
  migration/ram: Factor out populating pages readable in ram_block_populate_pages()
  migration: Simplify alignment and alignment checks
  migration/postcopy: Handle RAMBlocks with a RamDiscardManager on the destination
  virtio-mem: Drop precopy notifier
  migration/ram: Handle RAMBlocks with a RamDiscardManager on the migration source
  virtio-mem: Implement replay_discarded RamDiscardManager callback
  memory: Introduce replay_discarded callback for RamDiscardManager
  dump-guest-memory: Block live migration
  migration: Add migrate_add_blocker_internal()
  migration: Make migration blocker work for snapshots too
  migration/dirtyrate: implement dirty-ring dirtyrate calculation
  migration/dirtyrate: move init step of calculation to main thread
  migration/dirtyrate: adjust order of registering thread
  migration/dirtyrate: introduce struct and adjust DirtyRateStat
  memory: make global_dirty_tracking a bitmask
  KVM: introduce dirty_pages and kvm_dirty_ring_enabled
  migration/rdma: Fix out of order wrid

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
2021-11-02 10:07:27 -04:00
parent f79bb385c0 826b8bc80c
commit 91e8394415
23 changed files with 908 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
accel/kvm/kvm-all.c
+7
View File
@@ -469,6 +469,7 @@ int kvm_init_vcpu(CPUState *cpu, Error **errp)
cpu->kvm_fd = ret;
cpu->kvm_state = s;
cpu->vcpu_dirty = true;
cpu->dirty_pages = 0;
mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
if (mmap_size < 0) {
@@ -743,6 +744,7 @@ static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu)
count++;
}
cpu->kvm_fetch_index = fetch;
cpu->dirty_pages += count;
return count;
}
@@ -2296,6 +2298,11 @@ bool kvm_vcpu_id_is_valid(int vcpu_id)
return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
}
bool kvm_dirty_ring_enabled(void)
{
return kvm_state->kvm_dirty_ring_size ? true : false;
}
static int kvm_init(MachineState *ms)
{
MachineClass *mc = MACHINE_GET_CLASS(ms);
accel/stubs/kvm-stub.c
+5
View File
@@ -147,4 +147,9 @@ bool kvm_arm_supports_user_irq(void)
{
return false;
}
bool kvm_dirty_ring_enabled(void)
{
return false;
}
#endif
dump/dump.c
+19
View File
@@ -29,6 +29,7 @@
#include "qemu/error-report.h"
#include "qemu/main-loop.h"
#include "hw/misc/vmcoreinfo.h"
#include "migration/blocker.h"
#ifdef TARGET_X86_64
#include "win_dump.h"
@@ -47,6 +48,8 @@
#define MAX_GUEST_NOTE_SIZE (1 << 20) /* 1MB should be enough */
static Error *dump_migration_blocker;
#define ELF_NOTE_SIZE(hdr_size, name_size, desc_size) \
((DIV_ROUND_UP((hdr_size), 4) + \
DIV_ROUND_UP((name_size), 4) + \
@@ -101,6 +104,7 @@ static int dump_cleanup(DumpState *s)
qemu_mutex_unlock_iothread();
}
}
migrate_del_blocker(dump_migration_blocker);
return 0;
}
@@ -2005,6 +2009,21 @@ void qmp_dump_guest_memory(bool paging, const char *file,
return;
}
if (!dump_migration_blocker) {
error_setg(&dump_migration_blocker,
"Live migration disabled: dump-guest-memory in progress");
}
/*
* Allows even for -only-migratable, but forbid migration during the
* process of dump guest memory.
*/
if (migrate_add_blocker_internal(dump_migration_blocker, errp)) {
/* Remember to release the fd before passing it over to dump state */
close(fd);
return;
}
s = &dump_state_global;
dump_state_prepare(s);
hmp-commands.hx
+5 -3
View File
@@ -1737,8 +1737,10 @@ ERST
{
.name = "calc_dirty_rate",
.args_type = "second:l,sample_pages_per_GB:l?",
.params = "second [sample_pages_per_GB]",
.help = "start a round of guest dirty rate measurement",
.args_type = "dirty_ring:-r,dirty_bitmap:-b,second:l,sample_pages_per_GB:l?",
.params = "[-r] [-b] second [sample_pages_per_GB]",
.help = "start a round of guest dirty rate measurement (using -r to"
"\n\t\t\t specify dirty ring as the method of calculation and"
"\n\t\t\t -b to specify dirty bitmap as method of calculation)",
.cmd = hmp_calc_dirty_rate,
},
hw/i386/xen/xen-hvm.c
+2 -2
View File
@@ -1613,8 +1613,8 @@ void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t length)
void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
{
if (enable) {
memory_global_dirty_log_start();
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
} else {
memory_global_dirty_log_stop();
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
}
}
hw/virtio/virtio-mem.c
+58 -34
View File
@@ -228,6 +228,38 @@ static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem,
return ret;
}
static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem,
MemoryRegionSection *s,
void *arg,
virtio_mem_section_cb cb)
{
unsigned long first_bit, last_bit;
uint64_t offset, size;
int ret = 0;
first_bit = s->offset_within_region / vmem->bitmap_size;
first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit);
while (first_bit < vmem->bitmap_size) {
MemoryRegionSection tmp = *s;
offset = first_bit * vmem->block_size;
last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
first_bit + 1) - 1;
size = (last_bit - first_bit + 1) * vmem->block_size;
if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
break;
}
ret = cb(&tmp, arg);
if (ret) {
break;
}
first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
last_bit + 2);
}
return ret;
}
static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg)
{
RamDiscardListener *rdl = arg;
@@ -744,7 +776,6 @@ static void virtio_mem_device_realize(DeviceState *dev, Error **errp)
host_memory_backend_set_mapped(vmem->memdev, true);
vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem));
qemu_register_reset(virtio_mem_system_reset, vmem);
precopy_add_notifier(&vmem->precopy_notifier);
/*
* Set ourselves as RamDiscardManager before the plug handler maps the
@@ -764,7 +795,6 @@ static void virtio_mem_device_unrealize(DeviceState *dev)
* found via an address space anymore. Unset ourselves.
*/
memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL);
precopy_remove_notifier(&vmem->precopy_notifier);
qemu_unregister_reset(virtio_mem_system_reset, vmem);
vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem));
host_memory_backend_set_mapped(vmem->memdev, false);
@@ -1057,43 +1087,11 @@ static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name,
vmem->block_size = value;
}
static int virtio_mem_precopy_exclude_range_cb(const VirtIOMEM *vmem, void *arg,
uint64_t offset, uint64_t size)
{
void * const host = qemu_ram_get_host_addr(vmem->memdev->mr.ram_block);
qemu_guest_free_page_hint(host + offset, size);
return 0;
}
static void virtio_mem_precopy_exclude_unplugged(VirtIOMEM *vmem)
{
virtio_mem_for_each_unplugged_range(vmem, NULL,
virtio_mem_precopy_exclude_range_cb);
}
static int virtio_mem_precopy_notify(NotifierWithReturn *n, void *data)
{
VirtIOMEM *vmem = container_of(n, VirtIOMEM, precopy_notifier);
PrecopyNotifyData *pnd = data;
switch (pnd->reason) {
case PRECOPY_NOTIFY_AFTER_BITMAP_SYNC:
virtio_mem_precopy_exclude_unplugged(vmem);
break;
default:
break;
}
return 0;
}
static void virtio_mem_instance_init(Object *obj)
{
VirtIOMEM *vmem = VIRTIO_MEM(obj);
notifier_list_init(&vmem->size_change_notifiers);
vmem->precopy_notifier.notify = virtio_mem_precopy_notify;
QLIST_INIT(&vmem->rdl_list);
object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size,
@@ -1170,6 +1168,31 @@ static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm,
virtio_mem_rdm_replay_populated_cb);
}
static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s,
void *arg)
{
struct VirtIOMEMReplayData *data = arg;
((ReplayRamDiscard)data->fn)(s, data->opaque);
return 0;
}
static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm,
MemoryRegionSection *s,
ReplayRamDiscard replay_fn,
void *opaque)
{
const VirtIOMEM *vmem = VIRTIO_MEM(rdm);
struct VirtIOMEMReplayData data = {
.fn = replay_fn,
.opaque = opaque,
};
g_assert(s->mr == &vmem->memdev->mr);
virtio_mem_for_each_unplugged_section(vmem, s, &data,
virtio_mem_rdm_replay_discarded_cb);
}
static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm,
RamDiscardListener *rdl,
MemoryRegionSection *s)
@@ -1234,6 +1257,7 @@ static void virtio_mem_class_init(ObjectClass *klass, void *data)
rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity;
rdmc->is_populated = virtio_mem_rdm_is_populated;
rdmc->replay_populated = virtio_mem_rdm_replay_populated;
rdmc->replay_discarded = virtio_mem_rdm_replay_discarded;
rdmc->register_listener = virtio_mem_rdm_register_listener;
rdmc->unregister_listener = virtio_mem_rdm_unregister_listener;
}
include/exec/memory.h
+38 -3
View File
@@ -61,7 +61,17 @@ static inline void fuzz_dma_read_cb(size_t addr,
}
#endif
extern bool global_dirty_log;
/* Possible bits for global_dirty_log_{start|stop} */
/* Dirty tracking enabled because migration is running */
#define GLOBAL_DIRTY_MIGRATION (1U << 0)
/* Dirty tracking enabled because measuring dirty rate */
#define GLOBAL_DIRTY_DIRTY_RATE (1U << 1)
#define GLOBAL_DIRTY_MASK (0x3)
extern unsigned int global_dirty_tracking;
typedef struct MemoryRegionOps MemoryRegionOps;
@@ -540,6 +550,7 @@ static inline void ram_discard_listener_init(RamDiscardListener *rdl,
}
typedef int (*ReplayRamPopulate)(MemoryRegionSection *section, void *opaque);
typedef void (*ReplayRamDiscard)(MemoryRegionSection *section, void *opaque);
/*
* RamDiscardManagerClass:
@@ -628,6 +639,21 @@ struct RamDiscardManagerClass {
MemoryRegionSection *section,
ReplayRamPopulate replay_fn, void *opaque);
/**
* @replay_discarded:
*
* Call the #ReplayRamDiscard callback for all discarded parts within the
* #MemoryRegionSection via the #RamDiscardManager.
*
* @rdm: the #RamDiscardManager
* @section: the #MemoryRegionSection
* @replay_fn: the #ReplayRamDiscard callback
* @opaque: pointer to forward to the callback
*/
void (*replay_discarded)(const RamDiscardManager *rdm,
MemoryRegionSection *section,
ReplayRamDiscard replay_fn, void *opaque);
/**
* @register_listener:
*
@@ -672,6 +698,11 @@ int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
ReplayRamPopulate replay_fn,
void *opaque);
void ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
MemoryRegionSection *section,
ReplayRamDiscard replay_fn,
void *opaque);
void ram_discard_manager_register_listener(RamDiscardManager *rdm,
RamDiscardListener *rdl,
MemoryRegionSection *section);
@@ -2388,13 +2419,17 @@ void memory_listener_unregister(MemoryListener *listener);
/**
* memory_global_dirty_log_start: begin dirty logging for all regions
*
* @flags: purpose of starting dirty log, migration or dirty rate
*/
void memory_global_dirty_log_start(void);
void memory_global_dirty_log_start(unsigned int flags);
/**
* memory_global_dirty_log_stop: end dirty logging for all regions
*
* @flags: purpose of stopping dirty log, migration or dirty rate
*/
void memory_global_dirty_log_stop(void);
void memory_global_dirty_log_stop(unsigned int flags);
void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled);
include/exec/ram_addr.h
+11 -2
View File
@@ -26,6 +26,8 @@
#include "exec/ramlist.h"
#include "exec/ramblock.h"
extern uint64_t total_dirty_pages;
/**
* clear_bmap_size: calculate clear bitmap size
*
@@ -369,10 +371,14 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
if (global_dirty_log) {
if (global_dirty_tracking) {
qatomic_or(
&blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
temp);
if (unlikely(
global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
total_dirty_pages += ctpopl(temp);
}
}
if (tcg_enabled()) {
@@ -392,7 +398,7 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
} else {
uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
if (!global_dirty_log) {
if (!global_dirty_tracking) {
clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
}
@@ -403,6 +409,9 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
for (i = 0; i < len; i++) {
if (bitmap[i] != 0) {
c = leul_to_cpu(bitmap[i]);
if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
total_dirty_pages += ctpopl(c);
}
do {
j = ctzl(c);
c &= ~(1ul << j);
include/hw/core/cpu.h
+1
View File
@@ -381,6 +381,7 @@ struct CPUState {
struct kvm_run *kvm_run;
struct kvm_dirty_gfn *kvm_dirty_gfns;
uint32_t kvm_fetch_index;
uint64_t dirty_pages;
/* Used for events with 'vcpu' and *without* the 'disabled' properties */
DECLARE_BITMAP(trace_dstate_delayed, CPU_TRACE_DSTATE_MAX_EVENTS);
include/hw/virtio/virtio-mem.h
-3
View File
@@ -65,9 +65,6 @@ struct VirtIOMEM {
/* notifiers to notify when "size" changes */
NotifierList size_change_notifiers;
/* don't migrate unplugged memory */
NotifierWithReturn precopy_notifier;
/* listeners to notify on plug/unplug activity. */
QLIST_HEAD(, RamDiscardListener) rdl_list;
};
include/migration/blocker.h
+16
View File
@@ -25,6 +25,22 @@
*/
int migrate_add_blocker(Error *reason, Error **errp);
/**
* @migrate_add_blocker_internal - prevent migration from proceeding without
* only-migrate implications
*
* @reason - an error to be returned whenever migration is attempted
*
* @errp - [out] The reason (if any) we cannot block migration right now.
*
* @returns - 0 on success, -EBUSY on failure, with errp set.
*
* Some of the migration blockers can be temporary (e.g., for a few seconds),
* so it shouldn't need to conflict with "-only-migratable". For those cases,
* we can call this function rather than @migrate_add_blocker().
*/
int migrate_add_blocker_internal(Error *reason, Error **errp);
/**
* @migrate_del_blocker - remove a blocking error from migration
*
include/sysemu/kvm.h
+1
View File
@@ -547,4 +547,5 @@ bool kvm_cpu_check_are_resettable(void);
bool kvm_arch_cpu_check_are_resettable(void);
bool kvm_dirty_ring_enabled(void);
#endif
migration/dirtyrate.c
+333 -34
View File
File diff suppressed because it is too large Load Diff
migration/dirtyrate.h
+16 -3
View File
@@ -43,6 +43,7 @@
struct DirtyRateConfig {
uint64_t sample_pages_per_gigabytes; /* sample pages per GB */
int64_t sample_period_seconds; /* time duration between two sampling */
DirtyRateMeasureMode mode; /* mode of dirtyrate measurement */
};
/*
@@ -58,17 +59,29 @@ struct RamblockDirtyInfo {
uint32_t *hash_result; /* array of hash result for sampled pages */
};
typedef struct SampleVMStat {
uint64_t total_dirty_samples; /* total dirty sampled page */
uint64_t total_sample_count; /* total sampled pages */
uint64_t total_block_mem_MB; /* size of total sampled pages in MB */
} SampleVMStat;
typedef struct VcpuStat {
int nvcpu; /* number of vcpu */
DirtyRateVcpu *rates; /* array of dirty rate for each vcpu */
} VcpuStat;
/*
* Store calculation statistics for each measure.
*/
struct DirtyRateStat {
uint64_t total_dirty_samples; /* total dirty sampled page */
uint64_t total_sample_count; /* total sampled pages */
uint64_t total_block_mem_MB; /* size of total sampled pages in MB */
int64_t dirty_rate; /* dirty rate in MB/s */
int64_t start_time; /* calculation start time in units of second */
int64_t calc_time; /* time duration of two sampling in units of second */
uint64_t sample_pages; /* sample pages per GB */
union {
SampleVMStat page_sampling;
VcpuStat dirty_ring;
};
};
void *get_dirtyrate_thread(void *arg);
migration/migration.c
+18 -12
View File
@@ -391,7 +391,7 @@ int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
int migrate_send_rp_req_pages(MigrationIncomingState *mis,
RAMBlock *rb, ram_addr_t start, uint64_t haddr)
{
void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb)));
void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
bool received = false;
WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
@@ -2049,6 +2049,20 @@ void migrate_init(MigrationState *s)
s->threshold_size = 0;
}
int migrate_add_blocker_internal(Error *reason, Error **errp)
{
/* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
error_propagate_prepend(errp, error_copy(reason),
"disallowing migration blocker "
"(migration/snapshot in progress) for: ");
return -EBUSY;
}
migration_blockers = g_slist_prepend(migration_blockers, reason);
return 0;
}
int migrate_add_blocker(Error *reason, Error **errp)
{
if (only_migratable) {
@@ -2058,15 +2072,7 @@ int migrate_add_blocker(Error *reason, Error **errp)
return -EACCES;
}
if (migration_is_idle()) {
migration_blockers = g_slist_prepend(migration_blockers, reason);
return 0;
}
error_propagate_prepend(errp, error_copy(reason),
"disallowing migration blocker "
"(migration in progress) for: ");
return -EBUSY;
return migrate_add_blocker_internal(reason, errp);
}
void migrate_del_blocker(Error *reason)
@@ -2631,8 +2637,8 @@ static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
* Since we currently insist on matching page sizes, just sanity check
* we're being asked for whole host pages.
*/
if (start & (our_host_ps - 1) ||
(len & (our_host_ps - 1))) {
if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
!QEMU_IS_ALIGNED(len, our_host_ps)) {
error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
" len: %zd", __func__, start, len);
mark_source_rp_bad(ms);
migration/postcopy-ram.c
+31 -9
View File
@@ -402,7 +402,7 @@ bool postcopy_ram_supported_by_host(MigrationIncomingState *mis)
strerror(errno));
goto out;
}
g_assert(((size_t)testarea & (pagesize - 1)) == 0);
g_assert(QEMU_PTR_IS_ALIGNED(testarea, pagesize));
reg_struct.range.start = (uintptr_t)testarea;
reg_struct.range.len = pagesize;
@@ -660,7 +660,7 @@ int postcopy_wake_shared(struct PostCopyFD *pcfd,
struct uffdio_range range;
int ret;
trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb));
range.start = client_addr & ~(pagesize - 1);
range.start = ROUND_DOWN(client_addr, pagesize);
range.len = pagesize;
ret = ioctl(pcfd->fd, UFFDIO_WAKE, &range);
if (ret) {
@@ -671,6 +671,29 @@ int postcopy_wake_shared(struct PostCopyFD *pcfd,
return ret;
}
static int postcopy_request_page(MigrationIncomingState *mis, RAMBlock *rb,
ram_addr_t start, uint64_t haddr)
{
void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
/*
* Discarded pages (via RamDiscardManager) are never migrated. On unlikely
* access, place a zeropage, which will also set the relevant bits in the
* recv_bitmap accordingly, so we won't try placing a zeropage twice.
*
* Checking a single bit is sufficient to handle pagesize > TPS as either
* all relevant bits are set or not.
*/
assert(QEMU_IS_ALIGNED(start, qemu_ram_pagesize(rb)));
if (ramblock_page_is_discarded(rb, start)) {
bool received = ramblock_recv_bitmap_test_byte_offset(rb, start);
return received ? 0 : postcopy_place_page_zero(mis, aligned, rb);
}
return migrate_send_rp_req_pages(mis, rb, start, haddr);
}
/*
* Callback from shared fault handlers to ask for a page,
* the page must be specified by a RAMBlock and an offset in that rb
@@ -679,8 +702,7 @@ int postcopy_wake_shared(struct PostCopyFD *pcfd,
int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb,
uint64_t client_addr, uint64_t rb_offset)
{
size_t pagesize = qemu_ram_pagesize(rb);
uint64_t aligned_rbo = rb_offset & ~(pagesize - 1);
uint64_t aligned_rbo = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb));
MigrationIncomingState *mis = migration_incoming_get_current();
trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb),
@@ -690,7 +712,7 @@ int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb,
qemu_ram_get_idstr(rb), rb_offset);
return postcopy_wake_shared(pcfd, client_addr, rb);
}
migrate_send_rp_req_pages(mis, rb, aligned_rbo, client_addr);
postcopy_request_page(mis, rb, aligned_rbo, client_addr);
return 0;
}
@@ -970,7 +992,7 @@ static void *postcopy_ram_fault_thread(void *opaque)
break;
}
rb_offset &= ~(qemu_ram_pagesize(rb) - 1);
rb_offset = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb));
trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address,
qemu_ram_get_idstr(rb),
rb_offset,
@@ -984,8 +1006,8 @@ retry:
* Send the request to the source - we want to request one
* of our host page sizes (which is >= TPS)
*/
ret = migrate_send_rp_req_pages(mis, rb, rb_offset,
msg.arg.pagefault.address);
ret = postcopy_request_page(mis, rb, rb_offset,
msg.arg.pagefault.address);
if (ret) {
/* May be network failure, try to wait for recovery */
if (ret == -EIO && postcopy_pause_fault_thread(mis)) {
@@ -993,7 +1015,7 @@ retry:
goto retry;
} else {
/* This is a unavoidable fault */
error_report("%s: migrate_send_rp_req_pages() get %d",
error_report("%s: postcopy_request_page() get %d",
__func__, ret);
break;
}
migration/ram.c
+164 -16
View File
@@ -811,7 +811,7 @@ static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
assert(shift >= 6);
size = 1ULL << (TARGET_PAGE_BITS + shift);
start = (((ram_addr_t)page) << TARGET_PAGE_BITS) & (-size);
start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
memory_region_clear_dirty_bitmap(rb->mr, start, size);
}
@@ -858,6 +858,81 @@ static inline bool migration_bitmap_clear_dirty(RAMState *rs,
return ret;
}
static void dirty_bitmap_clear_section(MemoryRegionSection *section,
void *opaque)
{
const hwaddr offset = section->offset_within_region;
const hwaddr size = int128_get64(section->size);
const unsigned long start = offset >> TARGET_PAGE_BITS;
const unsigned long npages = size >> TARGET_PAGE_BITS;
RAMBlock *rb = section->mr->ram_block;
uint64_t *cleared_bits = opaque;
/*
* We don't grab ram_state->bitmap_mutex because we expect to run
* only when starting migration or during postcopy recovery where
* we don't have concurrent access.
*/
if (!migration_in_postcopy() && !migrate_background_snapshot()) {
migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
}
*cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
bitmap_clear(rb->bmap, start, npages);
}
/*
* Exclude all dirty pages from migration that fall into a discarded range as
* managed by a RamDiscardManager responsible for the mapped memory region of
* the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
*
* Discarded pages ("logically unplugged") have undefined content and must
* not get migrated, because even reading these pages for migration might
* result in undesired behavior.
*
* Returns the number of cleared bits in the RAMBlock dirty bitmap.
*
* Note: The result is only stable while migrating (precopy/postcopy).
*/
static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
{
uint64_t cleared_bits = 0;
if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
MemoryRegionSection section = {
.mr = rb->mr,
.offset_within_region = 0,
.size = int128_make64(qemu_ram_get_used_length(rb)),
};
ram_discard_manager_replay_discarded(rdm, &section,
dirty_bitmap_clear_section,
&cleared_bits);
}
return cleared_bits;
}
/*
* Check if a host-page aligned page falls into a discarded range as managed by
* a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
*
* Note: The result is only stable while migrating (precopy/postcopy).
*/
bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
{
if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
MemoryRegionSection section = {
.mr = rb->mr,
.offset_within_region = start,
.size = int128_make64(qemu_ram_pagesize(rb)),
};
return !ram_discard_manager_is_populated(rdm, &section);
}
return false;
}
/* Called with RCU critical section */
static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
{
@@ -1564,25 +1639,68 @@ out:
return ret;
}
static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
ram_addr_t size)
{
/*
* We read one byte of each page; this will preallocate page tables if
* required and populate the shared zeropage on MAP_PRIVATE anonymous memory
* where no page was populated yet. This might require adaption when
* supporting other mappings, like shmem.
*/
for (; offset < size; offset += block->page_size) {
char tmp = *((char *)block->host + offset);
/* Don't optimize the read out */
asm volatile("" : "+r" (tmp));
}
}
static inline int populate_read_section(MemoryRegionSection *section,
void *opaque)
{
const hwaddr size = int128_get64(section->size);
hwaddr offset = section->offset_within_region;
RAMBlock *block = section->mr->ram_block;
populate_read_range(block, offset, size);
return 0;
}
/*
* ram_block_populate_pages: populate memory in the RAM block by reading
* an integer from the beginning of each page.
* ram_block_populate_read: preallocate page tables and populate pages in the
* RAM block by reading a byte of each page.
*
* Since it's solely used for userfault_fd WP feature, here we just
* hardcode page size to qemu_real_host_page_size.
*
* @block: RAM block to populate
*/
static void ram_block_populate_pages(RAMBlock *block)
static void ram_block_populate_read(RAMBlock *rb)
{
char *ptr = (char *) block->host;
/*
* Skip populating all pages that fall into a discarded range as managed by
* a RamDiscardManager responsible for the mapped memory region of the
* RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
* must not get populated automatically. We don't have to track
* modifications via userfaultfd WP reliably, because these pages will
* not be part of the migration stream either way -- see
* ramblock_dirty_bitmap_exclude_discarded_pages().
*
* Note: The result is only stable while migrating (precopy/postcopy).
*/
if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
MemoryRegionSection section = {
.mr = rb->mr,
.offset_within_region = 0,
.size = rb->mr->size,
};
for (ram_addr_t offset = 0; offset < block->used_length;
offset += qemu_real_host_page_size) {
char tmp = *(ptr + offset);
/* Don't optimize the read out */
asm volatile("" : "+r" (tmp));
ram_discard_manager_replay_populated(rdm, &section,
populate_read_section, NULL);
} else {
populate_read_range(rb, 0, rb->used_length);
}
}
@@ -1609,7 +1727,7 @@ void ram_write_tracking_prepare(void)
* UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
* pages with pte_none() entries in page table.
*/
ram_block_populate_pages(block);
ram_block_populate_read(block);
}
}
@@ -2216,7 +2334,14 @@ static void ram_save_cleanup(void *opaque)
/* caller have hold iothread lock or is in a bh, so there is
* no writing race against the migration bitmap
*/
memory_global_dirty_log_stop();
if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
/*
* do not stop dirty log without starting it, since
* memory_global_dirty_log_stop will assert that
* memory_global_dirty_log_start/stop used in pairs
*/
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
}
}
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
@@ -2668,6 +2793,19 @@ static void ram_list_init_bitmaps(void)
}
}
static void migration_bitmap_clear_discarded_pages(RAMState *rs)
{
unsigned long pages;
RAMBlock *rb;
RCU_READ_LOCK_GUARD();
RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
rs->migration_dirty_pages -= pages;
}
}
static void ram_init_bitmaps(RAMState *rs)
{
/* For memory_global_dirty_log_start below. */
@@ -2678,12 +2816,18 @@ static void ram_init_bitmaps(RAMState *rs)
ram_list_init_bitmaps();
/* We don't use dirty log with background snapshots */
if (!migrate_background_snapshot()) {
memory_global_dirty_log_start();
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
migration_bitmap_sync_precopy(rs);
}
}
qemu_mutex_unlock_ramlist();
qemu_mutex_unlock_iothread();
/*
* After an eventual first bitmap sync, fixup the initial bitmap
* containing all 1s to exclude any discarded pages from migration.
*/
migration_bitmap_clear_discarded_pages(rs);
}
static int ram_init_all(RAMState **rsp)
@@ -3434,7 +3578,7 @@ void colo_incoming_start_dirty_log(void)
/* Discard this dirty bitmap record */
bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
}
memory_global_dirty_log_start();
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
}
ram_state->migration_dirty_pages = 0;
qemu_mutex_unlock_ramlist();
@@ -3446,7 +3590,7 @@ void colo_release_ram_cache(void)
{
RAMBlock *block;
memory_global_dirty_log_stop();
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
g_free(block->bmap);
block->bmap = NULL;
@@ -4112,6 +4256,10 @@ int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
*/
bitmap_complement(block->bmap, block->bmap, nbits);
/* Clear dirty bits of discarded ranges that we don't want to migrate. */
ramblock_dirty_bitmap_clear_discarded_pages(block);
/* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
trace_ram_dirty_bitmap_reload_complete(block->idstr);
/*
migration/ram.h
+1
View File
@@ -72,6 +72,7 @@ void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr, size_t nr);
int64_t ramblock_recv_bitmap_send(QEMUFile *file,
const char *block_name);
int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb);
bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start);
/* ram cache */
int colo_init_ram_cache(void);
migration/rdma.c
+101 -37
View File
@@ -358,9 +358,11 @@ typedef struct RDMAContext {
struct ibv_context *verbs;
struct rdma_event_channel *channel;
struct ibv_qp *qp; /* queue pair */
struct ibv_comp_channel *comp_channel; /* completion channel */
struct ibv_comp_channel *recv_comp_channel; /* recv completion channel */
struct ibv_comp_channel *send_comp_channel; /* send completion channel */
struct ibv_pd *pd; /* protection domain */
struct ibv_cq *cq; /* completion queue */
struct ibv_cq *recv_cq; /* recvieve completion queue */
struct ibv_cq *send_cq; /* send completion queue */
/*
* If a previous write failed (perhaps because of a failed
@@ -1059,21 +1061,34 @@ static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
return -1;
}
/* create completion channel */
rdma->comp_channel = ibv_create_comp_channel(rdma->verbs);
if (!rdma->comp_channel) {
error_report("failed to allocate completion channel");
/* create receive completion channel */
rdma->recv_comp_channel = ibv_create_comp_channel(rdma->verbs);
if (!rdma->recv_comp_channel) {
error_report("failed to allocate receive completion channel");
goto err_alloc_pd_cq;
}
/*
* Completion queue can be filled by both read and write work requests,
* so must reflect the sum of both possible queue sizes.
* Completion queue can be filled by read work requests.
*/
rdma->cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
NULL, rdma->comp_channel, 0);
if (!rdma->cq) {
error_report("failed to allocate completion queue");
rdma->recv_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
NULL, rdma->recv_comp_channel, 0);
if (!rdma->recv_cq) {
error_report("failed to allocate receive completion queue");
goto err_alloc_pd_cq;
}
/* create send completion channel */
rdma->send_comp_channel = ibv_create_comp_channel(rdma->verbs);
if (!rdma->send_comp_channel) {
error_report("failed to allocate send completion channel");
goto err_alloc_pd_cq;
}
rdma->send_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
NULL, rdma->send_comp_channel, 0);
if (!rdma->send_cq) {
error_report("failed to allocate send completion queue");
goto err_alloc_pd_cq;
}
@@ -1083,11 +1098,19 @@ err_alloc_pd_cq:
if (rdma->pd) {
ibv_dealloc_pd(rdma->pd);
}
if (rdma->comp_channel) {
ibv_destroy_comp_channel(rdma->comp_channel);
if (rdma->recv_comp_channel) {
ibv_destroy_comp_channel(rdma->recv_comp_channel);
}
if (rdma->send_comp_channel) {
ibv_destroy_comp_channel(rdma->send_comp_channel);
}
if (rdma->recv_cq) {
ibv_destroy_cq(rdma->recv_cq);
rdma->recv_cq = NULL;
}
rdma->pd = NULL;
rdma->comp_channel = NULL;
rdma->recv_comp_channel = NULL;
rdma->send_comp_channel = NULL;
return -1;
}
@@ -1104,8 +1127,8 @@ static int qemu_rdma_alloc_qp(RDMAContext *rdma)
attr.cap.max_recv_wr = 3;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 1;
attr.send_cq = rdma->cq;
attr.recv_cq = rdma->cq;
attr.send_cq = rdma->send_cq;
attr.recv_cq = rdma->recv_cq;
attr.qp_type = IBV_QPT_RC;
ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr);
@@ -1496,14 +1519,14 @@ static void qemu_rdma_signal_unregister(RDMAContext *rdma, uint64_t index,
* (of any kind) has completed.
* Return the work request ID that completed.
*/
static uint64_t qemu_rdma_poll(RDMAContext *rdma, uint64_t *wr_id_out,
uint32_t *byte_len)
static uint64_t qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq,
uint64_t *wr_id_out, uint32_t *byte_len)
{
int ret;
struct ibv_wc wc;
uint64_t wr_id;
ret = ibv_poll_cq(rdma->cq, 1, &wc);
ret = ibv_poll_cq(cq, 1, &wc);
if (!ret) {
*wr_id_out = RDMA_WRID_NONE;
@@ -1575,7 +1598,8 @@ static uint64_t qemu_rdma_poll(RDMAContext *rdma, uint64_t *wr_id_out,
/* Wait for activity on the completion channel.
* Returns 0 on success, none-0 on error.
*/
static int qemu_rdma_wait_comp_channel(RDMAContext *rdma)
static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
struct ibv_comp_channel *comp_channel)
{
struct rdma_cm_event *cm_event;
int ret = -1;
@@ -1586,7 +1610,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma)
*/
if (rdma->migration_started_on_destination &&
migration_incoming_get_current()->state == MIGRATION_STATUS_ACTIVE) {
yield_until_fd_readable(rdma->comp_channel->fd);
yield_until_fd_readable(comp_channel->fd);
} else {
/* This is the source side, we're in a separate thread
* or destination prior to migration_fd_process_incoming()
@@ -1597,7 +1621,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma)
*/
while (!rdma->error_state && !rdma->received_error) {
GPollFD pfds[2];
pfds[0].fd = rdma->comp_channel->fd;
pfds[0].fd = comp_channel->fd;
pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR;
pfds[0].revents = 0;
@@ -1655,6 +1679,17 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma)
return rdma->error_state;
}
static struct ibv_comp_channel *to_channel(RDMAContext *rdma, int wrid)
{
return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_comp_channel :
rdma->recv_comp_channel;
}
static struct ibv_cq *to_cq(RDMAContext *rdma, int wrid)
{
return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_cq : rdma->recv_cq;
}
/*
* Block until the next work request has completed.
*
@@ -1675,13 +1710,15 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
struct ibv_cq *cq;
void *cq_ctx;
uint64_t wr_id = RDMA_WRID_NONE, wr_id_in;
struct ibv_comp_channel *ch = to_channel(rdma, wrid_requested);
struct ibv_cq *poll_cq = to_cq(rdma, wrid_requested);
if (ibv_req_notify_cq(rdma->cq, 0)) {
if (ibv_req_notify_cq(poll_cq, 0)) {
return -1;
}
/* poll cq first */
while (wr_id != wrid_requested) {
ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len);
if (ret < 0) {
return ret;
}
@@ -1702,12 +1739,12 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
}
while (1) {
ret = qemu_rdma_wait_comp_channel(rdma);
ret = qemu_rdma_wait_comp_channel(rdma, ch);
if (ret) {
goto err_block_for_wrid;
}
ret = ibv_get_cq_event(rdma->comp_channel, &cq, &cq_ctx);
ret = ibv_get_cq_event(ch, &cq, &cq_ctx);
if (ret) {
perror("ibv_get_cq_event");
goto err_block_for_wrid;
@@ -1721,7 +1758,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
}
while (wr_id != wrid_requested) {
ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len);
if (ret < 0) {
goto err_block_for_wrid;
}
@@ -2437,13 +2474,21 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
rdma_destroy_qp(rdma->cm_id);
rdma->qp = NULL;
}
if (rdma->cq) {
ibv_destroy_cq(rdma->cq);
rdma->cq = NULL;
if (rdma->recv_cq) {
ibv_destroy_cq(rdma->recv_cq);
rdma->recv_cq = NULL;
}
if (rdma->comp_channel) {
ibv_destroy_comp_channel(rdma->comp_channel);
rdma->comp_channel = NULL;
if (rdma->send_cq) {
ibv_destroy_cq(rdma->send_cq);
rdma->send_cq = NULL;
}
if (rdma->recv_comp_channel) {
ibv_destroy_comp_channel(rdma->recv_comp_channel);
rdma->recv_comp_channel = NULL;
}
if (rdma->send_comp_channel) {
ibv_destroy_comp_channel(rdma->send_comp_channel);
rdma->send_comp_channel = NULL;
}
if (rdma->pd) {
ibv_dealloc_pd(rdma->pd);
@@ -3115,10 +3160,14 @@ static void qio_channel_rdma_set_aio_fd_handler(QIOChannel *ioc,
{
QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
if (io_read) {
aio_set_fd_handler(ctx, rioc->rdmain->comp_channel->fd,
aio_set_fd_handler(ctx, rioc->rdmain->recv_comp_channel->fd,
false, io_read, io_write, NULL, opaque);
aio_set_fd_handler(ctx, rioc->rdmain->send_comp_channel->fd,
false, io_read, io_write, NULL, opaque);
} else {
aio_set_fd_handler(ctx, rioc->rdmaout->comp_channel->fd,
aio_set_fd_handler(ctx, rioc->rdmaout->recv_comp_channel->fd,
false, io_read, io_write, NULL, opaque);
aio_set_fd_handler(ctx, rioc->rdmaout->send_comp_channel->fd,
false, io_read, io_write, NULL, opaque);
}
}
@@ -3332,7 +3381,22 @@ static size_t qemu_rdma_save_page(QEMUFile *f, void *opaque,
*/
while (1) {
uint64_t wr_id, wr_id_in;
int ret = qemu_rdma_poll(rdma, &wr_id_in, NULL);
int ret = qemu_rdma_poll(rdma, rdma->recv_cq, &wr_id_in, NULL);
if (ret < 0) {
error_report("rdma migration: polling error! %d", ret);
goto err;
}
wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
if (wr_id == RDMA_WRID_NONE) {
break;
}
}
while (1) {
uint64_t wr_id, wr_id_in;
int ret = qemu_rdma_poll(rdma, rdma->send_cq, &wr_id_in, NULL);
if (ret < 0) {
error_report("rdma migration: polling error! %d", ret);
goto err;
migration/trace-events
+2
View File
@@ -333,6 +333,8 @@ get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock n
calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32
skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64
find_page_matched(const char *idstr) "ramblock %s addr or size changed"
dirtyrate_calculate(int64_t dirtyrate) "dirty rate: %" PRIi64 " MB/s"
dirtyrate_do_calculate_vcpu(int idx, uint64_t rate) "vcpu[%d]: %"PRIu64 " MB/s"
# block.c
migration_block_init_shared(const char *blk_device_name) "Start migration for %s with shared base image"

Some files were not shown because too many files have changed in this diff Show More