Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into staging

# gpg: Signature made Fri 12 Dec 2014 17:09:56 GMT using RSA key ID 81AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>"
# gpg:                 aka "Stefan Hajnoczi <stefanha@gmail.com>"

* remotes/stefanha/tags/block-pull-request:
  linux-aio: simplify removal of completed iocbs from the list
  linux-aio: drop return code from laio_io_unplug and ioq_submit
  linux-aio: rename LaioQueue idx field to "n"
  linux-aio: track whether the queue is blocked
  linux-aio: queue requests that cannot be submitted
  block: drop unused bdrv_clear_incoming_migration_all() prototype
  block: Don't add trailing space in "Formating..." message
  qemu-iotests: Remove traling whitespaces in *.out
  block: vhdx - set .bdrv_has_zero_init to bdrv_has_zero_init_1
  iotests: Fix test 039
  iotests: Filter for "Killed" in qemu-io output
  qemu-io: Add sigraise command
  block: vhdx - change .vhdx_create default block state to ZERO
  block: vhdx - update PAYLOAD_BLOCK_UNMAPPED value to match 1.00 spec
  block: vhdx - remove redundant comments
  block/rbd: fix memory leak
  iotests: Add test for vmdk JSON file names
  vmdk: Fix error for JSON descriptor file names
  block migration: fix return value

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

block-migration.c

@@ -653,6 +653,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
 {
     int ret;
     int64_t last_ftell = qemu_ftell(f);
+    int64_t delta_ftell;
 
     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
@@ -702,7 +703,14 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
     }
 
     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
-    return qemu_ftell(f) - last_ftell;
+    delta_ftell = qemu_ftell(f) - last_ftell;
+    if (delta_ftell > 0) {
+        return 1;
+    } else if (delta_ftell < 0) {
+        return -1;
+    } else {
+        return 0;
+    }
 }
 
 /* Called with iothread lock taken.  */

block.c

@@ -229,7 +229,7 @@ size_t bdrv_opt_mem_align(BlockDriverState *bs)
 }
 
 /* check if the path starts with "<protocol>:" */
-static int path_has_protocol(const char *path)
+int path_has_protocol(const char *path)
 {
     const char *p;
 
@@ -5668,8 +5668,8 @@ void bdrv_img_create(const char *filename, const char *fmt,
     }
 
     if (!quiet) {
-        printf("Formatting '%s', fmt=%s ", filename, fmt);
-        qemu_opts_print(opts);
+        printf("Formatting '%s', fmt=%s", filename, fmt);
+        qemu_opts_print(opts, " ");
         puts("");
     }
 

block/linux-aio.c

@@ -35,14 +35,14 @@ struct qemu_laiocb {
     size_t nbytes;
     QEMUIOVector *qiov;
     bool is_read;
-    QLIST_ENTRY(qemu_laiocb) node;
+    QSIMPLEQ_ENTRY(qemu_laiocb) next;
 };
 
 typedef struct {
-    struct iocb *iocbs[MAX_QUEUED_IO];
     int plugged;
-    unsigned int size;
-    unsigned int idx;
+    unsigned int n;
+    bool blocked;
+    QSIMPLEQ_HEAD(, qemu_laiocb) pending;
 } LaioQueue;
 
 struct qemu_laio_state {
@@ -59,6 +59,8 @@ struct qemu_laio_state {
     int event_max;
 };
 
+static void ioq_submit(struct qemu_laio_state *s);
+
 static inline ssize_t io_event_ret(struct io_event *ev)
 {
     return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res);
@@ -135,6 +137,10 @@ static void qemu_laio_completion_bh(void *opaque)
 
         qemu_laio_process_completion(s, laiocb);
     }
+
+    if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
+        ioq_submit(s);
+    }
 }
 
 static void qemu_laio_completion_cb(EventNotifier *e)
@@ -172,50 +178,41 @@ static const AIOCBInfo laio_aiocb_info = {
 
 static void ioq_init(LaioQueue *io_q)
 {
-    io_q->size = MAX_QUEUED_IO;
-    io_q->idx = 0;
+    QSIMPLEQ_INIT(&io_q->pending);
     io_q->plugged = 0;
+    io_q->n = 0;
+    io_q->blocked = false;
 }
 
-static int ioq_submit(struct qemu_laio_state *s)
+static void ioq_submit(struct qemu_laio_state *s)
 {
-    int ret, i = 0;
-    int len = s->io_q.idx;
+    int ret, len;
+    struct qemu_laiocb *aiocb;
+    struct iocb *iocbs[MAX_QUEUED_IO];
+    QSIMPLEQ_HEAD(, qemu_laiocb) completed;
 
     do {
-        ret = io_submit(s->ctx, len, s->io_q.iocbs);
-    } while (i++ < 3 && ret == -EAGAIN);
+        len = 0;
+        QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) {
+            iocbs[len++] = &aiocb->iocb;
+            if (len == MAX_QUEUED_IO) {
+                break;
+            }
+        }
 
-    /* empty io queue */
-    s->io_q.idx = 0;
+        ret = io_submit(s->ctx, len, iocbs);
+        if (ret == -EAGAIN) {
+            break;
+        }
+        if (ret < 0) {
+            abort();
+        }
 
-    if (ret < 0) {
-        i = 0;
-    } else {
-        i = ret;
-    }
-
-    for (; i < len; i++) {
-        struct qemu_laiocb *laiocb =
-            container_of(s->io_q.iocbs[i], struct qemu_laiocb, iocb);
-
-        laiocb->ret = (ret < 0) ? ret : -EIO;
-        qemu_laio_process_completion(s, laiocb);
-    }
-    return ret;
-}
-
-static void ioq_enqueue(struct qemu_laio_state *s, struct iocb *iocb)
-{
-    unsigned int idx = s->io_q.idx;
-
-    s->io_q.iocbs[idx++] = iocb;
-    s->io_q.idx = idx;
-
-    /* submit immediately if queue is full */
-    if (idx == s->io_q.size) {
-        ioq_submit(s);
-    }
+        s->io_q.n -= ret;
+        aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb);
+        QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed);
+    } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending));
+    s->io_q.blocked = (s->io_q.n > 0);
 }
 
 void laio_io_plug(BlockDriverState *bs, void *aio_ctx)
@@ -225,22 +222,19 @@ void laio_io_plug(BlockDriverState *bs, void *aio_ctx)
     s->io_q.plugged++;
 }
 
-int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug)
+void laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug)
 {
     struct qemu_laio_state *s = aio_ctx;
-    int ret = 0;
 
     assert(s->io_q.plugged > 0 || !unplug);
 
     if (unplug && --s->io_q.plugged > 0) {
-        return 0;
+        return;
     }
 
-    if (s->io_q.idx > 0) {
-        ret = ioq_submit(s);
+    if (!s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
+        ioq_submit(s);
     }
-
-    return ret;
 }
 
 BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
@@ -276,12 +270,11 @@ BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
     }
     io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
 
-    if (!s->io_q.plugged) {
-        if (io_submit(s->ctx, 1, &iocbs) < 0) {
-            goto out_free_aiocb;
-        }
-    } else {
-        ioq_enqueue(s, iocbs);
+    QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next);
+    s->io_q.n++;
+    if (!s->io_q.blocked &&
+        (!s->io_q.plugged || s->io_q.n >= MAX_QUEUED_IO)) {
+        ioq_submit(s);
     }
     return &laiocb->common;
 

block/raw-aio.h

@@ -41,7 +41,7 @@ BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
 void laio_detach_aio_context(void *s, AioContext *old_context);
 void laio_attach_aio_context(void *s, AioContext *new_context);
 void laio_io_plug(BlockDriverState *bs, void *aio_ctx);
-int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug);
+void laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug);
 #endif
 
 #ifdef _WIN32

block/rbd.c

@@ -459,7 +459,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
     clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
     r = rados_create(&s->cluster, clientname);
     if (r < 0) {
-        error_setg(&local_err, "error initializing");
+        error_setg(errp, "error initializing");
         goto failed_opts;
     }
 
@@ -495,19 +495,19 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
 
     r = rados_connect(s->cluster);
     if (r < 0) {
-        error_setg(&local_err, "error connecting");
+        error_setg(errp, "error connecting");
         goto failed_shutdown;
     }
 
     r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
     if (r < 0) {
-        error_setg(&local_err, "error opening pool %s", pool);
+        error_setg(errp, "error opening pool %s", pool);
         goto failed_shutdown;
     }
 
     r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
     if (r < 0) {
-        error_setg(&local_err, "error reading header from %s", s->name);
+        error_setg(errp, "error reading header from %s", s->name);
         goto failed_open;
     }
 

block/vhdx.c

@@ -1109,8 +1109,9 @@ static coroutine_fn int vhdx_co_readv(BlockDriverState *bs, int64_t sector_num,
             /* check the payload block state */
             switch (s->bat[sinfo.bat_idx] & VHDX_BAT_STATE_BIT_MASK) {
             case PAYLOAD_BLOCK_NOT_PRESENT: /* fall through */
-            case PAYLOAD_BLOCK_UNDEFINED:   /* fall through */
-            case PAYLOAD_BLOCK_UNMAPPED:    /* fall through */
+            case PAYLOAD_BLOCK_UNDEFINED:
+            case PAYLOAD_BLOCK_UNMAPPED:
+            case PAYLOAD_BLOCK_UNMAPPED_v095:
             case PAYLOAD_BLOCK_ZERO:
                 /* return zero */
                 qemu_iovec_memset(&hd_qiov, 0, 0, sinfo.bytes_avail);
@@ -1277,11 +1278,11 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,
                         sectors_to_write += iov2.iov_len >> BDRV_SECTOR_BITS;
                     }
                 }
-
                 /* fall through */
             case PAYLOAD_BLOCK_NOT_PRESENT: /* fall through */
-            case PAYLOAD_BLOCK_UNMAPPED:    /* fall through */
-            case PAYLOAD_BLOCK_UNDEFINED:   /* fall through */
+            case PAYLOAD_BLOCK_UNMAPPED:
+            case PAYLOAD_BLOCK_UNMAPPED_v095:
+            case PAYLOAD_BLOCK_UNDEFINED:
                 bat_prior_offset = sinfo.file_offset;
                 ret = vhdx_allocate_block(bs, s, &sinfo.file_offset);
                 if (ret < 0) {
@@ -1773,7 +1774,7 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
     log_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_LOG_SIZE, 0);
     block_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_BLOCK_SIZE, 0);
     type = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
-    use_zero_blocks = qemu_opt_get_bool_del(opts, VHDX_BLOCK_OPT_ZERO, false);
+    use_zero_blocks = qemu_opt_get_bool_del(opts, VHDX_BLOCK_OPT_ZERO, true);
 
     if (image_size > VHDX_MAX_IMAGE_SIZE) {
         error_setg_errno(errp, EINVAL, "Image size too large; max of 64TB");
@@ -1935,7 +1936,9 @@ static QemuOptsList vhdx_create_opts = {
        {
            .name = VHDX_BLOCK_OPT_ZERO,
            .type = QEMU_OPT_BOOL,
-           .help = "Force use of payload blocks of type 'ZERO'.  Non-standard."
+           .help = "Force use of payload blocks of type 'ZERO'. "\
+                   "Non-standard, but default.  Do not set to 'off' when "\
+                   "using 'qemu-img convert' with subformat=dynamic."
        },
        { NULL }
     }
@@ -1953,6 +1956,7 @@ static BlockDriver bdrv_vhdx = {
     .bdrv_create            = vhdx_create,
     .bdrv_get_info          = vhdx_get_info,
     .bdrv_check             = vhdx_check,
+    .bdrv_has_zero_init     = bdrv_has_zero_init_1,
 
     .create_opts            = &vhdx_create_opts,
 };

block/vhdx.h

@@ -226,7 +226,8 @@ typedef struct QEMU_PACKED VHDXLogDataSector {
 #define PAYLOAD_BLOCK_NOT_PRESENT       0
 #define PAYLOAD_BLOCK_UNDEFINED         1
 #define PAYLOAD_BLOCK_ZERO              2
-#define PAYLOAD_BLOCK_UNMAPPED          5
+#define PAYLOAD_BLOCK_UNMAPPED          3
+#define PAYLOAD_BLOCK_UNMAPPED_v095     5
 #define PAYLOAD_BLOCK_FULLY_PRESENT     6
 #define PAYLOAD_BLOCK_PARTIALLY_PRESENT 7
 

block/vmdk.c

@@ -833,6 +833,14 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             goto next_line;
         }
 
+        if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
+            !desc_file_path[0])
+        {
+            error_setg(errp, "Cannot use relative extent paths with VMDK "
+                       "descriptor file '%s'", bs->file->filename);
+            return -EINVAL;
+        }
+
         path_combine(extent_path, sizeof(extent_path),
                 desc_file_path, fname);
         extent_file = NULL;
@@ -909,7 +917,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
     }
     s->create_type = g_strdup(ct);
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->filename, errp);
+    ret = vmdk_parse_extents(buf, bs, bs->file->exact_filename, errp);
 exit:
     return ret;
 }

include/block/block.h

@@ -329,8 +329,6 @@ BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
 void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp);
 void bdrv_invalidate_cache_all(Error **errp);
 
-void bdrv_clear_incoming_migration_all(void);
-
 /* Ensure contents are flushed to disk.  */
 int bdrv_flush(BlockDriverState *bs);
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
@@ -401,6 +399,7 @@ void bdrv_get_full_backing_filename(BlockDriverState *bs,
                                     char *dest, size_t sz);
 int bdrv_is_snapshot(BlockDriverState *bs);
 
+int path_has_protocol(const char *path);
 int path_is_absolute(const char *path);
 void path_combine(char *dest, int dest_size,
                   const char *base_path,

include/qemu/option.h

@@ -124,7 +124,7 @@ QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
 void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
 
 typedef int (*qemu_opts_loopfunc)(QemuOpts *opts, void *opaque);
-void qemu_opts_print(QemuOpts *opts);
+void qemu_opts_print(QemuOpts *opts, const char *sep);
 int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func, void *opaque,
                       int abort_on_failure);
 void qemu_opts_print_help(QemuOptsList *list);

include/qemu/queue.h

@@ -268,6 +268,17 @@ struct {                                                                \
         (head)->sqh_last = &(head)->sqh_first;                          \
 } while (/*CONSTCOND*/0)
 
+#define QSIMPLEQ_SPLIT_AFTER(head, elm, field, removed) do {            \
+    QSIMPLEQ_INIT(removed);                                             \
+    if (((removed)->sqh_first = (head)->sqh_first) != NULL) {           \
+        if (((head)->sqh_first = (elm)->field.sqe_next) == NULL) {      \
+            (head)->sqh_last = &(head)->sqh_first;                      \
+        }                                                               \
+        (removed)->sqh_last = &(elm)->field.sqe_next;                   \
+        (elm)->field.sqe_next = NULL;                                   \
+    }                                                                   \
+} while (/*CONSTCOND*/0)
+
 #define QSIMPLEQ_REMOVE(head, elm, type, field) do {                    \
     if ((head)->sqh_first == (elm)) {                                   \
         QSIMPLEQ_REMOVE_HEAD((head), field);                            \

qemu-doc.texi

@@ -698,7 +698,11 @@ Supported options:
 Specifies which VHDX subformat to use. Valid options are
 @code{dynamic} (default) and @code{fixed}.
 @item block_state_zero
-Force use of payload blocks of type 'ZERO'.
+Force use of payload blocks of type 'ZERO'.  Can be set to @code{on} (default)
+or @code{off}.  When set to @code{off}, new blocks will be created as
+@code{PAYLOAD_BLOCK_NOT_PRESENT}, which means parsers are free to return
+arbitrary data for those blocks.  Do not set to @code{off} when using
+@code{qemu-img convert} with @code{subformat=dynamic}.
 @item block_size
 Block size; min 1 MB, max 256 MB.  0 means auto-calculate based on image size.
 @item log_size

qemu-io-cmds.c

@@ -2048,6 +2048,51 @@ static const cmdinfo_t abort_cmd = {
        .oneline        = "simulate a program crash using abort(3)",
 };
 
+static void sigraise_help(void)
+{
+    printf(
+"\n"
+" raises the given signal\n"
+"\n"
+" Example:\n"
+" 'sigraise %i' - raises SIGTERM\n"
+"\n"
+" Invokes raise(signal), where \"signal\" is the mandatory integer argument\n"
+" given to sigraise.\n"
+"\n", SIGTERM);
+}
+
+static int sigraise_f(BlockDriverState *bs, int argc, char **argv);
+
+static const cmdinfo_t sigraise_cmd = {
+    .name       = "sigraise",
+    .cfunc      = sigraise_f,
+    .argmin     = 1,
+    .argmax     = 1,
+    .flags      = CMD_NOFILE_OK,
+    .args       = "signal",
+    .oneline    = "raises a signal",
+    .help       = sigraise_help,
+};
+
+static int sigraise_f(BlockDriverState *bs, int argc, char **argv)
+{
+    int sig = cvtnum(argv[1]);
+    if (sig < 0) {
+        printf("non-numeric signal number argument -- %s\n", argv[1]);
+        return 0;
+    }
+
+    /* Using raise() to kill this process does not necessarily flush all open
+     * streams. At least stdout and stderr (although the latter should be
+     * non-buffered anyway) should be flushed, though. */
+    fflush(stdout);
+    fflush(stderr);
+
+    raise(sig);
+    return 0;
+}
+
 static void sleep_cb(void *opaque)
 {
     bool *expired = opaque;
@@ -2202,4 +2247,5 @@ static void __attribute((constructor)) init_qemuio_commands(void)
     qemuio_add_command(&wait_break_cmd);
     qemuio_add_command(&abort_cmd);
     qemuio_add_command(&sleep_cmd);
+    qemuio_add_command(&sigraise_cmd);
 }

tests/qemu-iotests/001.out

@@ -1,5 +1,5 @@
 QA output created by 001
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0

tests/qemu-iotests/002.out

@@ -1,5 +1,5 @@
 QA output created by 002
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0

tests/qemu-iotests/003.out

@@ -1,5 +1,5 @@
 QA output created by 003
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0

tests/qemu-iotests/004.out

@@ -1,5 +1,5 @@
 QA output created by 004
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 write before image boundary
 wrote 1048576/1048576 bytes at offset 133169152

tests/qemu-iotests/005.out

@@ -1,7 +1,7 @@
 QA output created by 005
 
 creating large image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5368709120000 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5368709120000
 
 small read
 read 4096/4096 bytes at offset 1024

tests/qemu-iotests/006.out

@@ -2,5 +2,5 @@ QA output created by 006
 
 creating 128GB image
 qemu-img: The image size is too large for file format 'vpc'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=137438953472 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=137438953472
 *** done

tests/qemu-iotests/007.out

@@ -1,7 +1,7 @@
 QA output created by 007
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
 savevm 1
 savevm 2
 savevm 3