Merge remote-tracking branch 'remotes/ericb/tags/pull-nbd-2018-06-20-v2' into staging

nbd patches for 2018-06-20

Add experimental x-nbd-server-add-bitmap to expose a disabled
bitmap over NBD, in preparation for a pull model incremental
backup scheme. Also fix a corner case protocol issue with
NBD_CMD_BLOCK_STATUS, and add new NBD_CMD_CACHE.

- Eric Blake: tests: Simplify .gitignore
- Eric Blake: nbd/server: Reject 0-length block status request
- Vladimir Sementsov-Ogievskiy: 0/6 NBD export bitmaps
- Vladimir Sementsov-Ogievskiy: nbd/server: introduce NBD_CMD_CACHE

# gpg: Signature made Thu 21 Jun 2018 15:53:55 BST
# gpg:                using RSA key A7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>"
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>"
# gpg:                 aka "[jpeg image of size 6874]"
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-nbd-2018-06-20-v2:
  nbd/server: introduce NBD_CMD_CACHE
  docs/interop: add nbd.txt
  qapi: new qmp command nbd-server-add-bitmap
  nbd/server: implement dirty bitmap export
  nbd/server: add nbd_meta_empty_or_pattern helper
  nbd/server: refactor NBDExportMetaContexts
  nbd/server: fix trace
  nbd/server: Reject 0-length block status request
  tests: Simplify .gitignore

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

MAINTAINERS

@@ -1972,6 +1972,7 @@ F: nbd/
 F: include/block/nbd*
 F: qemu-nbd.*
 F: blockdev-nbd.c
+F: docs/interop/nbd.txt
 T: git git://repo.or.cz/qemu/ericb.git nbd
 
 NFS

blockdev-nbd.c

@@ -220,3 +220,26 @@ void qmp_nbd_server_stop(Error **errp)
     nbd_server_free(nbd_server);
     nbd_server = NULL;
 }
+
+void qmp_x_nbd_server_add_bitmap(const char *name, const char *bitmap,
+                                 bool has_bitmap_export_name,
+                                 const char *bitmap_export_name,
+                                 Error **errp)
+{
+    NBDExport *exp;
+
+    if (!nbd_server) {
+        error_setg(errp, "NBD server not running");
+        return;
+    }
+
+    exp = nbd_export_find(name);
+    if (exp == NULL) {
+        error_setg(errp, "Export '%s' is not found", name);
+        return;
+    }
+
+    nbd_export_bitmap(exp, bitmap,
+                      has_bitmap_export_name ? bitmap_export_name : bitmap,
+                      errp);
+}

docs/interop/nbd.txt

@@ -0,0 +1,38 @@
+Qemu supports the NBD protocol, and has an internal NBD client (see
+block/nbd.c), an internal NBD server (see blockdev-nbd.c), and an
+external NBD server tool (see qemu-nbd.c). The common code is placed
+in nbd/*.
+
+The NBD protocol is specified here:
+https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
+
+The following paragraphs describe some specific properties of NBD
+protocol realization in Qemu.
+
+= Metadata namespaces =
+
+Qemu supports the "base:allocation" metadata context as defined in the
+NBD protocol specification, and also defines an additional metadata
+namespace "qemu".
+
+
+== "qemu" namespace ==
+
+The "qemu" namespace currently contains only one type of context,
+related to exposing the contents of a dirty bitmap alongside the
+associated disk contents.  That context has the following form:
+
+    qemu:dirty-bitmap:<dirty-bitmap-export-name>
+
+Each dirty-bitmap metadata context defines only one flag for extents
+in reply for NBD_CMD_BLOCK_STATUS:
+
+    bit 0: NBD_STATE_DIRTY, means that the extent is "dirty"
+
+For NBD_OPT_LIST_META_CONTEXT the following queries are supported
+in addition to "qemu:dirty-bitmap:<dirty-bitmap-export-name>":
+
+* "qemu:" - returns list of all available metadata contexts in the
+            namespace.
+* "qemu:dirty-bitmap:" - returns list of all available dirty-bitmap
+                         metadata contexts.

include/block/nbd.h

@@ -135,6 +135,7 @@ typedef struct NBDExtent {
 #define NBD_FLAG_SEND_TRIM         (1 << 5) /* Send TRIM (discard) */
 #define NBD_FLAG_SEND_WRITE_ZEROES (1 << 6) /* Send WRITE_ZEROES */
 #define NBD_FLAG_SEND_DF           (1 << 7) /* Send DF (Do not Fragment) */
+#define NBD_FLAG_SEND_CACHE        (1 << 8) /* Send CACHE (prefetch) */
 
 /* New-style handshake (global) flags, sent from server to client, and
    control what will happen during handshake phase. */
@@ -195,7 +196,7 @@ enum {
     NBD_CMD_DISC = 2,
     NBD_CMD_FLUSH = 3,
     NBD_CMD_TRIM = 4,
-    /* 5 reserved for failed experiment NBD_CMD_CACHE */
+    NBD_CMD_CACHE = 5,
     NBD_CMD_WRITE_ZEROES = 6,
     NBD_CMD_BLOCK_STATUS = 7,
 };
@@ -229,11 +230,13 @@ enum {
 #define NBD_REPLY_TYPE_ERROR         NBD_REPLY_ERR(1)
 #define NBD_REPLY_TYPE_ERROR_OFFSET  NBD_REPLY_ERR(2)
 
-/* Flags for extents (NBDExtent.flags) of NBD_REPLY_TYPE_BLOCK_STATUS,
- * for base:allocation meta context */
+/* Extent flags for base:allocation in NBD_REPLY_TYPE_BLOCK_STATUS */
 #define NBD_STATE_HOLE (1 << 0)
 #define NBD_STATE_ZERO (1 << 1)
 
+/* Extent flags for qemu:dirty-bitmap in NBD_REPLY_TYPE_BLOCK_STATUS */
+#define NBD_STATE_DIRTY (1 << 0)
+
 static inline bool nbd_reply_type_is_error(int type)
 {
     return type & (1 << 15);
@@ -315,6 +318,8 @@ void nbd_client_put(NBDClient *client);
 void nbd_server_start(SocketAddress *addr, const char *tls_creds,
                       Error **errp);
 
+void nbd_export_bitmap(NBDExport *exp, const char *bitmap,
+                       const char *bitmap_export_name, Error **errp);
 
 /* nbd_read
  * Reads @size bytes from @ioc. Returns 0 on success.

nbd/common.c

@@ -148,6 +148,8 @@ const char *nbd_cmd_lookup(uint16_t cmd)
         return "flush";
     case NBD_CMD_TRIM:
         return "trim";
+    case NBD_CMD_CACHE:
+        return "cache";
     case NBD_CMD_WRITE_ZEROES:
         return "write zeroes";
     case NBD_CMD_BLOCK_STATUS:

nbd/trace-events

@@ -64,6 +64,7 @@ nbd_co_send_simple_reply(uint64_t handle, uint32_t error, const char *errname, i
 nbd_co_send_structured_done(uint64_t handle) "Send structured reply done: handle = %" PRIu64
 nbd_co_send_structured_read(uint64_t handle, uint64_t offset, void *data, size_t size) "Send structured read data reply: handle = %" PRIu64 ", offset = %" PRIu64 ", data = %p, len = %zu"
 nbd_co_send_structured_read_hole(uint64_t handle, uint64_t offset, size_t size) "Send structured read hole reply: handle = %" PRIu64 ", offset = %" PRIu64 ", len = %zu"
+nbd_co_send_extents(uint64_t handle, unsigned int extents, uint32_t id, uint64_t length, int last) "Send block status reply: handle = %" PRIu64 ", extents = %u, context = %d (extents cover %" PRIu64 " bytes, last chunk = %d)"
 nbd_co_send_structured_error(uint64_t handle, int err, const char *errname, const char *msg) "Send structured error reply: handle = %" PRIu64 ", error = %d (%s), msg = '%s'"
 nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *name) "Decoding type: handle = %" PRIu64 ", type = %" PRIu16 " (%s)"
 nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32

qapi/block.json

@@ -268,6 +268,29 @@
 { 'command': 'nbd-server-remove',
   'data': {'name': 'str', '*mode': 'NbdServerRemoveMode'} }
 
+##
+# @x-nbd-server-add-bitmap:
+#
+# Expose a dirty bitmap associated with the selected export. The bitmap search
+# starts at the device attached to the export, and includes all backing files.
+# The exported bitmap is then locked until the NBD export is removed.
+#
+# @name: Export name.
+#
+# @bitmap: Bitmap name to search for.
+#
+# @bitmap-export-name: How the bitmap will be seen by nbd clients
+#                      (default @bitmap)
+#
+# Note: the client must use NBD_OPT_SET_META_CONTEXT with a query of
+# "qemu:dirty-bitmap:NAME" (where NAME matches @bitmap-export-name) to access
+# the exposed bitmap.
+#
+# Since: 3.0
+##
+  { 'command': 'x-nbd-server-add-bitmap',
+    'data': {'name': 'str', 'bitmap': 'str', '*bitmap-export-name': 'str'} }
+
 ##
 # @nbd-server-stop:
 #

tests/.gitignore

@@ -2,101 +2,18 @@ atomic_add-bench
 benchmark-crypto-cipher
 benchmark-crypto-hash
 benchmark-crypto-hmac
-check-qdict
-check-qnum
-check-qjson
-check-qlist
-check-qlit
-check-qnull
-check-qobject
-check-qstring
-check-qom-interface
-check-qom-proplist
+check-*
+!check-*.c
+!check-*.sh
 qht-bench
 rcutorture
-test-aio
-test-aio-multithread
-test-arm-mptimer
-test-base64
-test-bdrv-drain
-test-bitops
-test-bitcnt
-test-block-backend
-test-blockjob
-test-blockjob-txn
-test-bufferiszero
-test-char
-test-clone-visitor
-test-coroutine
-test-crypto-afsplit
-test-crypto-block
-test-crypto-cipher
-test-crypto-hash
-test-crypto-hmac
-test-crypto-ivgen
-test-crypto-pbkdf
-test-crypto-secret
-test-crypto-tlscredsx509
-test-crypto-tlscredsx509-work/
-test-crypto-tlscredsx509-certs/
-test-crypto-tlssession
-test-crypto-tlssession-work/
-test-crypto-tlssession-client/
-test-crypto-tlssession-server/
-test-crypto-xts
-test-cutils
-test-hbitmap
-test-hmp
-test-int128
-test-iov
-test-io-channel-buffer
-test-io-channel-command
-test-io-channel-command.fifo
-test-io-channel-file
-test-io-channel-file.txt
-test-io-channel-socket
-test-io-channel-tls
-test-io-task
-test-keyval
-test-logging
-test-mul64
-test-opts-visitor
+test-*
+!test-*.c
 test-qapi-commands.[ch]
 test-qapi-events.[ch]
 test-qapi-types.[ch]
-test-qapi-util
 test-qapi-visit.[ch]
-test-qdev-global-props
-test-qemu-opts
-test-qdist
-test-qga
-test-qht
-test-qht-par
-test-qmp-cmds
-test-qmp-event
-test-qobject-input-strict
-test-qobject-input-visitor
 test-qapi-introspect.[ch]
-test-qobject-output-visitor
-test-rcu-list
-test-replication
-test-shift128
-test-string-input-visitor
-test-string-output-visitor
-test-thread-pool
-test-throttle
-test-timed-average
-test-uuid
-test-util-sockets
-test-visitor-serialization
-test-vmstate
-test-write-threshold
-test-x86-cpuid
-test-x86-cpuid-compat
-test-xbzrle
-test-netfilter
-test-filter-mirror
-test-filter-redirector
 *-test
 qapi-schema/*.test.*
 vm/*.img