python: create qemu packages

move python/qemu/*.py to python/qemu/[machine, qmp, utils]/*.py and
update import directives across the tree.

This is done to create a PEP420 namespace package, in which we may
create subpackages. To do this, the namespace directory ("qemu") should
not have any modules in it. Those files will go into new 'machine',
'qmp' and 'utils' subpackages instead.

Implement machine/__init__.py making the top-level classes and functions
from its various modules available directly inside the package. Change
qmp.py to qmp/__init__.py similarly, such that all of the useful QMP
library classes are available directly from "qemu.qmp" instead of
"qemu.qmp.qmp".

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Cleber Rosa <crosa@redhat.com>
Message-id: 20210527211715.394144-10-jsnow@redhat.com
Signed-off-by: John Snow <jsnow@redhat.com>

python/qemu/machine/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+extend-ignore = E722  # Pylint handles this, but smarter.

python/qemu/machine/__init__.py

@@ -0,0 +1,33 @@
+"""
+QEMU development and testing library.
+
+This library provides a few high-level classes for driving QEMU from a
+test suite, not intended for production use.
+
+- QEMUMachine: Configure and Boot a QEMU VM
+ - QEMUQtestMachine: VM class, with a qtest socket.
+
+- QEMUQtestProtocol: Connect to, send/receive qtest messages.
+"""
+
+# Copyright (C) 2020-2021 John Snow for Red Hat Inc.
+# Copyright (C) 2015-2016 Red Hat Inc.
+# Copyright (C) 2012 IBM Corp.
+#
+# Authors:
+#  John Snow <jsnow@redhat.com>
+#  Fam Zheng <fam@euphon.net>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+
+from .machine import QEMUMachine
+from .qtest import QEMUQtestMachine, QEMUQtestProtocol
+
+
+__all__ = (
+    'QEMUMachine',
+    'QEMUQtestProtocol',
+    'QEMUQtestMachine',
+)

python/qemu/machine/console_socket.py

@@ -0,0 +1,129 @@
+"""
+QEMU Console Socket Module:
+
+This python module implements a ConsoleSocket object,
+which can drain a socket and optionally dump the bytes to file.
+"""
+# Copyright 2020 Linaro
+#
+# Authors:
+#  Robert Foley <robert.foley@linaro.org>
+#
+# This code is licensed under the GPL version 2 or later.  See
+# the COPYING file in the top-level directory.
+#
+
+from collections import deque
+import socket
+import threading
+import time
+from typing import Deque, Optional
+
+
+class ConsoleSocket(socket.socket):
+    """
+    ConsoleSocket represents a socket attached to a char device.
+
+    Optionally (if drain==True), drains the socket and places the bytes
+    into an in memory buffer for later processing.
+
+    Optionally a file path can be passed in and we will also
+    dump the characters to this file for debugging purposes.
+    """
+    def __init__(self, address: str, file: Optional[str] = None,
+                 drain: bool = False):
+        self._recv_timeout_sec = 300.0
+        self._sleep_time = 0.5
+        self._buffer: Deque[int] = deque()
+        socket.socket.__init__(self, socket.AF_UNIX, socket.SOCK_STREAM)
+        self.connect(address)
+        self._logfile = None
+        if file:
+            # pylint: disable=consider-using-with
+            self._logfile = open(file, "bw")
+        self._open = True
+        self._drain_thread = None
+        if drain:
+            self._drain_thread = self._thread_start()
+
+    def __repr__(self) -> str:
+        tmp = super().__repr__()
+        tmp = tmp.rstrip(">")
+        tmp = "%s,  logfile=%s, drain_thread=%s>" % (tmp, self._logfile,
+                                                     self._drain_thread)
+        return tmp
+
+    def _drain_fn(self) -> None:
+        """Drains the socket and runs while the socket is open."""
+        while self._open:
+            try:
+                self._drain_socket()
+            except socket.timeout:
+                # The socket is expected to timeout since we set a
+                # short timeout to allow the thread to exit when
+                # self._open is set to False.
+                time.sleep(self._sleep_time)
+
+    def _thread_start(self) -> threading.Thread:
+        """Kick off a thread to drain the socket."""
+        # Configure socket to not block and timeout.
+        # This allows our drain thread to not block
+        # on recieve and exit smoothly.
+        socket.socket.setblocking(self, False)
+        socket.socket.settimeout(self, 1)
+        drain_thread = threading.Thread(target=self._drain_fn)
+        drain_thread.daemon = True
+        drain_thread.start()
+        return drain_thread
+
+    def close(self) -> None:
+        """Close the base object and wait for the thread to terminate"""
+        if self._open:
+            self._open = False
+            if self._drain_thread is not None:
+                thread, self._drain_thread = self._drain_thread, None
+                thread.join()
+            socket.socket.close(self)
+            if self._logfile:
+                self._logfile.close()
+                self._logfile = None
+
+    def _drain_socket(self) -> None:
+        """process arriving characters into in memory _buffer"""
+        data = socket.socket.recv(self, 1)
+        if self._logfile:
+            self._logfile.write(data)
+            self._logfile.flush()
+        self._buffer.extend(data)
+
+    def recv(self, bufsize: int = 1, flags: int = 0) -> bytes:
+        """Return chars from in memory buffer.
+           Maintains the same API as socket.socket.recv.
+        """
+        if self._drain_thread is None:
+            # Not buffering the socket, pass thru to socket.
+            return socket.socket.recv(self, bufsize, flags)
+        assert not flags, "Cannot pass flags to recv() in drained mode"
+        start_time = time.time()
+        while len(self._buffer) < bufsize:
+            time.sleep(self._sleep_time)
+            elapsed_sec = time.time() - start_time
+            if elapsed_sec > self._recv_timeout_sec:
+                raise socket.timeout
+        return bytes((self._buffer.popleft() for i in range(bufsize)))
+
+    def setblocking(self, value: bool) -> None:
+        """When not draining we pass thru to the socket,
+           since when draining we control socket blocking.
+        """
+        if self._drain_thread is None:
+            socket.socket.setblocking(self, value)
+
+    def settimeout(self, value: Optional[float]) -> None:
+        """When not draining we pass thru to the socket,
+           since when draining we control the timeout.
+        """
+        if value is not None:
+            self._recv_timeout_sec = value
+        if self._drain_thread is None:
+            socket.socket.settimeout(self, value)

python/qemu/machine/pylintrc

@@ -0,0 +1,58 @@
+[MASTER]
+
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=too-many-arguments,
+        too-many-instance-attributes,
+        too-many-public-methods,
+
+[REPORTS]
+
+[REFACTORING]
+
+[MISCELLANEOUS]
+
+[LOGGING]
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           fd,
+           c,
+[VARIABLES]
+
+[STRING]
+
+[SPELLING]
+
+[FORMAT]
+
+[SIMILARITIES]
+
+# Ignore imports when computing similarities.
+ignore-imports=yes
+
+[TYPECHECK]
+
+[CLASSES]
+
+[IMPORTS]
+
+[DESIGN]
+
+[EXCEPTIONS]

python/qemu/machine/qtest.py

@@ -0,0 +1,160 @@
+"""
+QEMU qtest library
+
+qtest offers the QEMUQtestProtocol and QEMUQTestMachine classes, which
+offer a connection to QEMU's qtest protocol socket, and a qtest-enabled
+subclass of QEMUMachine, respectively.
+"""
+
+# Copyright (C) 2015 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Based on qmp.py.
+#
+
+import os
+import socket
+from typing import (
+    List,
+    Optional,
+    Sequence,
+    TextIO,
+)
+
+from qemu.qmp import SocketAddrT
+
+from .machine import QEMUMachine
+
+
+class QEMUQtestProtocol:
+    """
+    QEMUQtestProtocol implements a connection to a qtest socket.
+
+    :param address: QEMU address, can be either a unix socket path (string)
+                    or a tuple in the form ( address, port ) for a TCP
+                    connection
+    :param server: server mode, listens on the socket (bool)
+    :raise socket.error: on socket connection errors
+
+    .. note::
+       No conection is estabalished by __init__(), this is done
+       by the connect() or accept() methods.
+    """
+    def __init__(self, address: SocketAddrT,
+                 server: bool = False):
+        self._address = address
+        self._sock = self._get_sock()
+        self._sockfile: Optional[TextIO] = None
+        if server:
+            self._sock.bind(self._address)
+            self._sock.listen(1)
+
+    def _get_sock(self) -> socket.socket:
+        if isinstance(self._address, tuple):
+            family = socket.AF_INET
+        else:
+            family = socket.AF_UNIX
+        return socket.socket(family, socket.SOCK_STREAM)
+
+    def connect(self) -> None:
+        """
+        Connect to the qtest socket.
+
+        @raise socket.error on socket connection errors
+        """
+        self._sock.connect(self._address)
+        self._sockfile = self._sock.makefile(mode='r')
+
+    def accept(self) -> None:
+        """
+        Await connection from QEMU.
+
+        @raise socket.error on socket connection errors
+        """
+        self._sock, _ = self._sock.accept()
+        self._sockfile = self._sock.makefile(mode='r')
+
+    def cmd(self, qtest_cmd: str) -> str:
+        """
+        Send a qtest command on the wire.
+
+        @param qtest_cmd: qtest command text to be sent
+        """
+        assert self._sockfile is not None
+        self._sock.sendall((qtest_cmd + "\n").encode('utf-8'))
+        resp = self._sockfile.readline()
+        return resp
+
+    def close(self) -> None:
+        """
+        Close this socket.
+        """
+        self._sock.close()
+        if self._sockfile:
+            self._sockfile.close()
+            self._sockfile = None
+
+    def settimeout(self, timeout: Optional[float]) -> None:
+        """Set a timeout, in seconds."""
+        self._sock.settimeout(timeout)
+
+
+class QEMUQtestMachine(QEMUMachine):
+    """
+    A QEMU VM, with a qtest socket available.
+    """
+
+    def __init__(self,
+                 binary: str,
+                 args: Sequence[str] = (),
+                 name: Optional[str] = None,
+                 base_temp_dir: str = "/var/tmp",
+                 socket_scm_helper: Optional[str] = None,
+                 sock_dir: Optional[str] = None):
+        if name is None:
+            name = "qemu-%d" % os.getpid()
+        if sock_dir is None:
+            sock_dir = base_temp_dir
+        super().__init__(binary, args, name=name, base_temp_dir=base_temp_dir,
+                         socket_scm_helper=socket_scm_helper,
+                         sock_dir=sock_dir)
+        self._qtest: Optional[QEMUQtestProtocol] = None
+        self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock")
+
+    @property
+    def _base_args(self) -> List[str]:
+        args = super()._base_args
+        args.extend([
+            '-qtest', f"unix:path={self._qtest_path}",
+            '-accel', 'qtest'
+        ])
+        return args
+
+    def _pre_launch(self) -> None:
+        super()._pre_launch()
+        self._qtest = QEMUQtestProtocol(self._qtest_path, server=True)
+
+    def _post_launch(self) -> None:
+        assert self._qtest is not None
+        super()._post_launch()
+        self._qtest.accept()
+
+    def _post_shutdown(self) -> None:
+        super()._post_shutdown()
+        self._remove_if_exists(self._qtest_path)
+
+    def qtest(self, cmd: str) -> str:
+        """
+        Send a qtest command to the guest.
+
+        :param cmd: qtest command to send
+        :return: qtest server response
+        """
+        if self._qtest is None:
+            raise RuntimeError("qtest socket not available")
+        return self._qtest.cmd(cmd)