Merge remote-tracking branch 'remotes/philmd-gitlab/tags/python-next-20200207' into staging

- Python 3 cleanups:
  . Remove text about Python 2 in qemu-deprecated (Thomas)
  . Remove shebang header (Paolo, Philippe)
  . scripts/checkpatch.pl now allows Python 3 interpreter (Philippe)
  . Explicit usage of Python 3 interpreter in scripts (Philippe)
  . Fix Python scripts permissions (Paolo, Philippe)
  . Drop 'from __future__ import print_function' (Paolo)
  . Specify minimum python requirements in ReadTheDocs configuration (Alex)
- Test UNIX/EXEC transports with migration (Oksana)
- Added extract_from_rpm helper, improved extract_from_deb (Liam)
- Allow to use other serial consoles than default one (Philippe)
- Various improvements in QEMUMonitorProtocol (Wainer)
- Fix kvm_available() on ppc64le (Wainer)

# gpg: Signature made Fri 07 Feb 2020 15:01:56 GMT
# gpg:                using RSA key FAABE75E12917221DCFD6BB2E3E32C2CDEADC0DE
# gpg: Good signature from "Philippe Mathieu-Daudé (F4BUG) <f4bug@amsat.org>" [full]
# Primary key fingerprint: FAAB E75E 1291 7221 DCFD  6BB2 E3E3 2C2C DEAD C0DE

* remotes/philmd-gitlab/tags/python-next-20200207: (46 commits)
  .readthedocs.yml: specify some minimum python requirements
  drop "from __future__ import print_function"
  make all Python scripts executable
  scripts/signrom: remove Python 2 support, add shebang
  tests/qemu-iotests/check: Only check for Python 3 interpreter
  scripts: Explicit usage of Python 3 (scripts without __main__)
  tests/qemu-iotests: Explicit usage of Python3 (scripts without __main__)
  tests/vm: Remove shebang header
  tests/acceptance: Remove shebang header
  scripts/tracetool: Remove shebang header
  scripts/minikconf: Explicit usage of Python 3
  scripts: Explicit usage of Python 3 (scripts with __main__)
  tests: Explicit usage of Python 3
  tests/qemu-iotests: Explicit usage of Python 3 (scripts with __main__)
  tests/qemu-iotests/check: Allow use of python3 interpreter
  scripts/checkpatch.pl: Only allow Python 3 interpreter
  tests/acceptance/migration: Default to -nodefaults
  tests/acceptance/migration: Add the 'migration' tag
  tests/acceptance/migration: Test EXEC transport when migrating
  tests/acceptance/migration: Test UNIX transport when migrating
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

.readthedocs.yml

@@ -0,0 +1,20 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# We want all the document formats
+formats: all
+
+# For consistency, we require that QEMU's Sphinx extensions
+# run with at least the same minimum version of Python that
+# we require for other Python in our codebase (our conf.py
+# enforces this, and some code needs it.)
+python:
+  version: 3.5

.travis.yml

@@ -313,7 +313,7 @@ matrix:
     # Acceptance (Functional) tests
     - name: "GCC check-acceptance"
       env:
-        - CONFIG="--target-list=x86_64-softmmu,mips-softmmu,mips64el-softmmu,aarch64-softmmu,arm-softmmu,s390x-softmmu,alpha-softmmu,ppc-softmmu,ppc64-softmmu,m68k-softmmu,sparc-softmmu"
+        - CONFIG="--target-list=aarch64-softmmu,alpha-softmmu,arm-softmmu,m68k-softmmu,microblaze-softmmu,mips-softmmu,mips64el-softmmu,nios2-softmmu,or1k-softmmu,ppc-softmmu,ppc64-softmmu,s390x-softmmu,sparc-softmmu,x86_64-softmmu,xtensa-softmmu"
         - TEST_CMD="make check-acceptance"
       after_script:
         - python3 -c 'import json; r = json.load(open("tests/results/latest/results.json")); [print(t["logfile"]) for t in r["tests"] if t["status"] not in ("PASS", "SKIP")]' | xargs cat
@@ -323,6 +323,7 @@ matrix:
             - python3-pil
             - python3-pip
             - python3.5-venv
+            - rpm2cpio
             - tesseract-ocr
             - tesseract-ocr-eng
 

python/qemu/accel.py

@@ -24,7 +24,8 @@ LOG = logging.getLogger(__name__)
 # support which often includes its 32 bit cousin.
 ADDITIONAL_ARCHES = {
     "x86_64" : "i386",
-    "aarch64" : "armhf"
+    "aarch64" : "armhf",
+    "ppc64le" : "ppc64",
 }
 
 def list_accel(qemu_bin):

python/qemu/machine.py

@@ -112,6 +112,7 @@ class QEMUMachine(object):
         self._sock_dir = sock_dir
         self._launched = False
         self._machine = None
+        self._console_index = 0
         self._console_set = False
         self._console_device_type = None
         self._console_address = None
@@ -241,6 +242,8 @@ class QEMUMachine(object):
                          'chardev=mon,mode=control'])
         if self._machine is not None:
             args.extend(['-machine', self._machine])
+        for i in range(self._console_index):
+            args.extend(['-serial', 'null'])
         if self._console_set:
             self._console_address = os.path.join(self._sock_dir,
                                                  self._name + "-console.sock")
@@ -527,7 +530,7 @@ class QEMUMachine(object):
         """
         self._machine = machine_type
 
-    def set_console(self, device_type=None):
+    def set_console(self, device_type=None, console_index=0):
         """
         Sets the device type for a console device
 
@@ -548,9 +551,14 @@ class QEMUMachine(object):
                             chardev:console" command line argument will
                             be used instead, resorting to the machine's
                             default device type.
+        @param console_index: the index of the console device to use.
+                              If not zero, the command line will create
+                              'index - 1' consoles and connect them to
+                              the 'null' backing character device.
         """
         self._console_set = True
         self._console_device_type = device_type
+        self._console_index = console_index
 
     @property
     def console_socket(self):

python/qemu/qmp.py

@@ -1,5 +1,4 @@
-# QEMU Monitor Protocol Python class
-#
+""" QEMU Monitor Protocol Python class """
 # Copyright (C) 2009, 2010 Red Hat Inc.
 #
 # Authors:
@@ -15,29 +14,37 @@ import logging
 
 
 class QMPError(Exception):
-    pass
+    """
+    QMP base exception
+    """
 
 
 class QMPConnectError(QMPError):
-    pass
+    """
+    QMP connection exception
+    """
 
 
 class QMPCapabilitiesError(QMPError):
-    pass
+    """
+    QMP negotiate capabilities exception
+    """
 
 
 class QMPTimeoutError(QMPError):
-    pass
+    """
+    QMP timeout exception
+    """
 
 
-class QEMUMonitorProtocol(object):
+class QEMUMonitorProtocol:
+    """
+    Provide an API to connect to QEMU via QEMU Monitor Protocol (QMP) and then
+    allow to handle commands and events.
+    """
 
     #: Logger object for debugging messages
     logger = logging.getLogger('QMP')
-    #: Socket's error class
-    error = socket.error
-    #: Socket's timeout
-    timeout = socket.timeout
 
     def __init__(self, address, server=False):
         """
@@ -47,7 +54,7 @@ class QEMUMonitorProtocol(object):
                         or a tuple in the form ( address, port ) for a TCP
                         connection
         @param server: server mode listens on the socket (bool)
-        @raise socket.error on socket connection errors
+        @raise OSError on socket connection errors
         @note No connection is established, this is done by the connect() or
               accept() methods
         """
@@ -73,7 +80,7 @@ class QEMUMonitorProtocol(object):
             raise QMPConnectError
         # Greeting seems ok, negotiate capabilities
         resp = self.cmd('qmp_capabilities')
-        if "return" in resp:
+        if resp and "return" in resp:
             return greeting
         raise QMPCapabilitiesError
 
@@ -81,7 +88,7 @@ class QEMUMonitorProtocol(object):
         while True:
             data = self.__sockfile.readline()
             if not data:
-                return
+                return None
             resp = json.loads(data)
             if 'event' in resp:
                 self.logger.debug("<<< %s", resp)
@@ -107,8 +114,8 @@ class QEMUMonitorProtocol(object):
         self.__sock.setblocking(0)
         try:
             self.__json_read()
-        except socket.error as err:
-            if err[0] == errno.EAGAIN:
+        except OSError as err:
+            if err.errno == errno.EAGAIN:
                 # No data available
                 pass
         self.__sock.setblocking(1)
@@ -128,12 +135,21 @@ class QEMUMonitorProtocol(object):
                 raise QMPConnectError("Error while reading from socket")
             self.__sock.settimeout(None)
 
+    def __enter__(self):
+        # Implement context manager enter function.
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        # Implement context manager exit function.
+        self.close()
+        return False
+
     def connect(self, negotiate=True):
         """
         Connect to the QMP Monitor and perform capabilities negotiation.
 
-        @return QMP greeting dict
-        @raise socket.error on socket connection errors
+        @return QMP greeting dict, or None if negotiate is false
+        @raise OSError on socket connection errors
         @raise QMPConnectError if the greeting is not received
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
@@ -141,17 +157,25 @@ class QEMUMonitorProtocol(object):
         self.__sockfile = self.__sock.makefile()
         if negotiate:
             return self.__negotiate_capabilities()
+        return None
 
-    def accept(self):
+    def accept(self, timeout=15.0):
         """
         Await connection from QMP Monitor and perform capabilities negotiation.
 
+        @param timeout: timeout in seconds (nonnegative float number, or
+                        None). The value passed will set the behavior of the
+                        underneath QMP socket as described in [1]. Default value
+                        is set to 15.0.
         @return QMP greeting dict
-        @raise socket.error on socket connection errors
+        @raise OSError on socket connection errors
         @raise QMPConnectError if the greeting is not received
         @raise QMPCapabilitiesError if fails to negotiate capabilities
+
+        [1]
+        https://docs.python.org/3/library/socket.html#socket.socket.settimeout
         """
-        self.__sock.settimeout(15)
+        self.__sock.settimeout(timeout)
         self.__sock, _ = self.__sock.accept()
         self.__sockfile = self.__sock.makefile()
         return self.__negotiate_capabilities()
@@ -167,10 +191,10 @@ class QEMUMonitorProtocol(object):
         self.logger.debug(">>> %s", qmp_cmd)
         try:
             self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8'))
-        except socket.error as err:
-            if err[0] == errno.EPIPE:
-                return
-            raise socket.error(err)
+        except OSError as err:
+            if err.errno == errno.EPIPE:
+                return None
+            raise err
         resp = self.__json_read()
         self.logger.debug("<<< %s", resp)
         return resp
@@ -243,14 +267,35 @@ class QEMUMonitorProtocol(object):
         self.__events = []
 
     def close(self):
-        self.__sock.close()
-        self.__sockfile.close()
+        """
+        Close the socket and socket file.
+        """
+        if self.__sock:
+            self.__sock.close()
+        if self.__sockfile:
+            self.__sockfile.close()
 
     def settimeout(self, timeout):
+        """
+        Set the socket timeout.
+
+        @param timeout (float): timeout in seconds, or None.
+        @note This is a wrap around socket.settimeout
+        """
         self.__sock.settimeout(timeout)
 
     def get_sock_fd(self):
+        """
+        Get the socket file descriptor.
+
+        @return The file descriptor number.
+        """
         return self.__sock.fileno()
 
     def is_scm_available(self):
+        """
+        Check if the socket allows for SCM_RIGHTS.
+
+        @return True if SCM_RIGHTS is available, otherwise False.
+        """
         return self.__sock.family == socket.AF_UNIX

qemu-deprecated.texi

@@ -320,14 +320,6 @@ they have no effect when used with @option{-n} to skip image creation.
 Silently ignored options can be confusing, so this combination of
 options will be made an error in future versions.
 
-@section Build system
-
-@subsection Python 2 support (since 4.1.0)
-
-In the future, QEMU will require Python 3 to be available at
-build time.  Support for Python 2 in scripts shipped with QEMU
-is deprecated.
-
 @section Backwards compatibility
 
 @subsection Runnability guarantee of CPU models (since 4.1.0)

scripts/analyse-9p-simpletrace.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Pretty print 9p simpletrace log
 # Usage: ./analyse-9p-simpletrace <trace-events> <trace-pid>
 #
 # Author: Harsh Prateek Bora
-from __future__ import print_function
 import os
 import simpletrace
 

scripts/analyse-locks-simpletrace.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Analyse lock events and compute statistics
@@ -6,7 +6,6 @@
 # Author: Alex Bennée <alex.bennee@linaro.org>
 #
 
-from __future__ import print_function
 import simpletrace
 import argparse
 import numpy as np

scripts/checkpatch.pl

@@ -1460,6 +1460,12 @@ sub process {
 			}
 		}
 
+# Only allow Python 3 interpreter
+		if ($realline == 1 &&
+			$line =~ /^\+#!\ *\/usr\/bin\/(?:env )?python$/) {
+			ERROR("please use python3 interpreter\n" . $herecurr);
+		}
+
 # Accept git diff extended headers as valid patches
 		if ($line =~ /^(?:rename|copy) (?:from|to) [\w\/\.\-]+\s*$/) {
 			$is_patch = 1;

scripts/decodetree.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2018 Linaro Limited
 #
 # This library is free software; you can redistribute it and/or

scripts/device-crash-test

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 #  Copyright (c) 2017 Red Hat Inc
 #
@@ -23,7 +23,6 @@
 Run QEMU with all combinations of -machine and -device types,
 check for crashes and unexpected errors.
 """
-from __future__ import print_function
 
 import os
 import sys

scripts/dump-guest-memory.py

@@ -12,7 +12,6 @@ Authors:
 This work is licensed under the terms of the GNU GPL, version 2 or later. See
 the COPYING file in the top-level directory.
 """
-from __future__ import print_function
 
 import ctypes
 import struct

scripts/kvm/kvm_flightrecorder

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # KVM Flight Recorder - ring buffer tracing script
 #
@@ -32,7 +32,6 @@
 # consuming CPU cycles.  No disk I/O is performed since the ring buffer holds a
 # fixed-size in-memory trace.
 
-from __future__ import print_function
 import sys
 import os
 

scripts/kvm/vmxcap

@@ -10,7 +10,6 @@
 # This work is licensed under the terms of the GNU GPL, version 2.  See
 # the COPYING file in the top-level directory.
 
-from __future__ import print_function
 MSR_IA32_VMX_BASIC = 0x480
 MSR_IA32_VMX_PINBASED_CTLS = 0x481
 MSR_IA32_VMX_PROCBASED_CTLS = 0x482

scripts/minikconf.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #
 # Mini-Kconfig parser
 #
@@ -10,7 +11,6 @@
 # or, at your option, any later version.  See the COPYING file in
 # the top-level directory.
 
-from __future__ import print_function
 import os
 import sys
 import re

scripts/modules/module_block.py

@@ -10,7 +10,6 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from __future__ import print_function
 import sys
 import os
 

scripts/qapi-gen.py

@@ -1,10 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # QAPI generator
 #
 # This work is licensed under the terms of the GNU GPL, version 2 or later.
 # See the COPYING file in the top-level directory.
 
-from __future__ import print_function
 
 import argparse
 import re

scripts/qapi/doc.py

@@ -4,7 +4,6 @@
 # See the COPYING file in the top-level directory.
 """This script produces the documentation of a qapi schema in texinfo format"""
 
-from __future__ import print_function
 import re
 from qapi.gen import QAPIGenDoc, QAPISchemaVisitor
 

scripts/qmp/qemu-ga-client

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 # QEMU Guest Agent Client
 #
@@ -36,7 +36,6 @@
 # See also: https://wiki.qemu.org/Features/QAPI/GuestAgent
 #
 
-from __future__ import print_function
 import os
 import sys
 import base64

scripts/qmp/qmp

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # QMP command line tool
 #
@@ -10,7 +10,6 @@
 # This work is licensed under the terms of the GNU GPLv2 or later.
 # See the COPYING file in the top-level directory.
 
-from __future__ import print_function
 import sys, os
 from qmp import QEMUMonitorProtocol