izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/philmd-gitlab/tags/python-next-20200531' into staging

Browse Source 
Python queue:

* migration acceptance test fix
* introduce pylintrc & flake8 config
* various cleanups (Python3, style)
* vm-test can set QEMU_LOCAL=1 to use locally built binaries
* refactored BootLinuxBase & LinuxKernelTest acceptance classes

https://gitlab.com/philmd/qemu/pipelines/151323210
https://travis-ci.org/github/philmd/qemu/builds/693157969

# gpg: Signature made Sun 31 May 2020 17:37:35 BST
# gpg:                using RSA key FAABE75E12917221DCFD6BB2E3E32C2CDEADC0DE
# gpg: Good signature from "Philippe Mathieu-Daudé (F4BUG) <f4bug@amsat.org>" [full]
# Primary key fingerprint: FAAB E75E 1291 7221 DCFD  6BB2 E3E3 2C2C DEAD C0DE

* remotes/philmd-gitlab/tags/python-next-20200531: (25 commits)
  tests/acceptance: refactor boot_linux to allow code reuse
  tests/acceptance: refactor boot_linux_console test to allow code reuse
  tests/acceptance: allow console interaction with specific VMs
  tests/acceptance/migration.py: Wait for both sides
  tests/migration/guestperf: Use Python 3 interpreter
  tests/vm: allow wait_ssh() to specify command
  tests/vm: Add ability to select QEMU from current build
  tests/vm: Pass --debug through for vm-boot-ssh
  python/qemu/qtest: Check before accessing _qtest
  python/qemu/qmp: assert sockfile is not None
  python/qemu/qmp: use True/False for non/blocking modes
  python/qemu: Adjust traceback typing
  python/qemu: fix socket.makefile() typing
  python/qemu: remove Python2 style super() calls
  python/qemu: delint; add flake8 config
  python/qemu: delint and add pylintrc
  python/qemu/machine: remove logging configuration
  python/qemu/machine: add kill() method
  python: remove more instances of sys.version_info
  scripts/qmp: Fix shebang and imports
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
2020-05-31 21:49:07 +01:00
parent 4ec2a1f53e 1c80c87c8c
commit b73f417aae
35 changed files with 317 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/qemu/.flake8
+2
View File
@@ -0,0 +1,2 @@
[flake8]
extend-ignore = E722 # Pylint handles this, but smarter.
python/qemu/accel.py
+6 -3
View File
@@ -23,11 +23,12 @@ LOG = logging.getLogger(__name__)
# Mapping host architecture to any additional architectures it can
# support which often includes its 32 bit cousin.
ADDITIONAL_ARCHES = {
"x86_64" : "i386",
"aarch64" : "armhf",
"ppc64le" : "ppc64",
"x86_64": "i386",
"aarch64": "armhf",
"ppc64le": "ppc64",
}
def list_accel(qemu_bin):
"""
List accelerators enabled in the QEMU binary.
@@ -47,6 +48,7 @@ def list_accel(qemu_bin):
# Skip the first line which is the header.
return [acc.strip() for acc in out.splitlines()[1:]]
def kvm_available(target_arch=None, qemu_bin=None):
"""
Check if KVM is available using the following heuristic:
@@ -69,6 +71,7 @@ def kvm_available(target_arch=None, qemu_bin=None):
return False
return True
def tcg_available(qemu_bin):
"""
Check if TCG is available.
python/qemu/machine.py
+28 -16
View File
@@ -24,11 +24,14 @@ import subprocess
import shutil
import socket
import tempfile
from typing import Optional, Type
from types import TracebackType
from . import qmp
LOG = logging.getLogger(__name__)
class QEMUMachineError(Exception):
"""
Exception called when an error in QEMUMachine happens.
@@ -54,15 +57,16 @@ class MonitorResponseError(qmp.QMPError):
desc = reply["error"]["desc"]
except KeyError:
desc = reply
super(MonitorResponseError, self).__init__(desc)
super().__init__(desc)
self.reply = reply
class QEMUMachine(object):
class QEMUMachine:
"""
A QEMU VM
Use this object as a context manager to ensure the QEMU process terminates::
Use this object as a context manager to ensure
the QEMU process terminates::
with VM(binary) as vm:
...
@@ -119,15 +123,14 @@ class QEMUMachine(object):
self._console_socket = None
self._remove_files = []
# just in case logging wasn't configured by the main script:
logging.basicConfig()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
def __exit__(self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType]) -> None:
self.shutdown()
return False
def add_monitor_null(self):
"""
@@ -188,8 +191,10 @@ class QEMUMachine(object):
fd_param.append(str(fd))
devnull = open(os.path.devnull, 'rb')
proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, close_fds=False)
proc = subprocess.Popen(
fd_param, stdin=devnull, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, close_fds=False
)
output = proc.communicate()[0]
if output:
LOG.debug(output)
@@ -242,7 +247,7 @@ class QEMUMachine(object):
'chardev=mon,mode=control'])
if self._machine is not None:
args.extend(['-machine', self._machine])
for i in range(self._console_index):
for _ in range(self._console_index):
args.extend(['-serial', 'null'])
if self._console_set:
self._console_address = os.path.join(self._sock_dir,
@@ -342,7 +347,7 @@ class QEMUMachine(object):
self._load_io_log()
self._post_shutdown()
def shutdown(self, has_quit=False):
def shutdown(self, has_quit=False, hard=False):
"""
Terminate the VM and clean up
"""
@@ -354,7 +359,9 @@ class QEMUMachine(object):
self._console_socket = None
if self.is_running():
if self._qmp:
if hard:
self._popen.kill()
elif self._qmp:
try:
if not has_quit:
self._qmp.cmd('quit')
@@ -368,16 +375,20 @@ class QEMUMachine(object):
self._post_shutdown()
exitcode = self.exitcode()
if exitcode is not None and exitcode < 0:
if exitcode is not None and exitcode < 0 and \
not (exitcode == -9 and hard):
msg = 'qemu received signal %i: %s'
if self._qemu_full_args:
command = ' '.join(self._qemu_full_args)
else:
command = ''
LOG.warning(msg, -exitcode, command)
LOG.warning(msg, -int(exitcode), command)
self._launched = False
def kill(self):
self.shutdown(hard=True)
def set_qmp_monitor(self, enabled=True):
"""
Set the QMP monitor.
@@ -482,7 +493,8 @@ class QEMUMachine(object):
def events_wait(self, events, timeout=60.0):
"""
events_wait waits for and returns a named event from QMP with a timeout.
events_wait waits for and returns a named event
from QMP with a timeout.
events: a sequence of (name, match_criteria) tuples.
The match criteria are optional and may be None.
python/qemu/pylintrc
+58
View File
@@ -0,0 +1,58 @@
[MASTER]
[MESSAGES CONTROL]
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once). You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
disable=too-many-arguments,
too-many-instance-attributes,
too-many-public-methods,
[REPORTS]
[REFACTORING]
[MISCELLANEOUS]
[LOGGING]
[BASIC]
# Good variable names which should always be accepted, separated by a comma.
good-names=i,
j,
k,
ex,
Run,
_,
fd,
[VARIABLES]
[STRING]
[SPELLING]
[FORMAT]
[SIMILARITIES]
# Ignore imports when computing similarities.
ignore-imports=yes
[TYPECHECK]
[CLASSES]
[IMPORTS]
[DESIGN]
[EXCEPTIONS]
python/qemu/qmp.py
+20 -9
View File
@@ -11,6 +11,12 @@ import json
import errno
import socket
import logging
from typing import (
Optional,
TextIO,
Type,
)
from types import TracebackType
class QMPError(Exception):
@@ -61,7 +67,7 @@ class QEMUMonitorProtocol:
self.__events = []
self.__address = address
self.__sock = self.__get_sock()
self.__sockfile = None
self.__sockfile: Optional[TextIO] = None
self._nickname = nickname
if self._nickname:
self.logger = logging.getLogger('QMP').getChild(self._nickname)
@@ -88,6 +94,7 @@ class QEMUMonitorProtocol:
raise QMPCapabilitiesError
def __json_read(self, only_event=False):
assert self.__sockfile is not None
while True:
data = self.__sockfile.readline()
if not data:
@@ -114,14 +121,14 @@ class QEMUMonitorProtocol:
"""
# Check for new events regardless and pull them into the cache:
self.__sock.setblocking(0)
self.__sock.setblocking(False)
try:
self.__json_read()
except OSError as err:
if err.errno == errno.EAGAIN:
# No data available
pass
self.__sock.setblocking(1)
self.__sock.setblocking(True)
# Wait for new events, if needed.
# if wait is 0.0, this means "no wait" and is also implicitly false.
@@ -142,10 +149,14 @@ class QEMUMonitorProtocol:
# Implement context manager enter function.
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
def __exit__(self,
# pylint: disable=duplicate-code
# see https://github.com/PyCQA/pylint/issues/3619
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType]) -> None:
# Implement context manager exit function.
self.close()
return False
def connect(self, negotiate=True):
"""
@@ -157,7 +168,7 @@ class QEMUMonitorProtocol:
@raise QMPCapabilitiesError if fails to negotiate capabilities
"""
self.__sock.connect(self.__address)
self.__sockfile = self.__sock.makefile()
self.__sockfile = self.__sock.makefile(mode='r')
if negotiate:
return self.__negotiate_capabilities()
return None
@@ -168,8 +179,8 @@ class QEMUMonitorProtocol:
@param timeout: timeout in seconds (nonnegative float number, or
None). The value passed will set the behavior of the
underneath QMP socket as described in [1]. Default value
is set to 15.0.
underneath QMP socket as described in [1].
Default value is set to 15.0.
@return QMP greeting dict
@raise OSError on socket connection errors
@raise QMPConnectError if the greeting is not received
@@ -180,7 +191,7 @@ class QEMUMonitorProtocol:
"""
self.__sock.settimeout(timeout)
self.__sock, _ = self.__sock.accept()
self.__sockfile = self.__sock.makefile()
self.__sockfile = self.__sock.makefile(mode='r')
return self.__negotiate_capabilities()
def cmd_obj(self, qmp_cmd):
python/qemu/qtest.py
+53 -30
View File
@@ -1,5 +1,11 @@
# QEMU qtest library
#
"""
QEMU qtest library
qtest offers the QEMUQtestProtocol and QEMUQTestMachine classes, which
offer a connection to QEMU's qtest protocol socket, and a qtest-enabled
subclass of QEMUMachine, respectively.
"""
# Copyright (C) 2015 Red Hat Inc.
#
# Authors:
@@ -13,26 +19,29 @@
import socket
import os
from typing import Optional, TextIO
from .machine import QEMUMachine
class QEMUQtestProtocol(object):
def __init__(self, address, server=False):
"""
Create a QEMUQtestProtocol object.
class QEMUQtestProtocol:
"""
QEMUQtestProtocol implements a connection to a qtest socket.
@param address: QEMU address, can be either a unix socket path (string)
or a tuple in the form ( address, port ) for a TCP
connection
@param server: server mode, listens on the socket (bool)
@raise socket.error on socket connection errors
@note No connection is established, this is done by the connect() or
accept() methods
"""
:param address: QEMU address, can be either a unix socket path (string)
or a tuple in the form ( address, port ) for a TCP
connection
:param server: server mode, listens on the socket (bool)
:raise socket.error: on socket connection errors
.. note::
No conection is estabalished by __init__(), this is done
by the connect() or accept() methods.
"""
def __init__(self, address, server=False):
self._address = address
self._sock = self._get_sock()
self._sockfile = None
self._sockfile: Optional[TextIO] = None
if server:
self._sock.bind(self._address)
self._sock.listen(1)
@@ -51,7 +60,7 @@ class QEMUQtestProtocol(object):
@raise socket.error on socket connection errors
"""
self._sock.connect(self._address)
self._sockfile = self._sock.makefile()
self._sockfile = self._sock.makefile(mode='r')
def accept(self):
"""
@@ -60,7 +69,7 @@ class QEMUQtestProtocol(object):
@raise socket.error on socket connection errors
"""
self._sock, _ = self._sock.accept()
self._sockfile = self._sock.makefile()
self._sockfile = self._sock.makefile(mode='r')
def cmd(self, qtest_cmd):
"""
@@ -68,20 +77,27 @@ class QEMUQtestProtocol(object):
@param qtest_cmd: qtest command text to be sent
"""
assert self._sockfile is not None
self._sock.sendall((qtest_cmd + "\n").encode('utf-8'))
resp = self._sockfile.readline()
return resp
def close(self):
"""Close this socket."""
self._sock.close()
self._sockfile.close()
if self._sockfile:
self._sockfile.close()
self._sockfile = None
def settimeout(self, timeout):
"""Set a timeout, in seconds."""
self._sock.settimeout(timeout)
class QEMUQtestMachine(QEMUMachine):
'''A QEMU VM'''
"""
A QEMU VM, with a qtest socket available.
"""
def __init__(self, binary, args=None, name=None, test_dir="/var/tmp",
socket_scm_helper=None, sock_dir=None):
@@ -89,31 +105,38 @@ class QEMUQtestMachine(QEMUMachine):
name = "qemu-%d" % os.getpid()
if sock_dir is None:
sock_dir = test_dir
super(QEMUQtestMachine,
self).__init__(binary, args, name=name, test_dir=test_dir,
socket_scm_helper=socket_scm_helper,
sock_dir=sock_dir)
super().__init__(binary, args, name=name, test_dir=test_dir,
socket_scm_helper=socket_scm_helper,
sock_dir=sock_dir)
self._qtest = None
self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock")
def _base_args(self):
args = super(QEMUQtestMachine, self)._base_args()
args = super()._base_args()
args.extend(['-qtest', 'unix:path=' + self._qtest_path,
'-accel', 'qtest'])
return args
def _pre_launch(self):
super(QEMUQtestMachine, self)._pre_launch()
super()._pre_launch()
self._qtest = QEMUQtestProtocol(self._qtest_path, server=True)
def _post_launch(self):
super(QEMUQtestMachine, self)._post_launch()
def _post_launch(self) -> None:
assert self._qtest is not None
super()._post_launch()
self._qtest.accept()
def _post_shutdown(self):
super(QEMUQtestMachine, self)._post_shutdown()
super()._post_shutdown()
self._remove_if_exists(self._qtest_path)
def qtest(self, cmd):
'''Send a qtest command to guest'''
def qtest(self, cmd: str) -> str:
"""
Send a qtest command to the guest.
:param cmd: qtest command to send
:return: qtest server response
"""
if self._qtest is None:
raise RuntimeError("qtest socket not available")
return self._qtest.cmd(cmd)
scripts/analyze-migration.py
-5
View File
@@ -25,11 +25,6 @@ import struct
import sys
MIN_PYTHON = (3, 2)
if sys.version_info < MIN_PYTHON:
sys.exit("Python %s.%s or later is required.\n" % MIN_PYTHON)
def mkdir_p(path):
try:
os.makedirs(path)
scripts/decodetree.py
+9 -16
View File
@@ -75,13 +75,6 @@ def output(*args):
output_fd.write(a)
if sys.version_info >= (3, 4):
re_fullmatch = re.fullmatch
else:
def re_fullmatch(pat, str):
return re.match('^' + pat + '$', str)
def output_autogen():
output('/* This file is autogenerated by scripts/decodetree.py. */\n\n')
@@ -428,18 +421,18 @@ def parse_field(lineno, name, toks):
width = 0
func = None
for t in toks:
if re_fullmatch('!function=' + re_ident, t):
if re.fullmatch('!function=' + re_ident, t):
if func:
error(lineno, 'duplicate function')
func = t.split('=')
func = func[1]
continue
if re_fullmatch('[0-9]+:s[0-9]+', t):
if re.fullmatch('[0-9]+:s[0-9]+', t):
# Signed field extract
subtoks = t.split(':s')
sign = True
elif re_fullmatch('[0-9]+:[0-9]+', t):
elif re.fullmatch('[0-9]+:[0-9]+', t):
# Unsigned field extract
subtoks = t.split(':')
sign = False
@@ -488,11 +481,11 @@ def parse_arguments(lineno, name, toks):
flds = []
extern = False
for t in toks:
if re_fullmatch('!extern', t):
if re.fullmatch('!extern', t):
extern = True
anyextern = True
continue
if not re_fullmatch(re_ident, t):
if not re.fullmatch(re_ident, t):
error(lineno, 'invalid argument set token "{0}"'.format(t))
if t in flds:
error(lineno, 'duplicate argument "{0}"'.format(t))
@@ -621,13 +614,13 @@ def parse_generic(lineno, is_format, name, toks):
continue
# 'Foo=%Bar' imports a field with a different name.
if re_fullmatch(re_ident + '=%' + re_ident, t):
if re.fullmatch(re_ident + '=%' + re_ident, t):
(fname, iname) = t.split('=%')
flds = add_field_byname(lineno, flds, fname, iname)
continue
# 'Foo=number' sets an argument field to a constant value
if re_fullmatch(re_ident + '=[+-]?[0-9]+', t):
if re.fullmatch(re_ident + '=[+-]?[0-9]+', t):
(fname, value) = t.split('=')
value = int(value)
flds = add_field(lineno, flds, fname, ConstField(value))
@@ -635,7 +628,7 @@ def parse_generic(lineno, is_format, name, toks):
# Pattern of 0s, 1s, dots and dashes indicate required zeros,
# required ones, or dont-cares.
if re_fullmatch('[01.-]+', t):
if re.fullmatch('[01.-]+', t):
shift = len(t)
fms = t.replace('0', '1')
fms = fms.replace('.', '0')
@@ -652,7 +645,7 @@ def parse_generic(lineno, is_format, name, toks):
fixedmask = (fixedmask << shift) | fms
undefmask = (undefmask << shift) | ubm
# Otherwise, fieldname:fieldwidth
elif re_fullmatch(re_ident + ':s?[0-9]+', t):
elif re.fullmatch(re_ident + ':s?[0-9]+', t):
(fname, flen) = t.split(':')
sign = False
if flen[0] == 's':
scripts/kvm/vmxcap
+4 -3
View File
@@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# tool for querying VMX capabilities
#
@@ -275,5 +275,6 @@ controls = [
),
]
for c in controls:
c.show()
if __name__ == '__main__':
for c in controls:
c.show()
scripts/modules/module_block.py
+15 -14
View File
@@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Module information generator
#
@@ -80,19 +80,20 @@ def print_bottom(fheader):
#endif
''')
# First argument: output file
# All other arguments: modules source files (.c)
output_file = sys.argv[1]
with open(output_file, 'w') as fheader:
print_top(fheader)
if __name__ == '__main__':
# First argument: output file
# All other arguments: modules source files (.c)
output_file = sys.argv[1]
with open(output_file, 'w') as fheader:
print_top(fheader)
for filename in sys.argv[2:]:
if os.path.isfile(filename):
process_file(fheader, filename)
else:
print("File " + filename + " does not exist.", file=sys.stderr)
sys.exit(1)
for filename in sys.argv[2:]:
if os.path.isfile(filename):
process_file(fheader, filename)
else:
print("File " + filename + " does not exist.", file=sys.stderr)
sys.exit(1)
print_bottom(fheader)
print_bottom(fheader)
sys.exit(0)
sys.exit(0)
scripts/qemu-gdb.py
+2 -2
View File
@@ -1,5 +1,5 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# GDB debugging support
#
# Copyright 2012 Red Hat, Inc. and/or its affiliates
scripts/qemugdb/__init__.py
+1 -2
View File
@@ -1,5 +1,4 @@
#!/usr/bin/python
#
# GDB debugging support
#
# Copyright (c) 2015 Linaro Ltd
scripts/qemugdb/aio.py
+1 -2
View File
@@ -1,5 +1,4 @@
#!/usr/bin/python
#
# GDB debugging support: aio/iohandler debug
#
# Copyright (c) 2015 Red Hat, Inc.
scripts/qemugdb/coroutine.py
+1 -2
View File
@@ -1,5 +1,4 @@
#!/usr/bin/python
#
# GDB debugging support
#
# Copyright 2012 Red Hat, Inc. and/or its affiliates
scripts/qemugdb/mtree.py
+1 -3
View File
@@ -1,5 +1,4 @@
#!/usr/bin/python
#
# GDB debugging support
#
# Copyright 2012 Red Hat, Inc. and/or its affiliates
@@ -84,4 +83,3 @@ class MtreeCommand(gdb.Command):
while not isnull(subregion):
self.print_item(subregion, addr, level)
subregion = subregion['subregions_link']['tqe_next']
scripts/qemugdb/tcg.py
-1
View File
@@ -1,4 +1,3 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# GDB debugging support, TCG status
scripts/qemugdb/timers.py
-1
View File
@@ -1,4 +1,3 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# GDB debugging support
#
scripts/qmp/qmp
+3 -1
View File
@@ -11,7 +11,9 @@
# See the COPYING file in the top-level directory.
import sys, os
from qmp import QEMUMonitorProtocol
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
from qemu.qmp import QEMUMonitorProtocol
def print_response(rsp, prefix=[]):
if type(rsp) == list:
scripts/qmp/qmp-shell
-3
View File
@@ -77,9 +77,6 @@ import re
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
from qemu import qmp
if sys.version_info[0] == 2:
input = raw_input
class QMPCompleter(list):
def complete(self, text, state):
for cmd in self:
scripts/qmp/qom-fuse
+3 -1
View File
@@ -15,7 +15,9 @@ import fuse, stat
from fuse import Fuse
import os, posix
from errno import *
from qmp import QEMUMonitorProtocol
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
from qemu.qmp import QEMUMonitorProtocol
fuse.fuse_python_api = (0, 2)

Some files were not shown because too many files have changed in this diff Show More