From 44445d8668f458dec0ecb8bcecdcbf24e6d4251b Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:20 -0400 Subject: [PATCH 01/19] s390 vfio-ccw: Add bootindex property and IPLB data Add bootindex property and iplb data for vfio-ccw devices. This allows us to forward boot information into the bios for vfio-ccw devices. Refactor s390_get_ccw_device() to return device type. This prevents us from having to use messy casting logic in several places. Signed-off-by: Jason J. Herne Acked-by: Halil Pasic Reviewed-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-2-git-send-email-jjherne@linux.ibm.com> [thuth: fixed "typedef struct VFIOCCWDevice" build failure with clang] Signed-off-by: Thomas Huth --- MAINTAINERS | 1 + hw/s390x/ipl.c | 61 ++++++++++++++++++++++++++++--------- hw/s390x/s390-ccw.c | 9 ++++++ hw/vfio/ccw.c | 6 ++-- include/hw/s390x/s390-ccw.h | 1 + include/hw/s390x/vfio-ccw.h | 28 +++++++++++++++++ 6 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 include/hw/s390x/vfio-ccw.h diff --git a/MAINTAINERS b/MAINTAINERS index 56139ac8ab..f7976aa43d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1445,6 +1445,7 @@ S: Supported F: hw/vfio/ccw.c F: hw/s390x/s390-ccw.c F: include/hw/s390x/s390-ccw.h +F: include/hw/s390x/vfio-ccw.h T: git https://github.com/cohuck/qemu.git s390-next L: qemu-s390x@nongnu.org diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c index 51b272e190..d0cc06a05f 100644 --- a/hw/s390x/ipl.c +++ b/hw/s390x/ipl.c @@ -19,6 +19,7 @@ #include "hw/loader.h" #include "hw/boards.h" #include "hw/s390x/virtio-ccw.h" +#include "hw/s390x/vfio-ccw.h" #include "hw/s390x/css.h" #include "hw/s390x/ebcdic.h" #include "ipl.h" @@ -303,16 +304,36 @@ static void s390_ipl_set_boot_menu(S390IPLState *ipl) ipl->qipl.boot_menu_timeout = cpu_to_be32(splash_time); } -static CcwDevice *s390_get_ccw_device(DeviceState *dev_st) +#define CCW_DEVTYPE_NONE 0x00 +#define CCW_DEVTYPE_VIRTIO 0x01 +#define CCW_DEVTYPE_VIRTIO_NET 0x02 +#define CCW_DEVTYPE_SCSI 0x03 +#define CCW_DEVTYPE_VFIO 0x04 + +static CcwDevice *s390_get_ccw_device(DeviceState *dev_st, int *devtype) { CcwDevice *ccw_dev = NULL; + int tmp_dt = CCW_DEVTYPE_NONE; if (dev_st) { + VirtIONet *virtio_net_dev = (VirtIONet *) + object_dynamic_cast(OBJECT(dev_st), TYPE_VIRTIO_NET); VirtioCcwDevice *virtio_ccw_dev = (VirtioCcwDevice *) object_dynamic_cast(OBJECT(qdev_get_parent_bus(dev_st)->parent), TYPE_VIRTIO_CCW_DEVICE); + VFIOCCWDevice *vfio_ccw_dev = (VFIOCCWDevice *) + object_dynamic_cast(OBJECT(dev_st), TYPE_VFIO_CCW); + if (virtio_ccw_dev) { ccw_dev = CCW_DEVICE(virtio_ccw_dev); + if (virtio_net_dev) { + tmp_dt = CCW_DEVTYPE_VIRTIO_NET; + } else { + tmp_dt = CCW_DEVTYPE_VIRTIO; + } + } else if (vfio_ccw_dev) { + ccw_dev = CCW_DEVICE(vfio_ccw_dev); + tmp_dt = CCW_DEVTYPE_VFIO; } else { SCSIDevice *sd = (SCSIDevice *) object_dynamic_cast(OBJECT(dev_st), @@ -325,9 +346,13 @@ static CcwDevice *s390_get_ccw_device(DeviceState *dev_st) ccw_dev = (CcwDevice *)object_dynamic_cast(OBJECT(scsi_ccw), TYPE_CCW_DEVICE); + tmp_dt = CCW_DEVTYPE_SCSI; } } } + if (devtype) { + *devtype = tmp_dt; + } return ccw_dev; } @@ -335,20 +360,22 @@ static bool s390_gen_initial_iplb(S390IPLState *ipl) { DeviceState *dev_st; CcwDevice *ccw_dev = NULL; + SCSIDevice *sd; + int devtype; dev_st = get_boot_device(0); if (dev_st) { - ccw_dev = s390_get_ccw_device(dev_st); + ccw_dev = s390_get_ccw_device(dev_st, &devtype); } /* * Currently allow IPL only from CCW devices. */ if (ccw_dev) { - SCSIDevice *sd = (SCSIDevice *) object_dynamic_cast(OBJECT(dev_st), - TYPE_SCSI_DEVICE); - - if (sd) { + switch (devtype) { + case CCW_DEVTYPE_SCSI: + sd = (SCSIDevice *) object_dynamic_cast(OBJECT(dev_st), + TYPE_SCSI_DEVICE); ipl->iplb.len = cpu_to_be32(S390_IPLB_MIN_QEMU_SCSI_LEN); ipl->iplb.blk0_len = cpu_to_be32(S390_IPLB_MIN_QEMU_SCSI_LEN - S390_IPLB_HEADER_LEN); @@ -358,20 +385,24 @@ static bool s390_gen_initial_iplb(S390IPLState *ipl) ipl->iplb.scsi.channel = cpu_to_be16(sd->channel); ipl->iplb.scsi.devno = cpu_to_be16(ccw_dev->sch->devno); ipl->iplb.scsi.ssid = ccw_dev->sch->ssid & 3; - } else { - VirtIONet *vn = (VirtIONet *) object_dynamic_cast(OBJECT(dev_st), - TYPE_VIRTIO_NET); - + break; + case CCW_DEVTYPE_VFIO: + ipl->iplb.len = cpu_to_be32(S390_IPLB_MIN_CCW_LEN); + ipl->iplb.pbt = S390_IPL_TYPE_CCW; + ipl->iplb.ccw.devno = cpu_to_be16(ccw_dev->sch->devno); + ipl->iplb.ccw.ssid = ccw_dev->sch->ssid & 3; + break; + case CCW_DEVTYPE_VIRTIO_NET: + ipl->netboot = true; + /* Fall through to CCW_DEVTYPE_VIRTIO case */ + case CCW_DEVTYPE_VIRTIO: ipl->iplb.len = cpu_to_be32(S390_IPLB_MIN_CCW_LEN); ipl->iplb.blk0_len = cpu_to_be32(S390_IPLB_MIN_CCW_LEN - S390_IPLB_HEADER_LEN); ipl->iplb.pbt = S390_IPL_TYPE_CCW; ipl->iplb.ccw.devno = cpu_to_be16(ccw_dev->sch->devno); ipl->iplb.ccw.ssid = ccw_dev->sch->ssid & 3; - - if (vn) { - ipl->netboot = true; - } + break; } if (!s390_ipl_set_loadparm(ipl->iplb.loadparm)) { @@ -530,7 +561,7 @@ void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type) !ipl->netboot && ipl->iplb.pbt == S390_IPL_TYPE_CCW && is_virtio_scsi_device(&ipl->iplb)) { - CcwDevice *ccw_dev = s390_get_ccw_device(get_boot_device(0)); + CcwDevice *ccw_dev = s390_get_ccw_device(get_boot_device(0), NULL); if (ccw_dev && cpu_to_be16(ccw_dev->sch->devno) == ipl->iplb.ccw.devno && diff --git a/hw/s390x/s390-ccw.c b/hw/s390x/s390-ccw.c index cad91ee626..f5f025d1b6 100644 --- a/hw/s390x/s390-ccw.c +++ b/hw/s390x/s390-ccw.c @@ -124,6 +124,14 @@ static void s390_ccw_unrealize(S390CCWDevice *cdev, Error **errp) g_free(cdev->mdevid); } +static void s390_ccw_instance_init(Object *obj) +{ + S390CCWDevice *dev = S390_CCW_DEVICE(obj); + + device_add_bootindex_property(obj, &dev->bootindex, "bootindex", + "/disk@0,0", DEVICE(obj), NULL); +} + static void s390_ccw_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); @@ -137,6 +145,7 @@ static void s390_ccw_class_init(ObjectClass *klass, void *data) static const TypeInfo s390_ccw_info = { .name = TYPE_S390_CCW, .parent = TYPE_CCW_DEVICE, + .instance_init = s390_ccw_instance_init, .instance_size = sizeof(S390CCWDevice), .class_size = sizeof(S390CCWDeviceClass), .class_init = s390_ccw_class_init, diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c index c44d13cc50..31dd3a2a87 100644 --- a/hw/vfio/ccw.c +++ b/hw/vfio/ccw.c @@ -21,12 +21,12 @@ #include "hw/vfio/vfio.h" #include "hw/vfio/vfio-common.h" #include "hw/s390x/s390-ccw.h" +#include "hw/s390x/vfio-ccw.h" #include "hw/s390x/ccw-device.h" #include "exec/address-spaces.h" #include "qemu/error-report.h" -#define TYPE_VFIO_CCW "vfio-ccw" -typedef struct VFIOCCWDevice { +struct VFIOCCWDevice { S390CCWDevice cdev; VFIODevice vdev; uint64_t io_region_size; @@ -35,7 +35,7 @@ typedef struct VFIOCCWDevice { EventNotifier io_notifier; bool force_orb_pfch; bool warned_orb_pfch; -} VFIOCCWDevice; +}; static inline void warn_once_pfch(VFIOCCWDevice *vcdev, SubchDev *sch, const char *msg) diff --git a/include/hw/s390x/s390-ccw.h b/include/hw/s390x/s390-ccw.h index 7d15a1a5d4..901d805d79 100644 --- a/include/hw/s390x/s390-ccw.h +++ b/include/hw/s390x/s390-ccw.h @@ -27,6 +27,7 @@ typedef struct S390CCWDevice { CcwDevice parent_obj; CssDevId hostid; char *mdevid; + int32_t bootindex; } S390CCWDevice; typedef struct S390CCWDeviceClass { diff --git a/include/hw/s390x/vfio-ccw.h b/include/hw/s390x/vfio-ccw.h new file mode 100644 index 0000000000..ee5250d0d7 --- /dev/null +++ b/include/hw/s390x/vfio-ccw.h @@ -0,0 +1,28 @@ +/* + * vfio based subchannel assignment support + * + * Copyright 2017, 2019 IBM Corp. + * Author(s): Dong Jia Shi + * Xiao Feng Ren + * Pierre Morel + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#ifndef HW_VFIO_CCW_H +#define HW_VFIO_CCW_H + +#include "hw/vfio/vfio-common.h" +#include "hw/s390x/s390-ccw.h" +#include "hw/s390x/ccw-device.h" + +#define TYPE_VFIO_CCW "vfio-ccw" +#define VFIO_CCW(obj) \ + OBJECT_CHECK(VFIOCCWDevice, (obj), TYPE_VFIO_CCW) + +#define TYPE_VFIO_CCW "vfio-ccw" +typedef struct VFIOCCWDevice VFIOCCWDevice; + +#endif From 87f910c142d5589ef937ac216f92c6dcddae955e Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:21 -0400 Subject: [PATCH 02/19] s390-bios: decouple cio setup from virtio Move channel i/o setup code out to a separate function. This decouples cio setup from the virtio code path and allows us to make use of it for booting dasd devices. Signed-off-by: Jason J. Herne Acked-by: Halil Pasic Reviewed-by: Collin Walling Reviewed-by: Farhan Ali Reviewed-by: Thomas Huth Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-3-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index 544851d672..e82fe2ce23 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -99,6 +99,18 @@ static void menu_setup(void) } } +/* + * Initialize the channel I/O subsystem so we can talk to our ipl/boot device. + */ +static void css_setup(void) +{ + /* + * Unconditionally enable mss support. In every sane configuration this + * will succeed; and even if it doesn't, stsch_err() can handle it. + */ + enable_mss_facility(); +} + static void virtio_setup(void) { Schib schib; @@ -109,13 +121,6 @@ static void virtio_setup(void) VDev *vdev = virtio_get_device(); QemuIplParameters *early_qipl = (QemuIplParameters *)QIPL_ADDRESS; - /* - * We unconditionally enable mss support. In every sane configuration, - * this will succeed; and even if it doesn't, stsch_err() can deal - * with the consequences. - */ - enable_mss_facility(); - sclp_get_loadparm_ascii(loadparm_str); memcpy(ldp + 10, loadparm_str, LOADPARM_LEN); sclp_print(ldp); @@ -168,6 +173,7 @@ static void virtio_setup(void) int main(void) { sclp_setup(); + css_setup(); virtio_setup(); zipl_load(); /* no return */ From a5f6e0975b1f1b79f446c8323e62fd0534408da6 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:22 -0400 Subject: [PATCH 03/19] s390-bios: decouple common boot logic from virtio Create a boot_setup function to handle getting boot information from the machine/hypervisor. This decouples common boot logic from the virtio code path and allows us to make use of it for the real dasd boot scenario. Signed-off-by: Jason J. Herne Acked-by: Halil Pasic Reviewed-by: Collin Walling Reviewed-by: Farhan Ali Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-4-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index e82fe2ce23..67df42191b 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -14,16 +14,17 @@ char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE))); static SubChannelId blk_schid = { .one = 1 }; -IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE))); static char loadparm_str[LOADPARM_LEN + 1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; QemuIplParameters qipl; +IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE))); +static bool have_iplb; #define LOADPARM_PROMPT "PROMPT " #define LOADPARM_EMPTY " " #define BOOT_MENU_FLAG_MASK (QIPL_FLAG_BM_OPTS_CMD | QIPL_FLAG_BM_OPTS_ZIPL) /* - * Priniciples of Operations (SA22-7832-09) chapter 17 requires that + * Principles of Operations (SA22-7832-09) chapter 17 requires that * a subsystem-identification is at 184-187 and bytes 188-191 are zero * after list-directed-IPL and ccw-IPL. */ @@ -111,23 +112,33 @@ static void css_setup(void) enable_mss_facility(); } +/* + * Collect various pieces of information from the hypervisor/hardware that + * we'll use to determine exactly how we'll boot. + */ +static void boot_setup(void) +{ + char lpmsg[] = "LOADPARM=[________]\n"; + + sclp_get_loadparm_ascii(loadparm_str); + memcpy(lpmsg + 10, loadparm_str, 8); + sclp_print(lpmsg); + + have_iplb = store_iplb(&iplb); +} + static void virtio_setup(void) { Schib schib; int ssid; bool found = false; uint16_t dev_no; - char ldp[] = "LOADPARM=[________]\n"; VDev *vdev = virtio_get_device(); QemuIplParameters *early_qipl = (QemuIplParameters *)QIPL_ADDRESS; - sclp_get_loadparm_ascii(loadparm_str); - memcpy(ldp + 10, loadparm_str, LOADPARM_LEN); - sclp_print(ldp); - memcpy(&qipl, early_qipl, sizeof(QemuIplParameters)); - if (store_iplb(&iplb)) { + if (have_iplb) { switch (iplb.pbt) { case S390_IPL_TYPE_CCW: dev_no = iplb.ccw.devno; @@ -174,6 +185,7 @@ int main(void) { sclp_setup(); css_setup(); + boot_setup(); virtio_setup(); zipl_load(); /* no return */ From d96c5db77f1058ee9509554f43b945c66b3aa7c9 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:23 -0400 Subject: [PATCH 04/19] s390-bios: Clean up cio.h Add proper typedefs to all structs and modify all bit fields to use consistent formatting. Signed-off-by: Jason J. Herne Reviewed-by: Collin Walling Reviewed-by: Farhan Ali Acked-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-5-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/cio.h | 112 ++++++++++++++++++------------------ pc-bios/s390-ccw/s390-ccw.h | 8 --- 2 files changed, 56 insertions(+), 64 deletions(-) diff --git a/pc-bios/s390-ccw/cio.h b/pc-bios/s390-ccw/cio.h index 1a0795f645..ed5b2cbf72 100644 --- a/pc-bios/s390-ccw/cio.h +++ b/pc-bios/s390-ccw/cio.h @@ -17,35 +17,35 @@ * path management control word */ struct pmcw { - __u32 intparm; /* interruption parameter */ - __u32 qf : 1; /* qdio facility */ - __u32 w : 1; - __u32 isc : 3; /* interruption sublass */ - __u32 res5 : 3; /* reserved zeros */ - __u32 ena : 1; /* enabled */ - __u32 lm : 2; /* limit mode */ - __u32 mme : 2; /* measurement-mode enable */ - __u32 mp : 1; /* multipath mode */ - __u32 tf : 1; /* timing facility */ - __u32 dnv : 1; /* device number valid */ - __u32 dev : 16; /* device number */ - __u8 lpm; /* logical path mask */ - __u8 pnom; /* path not operational mask */ - __u8 lpum; /* last path used mask */ - __u8 pim; /* path installed mask */ - __u16 mbi; /* measurement-block index */ - __u8 pom; /* path operational mask */ - __u8 pam; /* path available mask */ - __u8 chpid[8]; /* CHPID 0-7 (if available) */ - __u32 unused1 : 8; /* reserved zeros */ - __u32 st : 3; /* subchannel type */ - __u32 unused2 : 18; /* reserved zeros */ - __u32 mbfc : 1; /* measurement block format control */ - __u32 xmwme : 1; /* extended measurement word mode enable */ - __u32 csense : 1; /* concurrent sense; can be enabled ...*/ - /* ... per MSCH, however, if facility */ - /* ... is not installed, this results */ - /* ... in an operand exception. */ + __u32 intparm; /* interruption parameter */ + __u32 qf:1; /* qdio facility */ + __u32 w:1; + __u32 isc:3; /* interruption sublass */ + __u32 res5:3; /* reserved zeros */ + __u32 ena:1; /* enabled */ + __u32 lm:2; /* limit mode */ + __u32 mme:2; /* measurement-mode enable */ + __u32 mp:1; /* multipath mode */ + __u32 tf:1; /* timing facility */ + __u32 dnv:1; /* device number valid */ + __u32 dev:16; /* device number */ + __u8 lpm; /* logical path mask */ + __u8 pnom; /* path not operational mask */ + __u8 lpum; /* last path used mask */ + __u8 pim; /* path installed mask */ + __u16 mbi; /* measurement-block index */ + __u8 pom; /* path operational mask */ + __u8 pam; /* path available mask */ + __u8 chpid[8]; /* CHPID 0-7 (if available) */ + __u32 unused1:8; /* reserved zeros */ + __u32 st:3; /* subchannel type */ + __u32 unused2:18; /* reserved zeros */ + __u32 mbfc:1; /* measurement block format control */ + __u32 xmwme:1; /* extended measurement word mode enable */ + __u32 csense:1; /* concurrent sense; can be enabled ...*/ + /* ... per MSCH, however, if facility */ + /* ... is not installed, this results */ + /* ... in an operand exception. */ } __attribute__ ((packed)); /* Target SCHIB configuration. */ @@ -77,28 +77,28 @@ struct scsw { /* * subchannel information block */ -struct schib { +typedef struct schib { struct pmcw pmcw; /* path management control word */ struct scsw scsw; /* subchannel status word */ __u64 mba; /* measurement block address */ __u8 mda[4]; /* model dependent area */ -} __attribute__ ((packed,aligned(4))); +} __attribute__ ((packed, aligned(4))) Schib; -struct subchannel_id { - __u32 cssid : 8; - __u32 : 4; - __u32 m : 1; - __u32 ssid : 2; - __u32 one : 1; - __u32 sch_no : 16; -} __attribute__ ((packed, aligned(4))); +typedef struct subchannel_id { + __u32 cssid:8; + __u32:4; + __u32 m:1; + __u32 ssid:2; + __u32 one:1; + __u32 sch_no:16; +} __attribute__ ((packed, aligned(4))) SubChannelId; struct chsc_header { __u16 length; __u16 code; } __attribute__((packed)); -struct chsc_area_sda { +typedef struct chsc_area_sda { struct chsc_header request; __u8 reserved1:4; __u8 format:4; @@ -111,29 +111,29 @@ struct chsc_area_sda { __u32 reserved5:4; __u32 format2:4; __u32 reserved6:24; -} __attribute__((packed)); +} __attribute__((packed)) ChscAreaSda; /* * TPI info structure */ struct tpi_info { struct subchannel_id schid; - __u32 intparm; /* interruption parameter */ - __u32 adapter_IO : 1; - __u32 reserved2 : 1; - __u32 isc : 3; - __u32 reserved3 : 12; - __u32 int_type : 3; - __u32 reserved4 : 12; + __u32 intparm; /* interruption parameter */ + __u32 adapter_IO:1; + __u32 reserved2:1; + __u32 isc:3; + __u32 reserved3:12; + __u32 int_type:3; + __u32 reserved4:12; } __attribute__ ((packed, aligned(4))); /* channel command word (type 1) */ -struct ccw1 { +typedef struct ccw1 { __u8 cmd_code; __u8 flags; __u16 count; __u32 cda; -} __attribute__ ((packed, aligned(8))); +} __attribute__ ((packed, aligned(8))) Ccw1; #define CCW_FLAG_DC 0x80 #define CCW_FLAG_CC 0x40 @@ -162,7 +162,7 @@ struct ccw1 { /* * Command-mode operation request block */ -struct cmd_orb { +typedef struct cmd_orb { __u32 intparm; /* interruption parameter */ __u32 key:4; /* flags, like key, suspend control, etc. */ __u32 spnd:1; /* suspend control */ @@ -182,7 +182,7 @@ struct cmd_orb { __u32 zero:6; /* reserved zeros */ __u32 orbx:1; /* ORB extension control */ __u32 cpa; /* channel program address */ -} __attribute__ ((packed, aligned(4))); +} __attribute__ ((packed, aligned(4))) CmdOrb; struct ciw { __u8 type; @@ -193,7 +193,7 @@ struct ciw { /* * sense-id response buffer layout */ -struct senseid { +typedef struct senseid { /* common part */ __u8 reserved; /* always 0x'FF' */ __u16 cu_type; /* control unit type */ @@ -203,15 +203,15 @@ struct senseid { __u8 unused; /* padding byte */ /* extended part */ struct ciw ciw[62]; -} __attribute__ ((packed, aligned(4))); +} __attribute__ ((packed, aligned(4))) SenseId; /* interruption response block */ -struct irb { +typedef struct irb { struct scsw scsw; __u32 esw[5]; __u32 ecw[8]; __u32 emw[8]; -} __attribute__ ((packed, aligned(4))); +} __attribute__ ((packed, aligned(4))) Irb; /* * Some S390 specific IO instructions as inline diff --git a/pc-bios/s390-ccw/s390-ccw.h b/pc-bios/s390-ccw/s390-ccw.h index 9828aa233d..241c6d0b69 100644 --- a/pc-bios/s390-ccw/s390-ccw.h +++ b/pc-bios/s390-ccw/s390-ccw.h @@ -49,14 +49,6 @@ typedef unsigned long long __u64; #include "cio.h" #include "iplb.h" -typedef struct irb Irb; -typedef struct ccw1 Ccw1; -typedef struct cmd_orb CmdOrb; -typedef struct schib Schib; -typedef struct chsc_area_sda ChscAreaSda; -typedef struct senseid SenseId; -typedef struct subchannel_id SubChannelId; - /* start.s */ void disabled_wait(void); void consume_sclp_int(void); From 120d04103e3f870d0fcd2a23c2ada0a4a4f036cc Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:24 -0400 Subject: [PATCH 05/19] s390-bios: Decouple channel i/o logic from virtio Create a separate library for channel i/o related code. This decouples channel i/o operations from virtio and allows us to make use of them for the real dasd boot path. Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-6-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/Makefile | 2 +- pc-bios/s390-ccw/cio.c | 44 ++++++++++++++++++++++++++++++++++++ pc-bios/s390-ccw/cio.h | 3 +++ pc-bios/s390-ccw/main.c | 1 + pc-bios/s390-ccw/netboot.mak | 2 +- pc-bios/s390-ccw/netmain.c | 1 + pc-bios/s390-ccw/s390-ccw.h | 1 - pc-bios/s390-ccw/virtio.c | 27 ++-------------------- 8 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 pc-bios/s390-ccw/cio.c diff --git a/pc-bios/s390-ccw/Makefile b/pc-bios/s390-ccw/Makefile index 1eb316b02f..12ad9c1d58 100644 --- a/pc-bios/s390-ccw/Makefile +++ b/pc-bios/s390-ccw/Makefile @@ -10,7 +10,7 @@ $(call set-vpath, $(SRC_PATH)/pc-bios/s390-ccw) .PHONY : all clean build-all OBJECTS = start.o main.o bootmap.o jump2ipl.o sclp.o menu.o \ - virtio.o virtio-scsi.o virtio-blkdev.o libc.o + virtio.o virtio-scsi.o virtio-blkdev.o libc.o cio.o QEMU_CFLAGS := $(filter -W%, $(QEMU_CFLAGS)) QEMU_CFLAGS += -ffreestanding -fno-delete-null-pointer-checks -msoft-float diff --git a/pc-bios/s390-ccw/cio.c b/pc-bios/s390-ccw/cio.c new file mode 100644 index 0000000000..87c6b34e88 --- /dev/null +++ b/pc-bios/s390-ccw/cio.c @@ -0,0 +1,44 @@ +/* + * S390 Channel I/O + * + * Copyright (c) 2013 Alexander Graf + * Copyright (c) 2019 IBM Corp. + * + * Author(s): Jason J. Herne + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#include "libc.h" +#include "s390-ccw.h" +#include "cio.h" + +static char chsc_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); + +int enable_mss_facility(void) +{ + int ret; + ChscAreaSda *sda_area = (ChscAreaSda *) chsc_page; + + memset(sda_area, 0, PAGE_SIZE); + sda_area->request.length = 0x0400; + sda_area->request.code = 0x0031; + sda_area->operation_code = 0x2; + + ret = chsc(sda_area); + if ((ret == 0) && (sda_area->response.code == 0x0001)) { + return 0; + } + return -EIO; +} + +void enable_subchannel(SubChannelId schid) +{ + Schib schib; + + stsch_err(schid, &schib); + schib.pmcw.ena = 1; + msch(schid, &schib); +} diff --git a/pc-bios/s390-ccw/cio.h b/pc-bios/s390-ccw/cio.h index ed5b2cbf72..218fd96ea3 100644 --- a/pc-bios/s390-ccw/cio.h +++ b/pc-bios/s390-ccw/cio.h @@ -213,6 +213,9 @@ typedef struct irb { __u32 emw[8]; } __attribute__ ((packed, aligned(4))) Irb; +int enable_mss_facility(void); +void enable_subchannel(SubChannelId schid); + /* * Some S390 specific IO instructions as inline */ diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index 67df42191b..10f04c6906 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -10,6 +10,7 @@ #include "libc.h" #include "s390-ccw.h" +#include "cio.h" #include "virtio.h" char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE))); diff --git a/pc-bios/s390-ccw/netboot.mak b/pc-bios/s390-ccw/netboot.mak index 14e96b2aa6..5eefb7c289 100644 --- a/pc-bios/s390-ccw/netboot.mak +++ b/pc-bios/s390-ccw/netboot.mak @@ -1,7 +1,7 @@ SLOF_DIR := $(SRC_PATH)/roms/SLOF -NETOBJS := start.o sclp.o virtio.o virtio-net.o jump2ipl.o netmain.o \ +NETOBJS := start.o sclp.o cio.o virtio.o virtio-net.o jump2ipl.o netmain.o \ libnet.a libc.a LIBC_INC := -nostdinc -I$(SLOF_DIR)/lib/libc/include diff --git a/pc-bios/s390-ccw/netmain.c b/pc-bios/s390-ccw/netmain.c index 0392131c27..5189c0fc39 100644 --- a/pc-bios/s390-ccw/netmain.c +++ b/pc-bios/s390-ccw/netmain.c @@ -33,6 +33,7 @@ #include #include "s390-ccw.h" +#include "cio.h" #include "virtio.h" #define DEFAULT_BOOT_RETRIES 10 diff --git a/pc-bios/s390-ccw/s390-ccw.h b/pc-bios/s390-ccw/s390-ccw.h index 241c6d0b69..b39ee5d323 100644 --- a/pc-bios/s390-ccw/s390-ccw.h +++ b/pc-bios/s390-ccw/s390-ccw.h @@ -72,7 +72,6 @@ unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2, bool virtio_is_supported(SubChannelId schid); void virtio_blk_setup_device(SubChannelId schid); int virtio_read(ulong sector, void *load_addr); -int enable_mss_facility(void); u64 get_clock(void); ulong get_second(void); diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c index cdb66f459e..aa9da7253f 100644 --- a/pc-bios/s390-ccw/virtio.c +++ b/pc-bios/s390-ccw/virtio.c @@ -10,6 +10,7 @@ #include "libc.h" #include "s390-ccw.h" +#include "cio.h" #include "virtio.h" #include "virtio-scsi.h" #include "bswap.h" @@ -20,8 +21,6 @@ static VRing block[VIRTIO_MAX_VQS]; static char ring_area[VIRTIO_RING_SIZE * VIRTIO_MAX_VQS] __attribute__((__aligned__(PAGE_SIZE))); -static char chsc_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); - static VDev vdev = { .nr_vqs = 1, .vrings = block, @@ -94,14 +93,9 @@ static int run_ccw(VDev *vdev, int cmd, void *ptr, int len) { Ccw1 ccw = {}; CmdOrb orb = {}; - Schib schib; int r; - /* start command processing */ - stsch_err(vdev->schid, &schib); - /* enable the subchannel for IPL device */ - schib.pmcw.ena = 1; - msch(vdev->schid, &schib); + enable_subchannel(vdev->schid); /* start subchannel command */ orb.fmt = 1; @@ -343,20 +337,3 @@ bool virtio_is_supported(SubChannelId schid) } return false; } - -int enable_mss_facility(void) -{ - int ret; - ChscAreaSda *sda_area = (ChscAreaSda *) chsc_page; - - memset(sda_area, 0, PAGE_SIZE); - sda_area->request.length = 0x0400; - sda_area->request.code = 0x0031; - sda_area->operation_code = 0x2; - - ret = chsc(sda_area); - if ((ret == 0) && (sda_area->response.code == 0x0001)) { - return 0; - } - return -EIO; -} From c95df3d108028ff5a709ee3aefdb14401b07cb39 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:25 -0400 Subject: [PATCH 06/19] s390-bios: Map low core memory Create a new header for basic architecture specific definitions and add a mapping of low core memory. This mapping will be used by the real dasd boot process. Signed-off-by: Jason J. Herne Acked-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-7-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 2 + pc-bios/s390-ccw/s390-arch.h | 90 ++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 pc-bios/s390-ccw/s390-arch.h diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index 10f04c6906..e403b5f733 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -9,6 +9,7 @@ */ #include "libc.h" +#include "s390-arch.h" #include "s390-ccw.h" #include "cio.h" #include "virtio.h" @@ -19,6 +20,7 @@ static char loadparm_str[LOADPARM_LEN + 1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; QemuIplParameters qipl; IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE))); static bool have_iplb; +LowCore const *lowcore; /* Yes, this *is* a pointer to address 0 */ #define LOADPARM_PROMPT "PROMPT " #define LOADPARM_EMPTY " " diff --git a/pc-bios/s390-ccw/s390-arch.h b/pc-bios/s390-ccw/s390-arch.h new file mode 100644 index 0000000000..5e92c7a27d --- /dev/null +++ b/pc-bios/s390-ccw/s390-arch.h @@ -0,0 +1,90 @@ +/* + * S390 Basic Architecture + * + * Copyright (c) 2019 Jason J. Herne + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#ifndef S390_ARCH_H +#define S390_ARCH_H + +typedef struct PSW { + uint64_t mask; + uint64_t addr; +} __attribute__ ((aligned(8))) PSW; +_Static_assert(sizeof(struct PSW) == 16, "PSW size incorrect"); + +/* Older PSW format used by LPSW instruction */ +typedef struct PSWLegacy { + uint32_t mask; + uint32_t addr; +} __attribute__ ((aligned(8))) PSWLegacy; +_Static_assert(sizeof(struct PSWLegacy) == 8, "PSWLegacy size incorrect"); + +/* s390 psw bit masks */ +#define PSW_MASK_IOINT 0x0200000000000000ULL +#define PSW_MASK_WAIT 0x0002000000000000ULL +#define PSW_MASK_EAMODE 0x0000000100000000ULL +#define PSW_MASK_BAMODE 0x0000000080000000ULL +#define PSW_MASK_ZMODE (PSW_MASK_EAMODE | PSW_MASK_BAMODE) + +/* Low core mapping */ +typedef struct LowCore { + /* prefix area: defined by architecture */ + PSWLegacy ipl_psw; /* 0x000 */ + uint32_t ccw1[2]; /* 0x008 */ + uint32_t ccw2[2]; /* 0x010 */ + uint8_t pad1[0x80 - 0x18]; /* 0x018 */ + uint32_t ext_params; /* 0x080 */ + uint16_t cpu_addr; /* 0x084 */ + uint16_t ext_int_code; /* 0x086 */ + uint16_t svc_ilen; /* 0x088 */ + uint16_t svc_code; /* 0x08a */ + uint16_t pgm_ilen; /* 0x08c */ + uint16_t pgm_code; /* 0x08e */ + uint32_t data_exc_code; /* 0x090 */ + uint16_t mon_class_num; /* 0x094 */ + uint16_t per_perc_atmid; /* 0x096 */ + uint64_t per_address; /* 0x098 */ + uint8_t exc_access_id; /* 0x0a0 */ + uint8_t per_access_id; /* 0x0a1 */ + uint8_t op_access_id; /* 0x0a2 */ + uint8_t ar_access_id; /* 0x0a3 */ + uint8_t pad2[0xA8 - 0xA4]; /* 0x0a4 */ + uint64_t trans_exc_code; /* 0x0a8 */ + uint64_t monitor_code; /* 0x0b0 */ + uint16_t subchannel_id; /* 0x0b8 */ + uint16_t subchannel_nr; /* 0x0ba */ + uint32_t io_int_parm; /* 0x0bc */ + uint32_t io_int_word; /* 0x0c0 */ + uint8_t pad3[0xc8 - 0xc4]; /* 0x0c4 */ + uint32_t stfl_fac_list; /* 0x0c8 */ + uint8_t pad4[0xe8 - 0xcc]; /* 0x0cc */ + uint64_t mcic; /* 0x0e8 */ + uint8_t pad5[0xf4 - 0xf0]; /* 0x0f0 */ + uint32_t external_damage_code; /* 0x0f4 */ + uint64_t failing_storage_address; /* 0x0f8 */ + uint8_t pad6[0x110 - 0x100]; /* 0x100 */ + uint64_t per_breaking_event_addr; /* 0x110 */ + uint8_t pad7[0x120 - 0x118]; /* 0x118 */ + PSW restart_old_psw; /* 0x120 */ + PSW external_old_psw; /* 0x130 */ + PSW svc_old_psw; /* 0x140 */ + PSW program_old_psw; /* 0x150 */ + PSW mcck_old_psw; /* 0x160 */ + PSW io_old_psw; /* 0x170 */ + uint8_t pad8[0x1a0 - 0x180]; /* 0x180 */ + PSW restart_new_psw; /* 0x1a0 */ + PSW external_new_psw; /* 0x1b0 */ + PSW svc_new_psw; /* 0x1c0 */ + PSW program_new_psw; /* 0x1d0 */ + PSW mcck_new_psw; /* 0x1e0 */ + PSW io_new_psw; /* 0x1f0 */ +} __attribute__((packed, aligned(8192))) LowCore; + +extern LowCore const *lowcore; + +#endif From 1fb3e5cde8dcd9b5917aea9a0b2918e16be8be1e Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:26 -0400 Subject: [PATCH 07/19] s390-bios: ptr2u32 and u32toptr Introduce inline functions to convert between pointers and unsigned 32-bit ints. These are used to hide the ugliness required to avoid compiler warnings. Signed-off-by: Jason J. Herne Acked-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-8-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/helper.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pc-bios/s390-ccw/helper.h diff --git a/pc-bios/s390-ccw/helper.h b/pc-bios/s390-ccw/helper.h new file mode 100644 index 0000000000..78d5bc7442 --- /dev/null +++ b/pc-bios/s390-ccw/helper.h @@ -0,0 +1,31 @@ +/* + * Helper Functions + * + * Copyright (c) 2019 IBM Corp. + * + * Author(s): Jason J. Herne + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#ifndef S390_CCW_HELPER_H +#define S390_CCW_HELPER_H + +#include "s390-ccw.h" + +/* Avoids compiler warnings when casting a pointer to a u32 */ +static inline uint32_t ptr2u32(void *ptr) +{ + IPL_assert((uint64_t)ptr <= 0xffffffff, "ptr2u32: ptr too large"); + return (uint32_t)(uint64_t)ptr; +} + +/* Avoids compiler warnings when casting a u32 to a pointer */ +static inline void *u32toptr(uint32_t n) +{ + return (void *)(uint64_t)n; +} + +#endif From 3083a1bbb8716e9052fe375f68f330107ee13127 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:27 -0400 Subject: [PATCH 08/19] s390-bios: Support for running format-0/1 channel programs Introduce a library function for executing format-0 and format-1 channel programs and waiting for their completion before continuing execution. Add cu_type() to channel io library. This will be used to query control unit type which is used to determine if we are booting a virtio device or a real dasd device. Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Reviewed-by: Farhan Ali Message-Id: <1554388475-18329-9-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/cio.c | 144 ++++++++++++++++++++++++++++++++++++ pc-bios/s390-ccw/cio.h | 130 +++++++++++++++++++++++++++++++- pc-bios/s390-ccw/s390-ccw.h | 1 + pc-bios/s390-ccw/start.S | 29 ++++++++ 4 files changed, 301 insertions(+), 3 deletions(-) diff --git a/pc-bios/s390-ccw/cio.c b/pc-bios/s390-ccw/cio.c index 87c6b34e88..c43e50b00b 100644 --- a/pc-bios/s390-ccw/cio.c +++ b/pc-bios/s390-ccw/cio.c @@ -13,10 +13,14 @@ #include "libc.h" #include "s390-ccw.h" +#include "s390-arch.h" +#include "helper.h" #include "cio.h" static char chsc_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); +static int __do_cio(SubChannelId schid, uint32_t ccw_addr, int fmt, Irb *irb); + int enable_mss_facility(void) { int ret; @@ -42,3 +46,143 @@ void enable_subchannel(SubChannelId schid) schib.pmcw.ena = 1; msch(schid, &schib); } + +uint16_t cu_type(SubChannelId schid) +{ + Ccw1 sense_id_ccw; + SenseId sense_data; + + sense_id_ccw.cmd_code = CCW_CMD_SENSE_ID; + sense_id_ccw.cda = ptr2u32(&sense_data); + sense_id_ccw.count = sizeof(sense_data); + sense_id_ccw.flags |= CCW_FLAG_SLI; + + if (do_cio(schid, CU_TYPE_UNKNOWN, ptr2u32(&sense_id_ccw), CCW_FMT1)) { + panic("Failed to run SenseID CCw\n"); + } + + return sense_data.cu_type; +} + +int basic_sense(SubChannelId schid, uint16_t cutype, void *sense_data, + uint16_t data_size) +{ + Ccw1 senseCcw; + Irb irb; + + senseCcw.cmd_code = CCW_CMD_BASIC_SENSE; + senseCcw.cda = ptr2u32(sense_data); + senseCcw.count = data_size; + + return __do_cio(schid, ptr2u32(&senseCcw), CCW_FMT1, &irb); +} + +static bool irb_error(Irb *irb) +{ + if (irb->scsw.cstat) { + return true; + } + return irb->scsw.dstat != (SCSW_DSTAT_DEVEND | SCSW_DSTAT_CHEND); +} + +/* + * Handles executing ssch, tsch and returns the irb obtained from tsch. + * Returns 0 on success, -1 if unexpected status pending and we need to retry, + * otherwise returns condition code from ssch/tsch for error cases. + */ +static int __do_cio(SubChannelId schid, uint32_t ccw_addr, int fmt, Irb *irb) +{ + CmdOrb orb = {}; + int rc; + + IPL_assert(fmt == 0 || fmt == 1, "Invalid ccw format"); + + /* ccw_addr must be <= 24 bits and point to at least one whole ccw. */ + if (fmt == 0) { + IPL_assert(ccw_addr <= 0xFFFFFF - 8, "Invalid ccw address"); + } + + orb.fmt = fmt; + orb.pfch = 1; /* QEMU's cio implementation requires prefetch */ + orb.c64 = 1; /* QEMU's cio implementation requires 64-bit idaws */ + orb.lpm = 0xFF; /* All paths allowed */ + orb.cpa = ccw_addr; + + rc = ssch(schid, &orb); + if (rc == 1 || rc == 2) { + /* Subchannel status pending or busy. Eat status and ask for retry. */ + tsch(schid, irb); + return -1; + } + if (rc) { + print_int("ssch failed with cc=", rc); + return rc; + } + + consume_io_int(); + + /* collect status */ + rc = tsch(schid, irb); + if (rc) { + print_int("tsch failed with cc=", rc); + } + + return rc; +} + +/* + * Executes a channel program at a given subchannel. The request to run the + * channel program is sent to the subchannel, we then wait for the interrupt + * signaling completion of the I/O operation(s) performed by the channel + * program. Lastly we verify that the i/o operation completed without error and + * that the interrupt we received was for the subchannel used to run the + * channel program. + * + * Note: This function assumes it is running in an environment where no other + * cpus are generating or receiving I/O interrupts. So either run it in a + * single-cpu environment or make sure all other cpus are not doing I/O and + * have I/O interrupts masked off. We also assume that only one device is + * active (generating i/o interrupts). + * + * Returns non-zero on error. + */ +int do_cio(SubChannelId schid, uint16_t cutype, uint32_t ccw_addr, int fmt) +{ + Irb irb = {}; + SenseDataEckdDasd sd; + int rc, retries = 0; + + while (true) { + rc = __do_cio(schid, ccw_addr, fmt, &irb); + + if (rc == -1) { + retries++; + continue; + } + if (rc) { + /* ssch/tsch error. Message already reported by __do_cio */ + break; + } + + if (!irb_error(&irb)) { + break; + } + + /* + * Unexpected unit check, or interface-control-check. Use sense to + * clear (unit check only) then retry. + */ + if ((unit_check(&irb) || iface_ctrl_check(&irb)) && retries <= 2) { + if (unit_check(&irb)) { + basic_sense(schid, cutype, &sd, sizeof(sd)); + } + retries++; + continue; + } + + rc = -1; + break; + } + + return rc; +} diff --git a/pc-bios/s390-ccw/cio.h b/pc-bios/s390-ccw/cio.h index 218fd96ea3..1637e32070 100644 --- a/pc-bios/s390-ccw/cio.h +++ b/pc-bios/s390-ccw/cio.h @@ -70,9 +70,46 @@ struct scsw { __u16 count; } __attribute__ ((packed)); -#define SCSW_FCTL_CLEAR_FUNC 0x1000 -#define SCSW_FCTL_HALT_FUNC 0x2000 +/* Function Control */ #define SCSW_FCTL_START_FUNC 0x4000 +#define SCSW_FCTL_HALT_FUNC 0x2000 +#define SCSW_FCTL_CLEAR_FUNC 0x1000 + +/* Activity Control */ +#define SCSW_ACTL_RESUME_PEND 0x0800 +#define SCSW_ACTL_START_PEND 0x0400 +#define SCSW_ACTL_HALT_PEND 0x0200 +#define SCSW_ACTL_CLEAR_PEND 0x0100 +#define SCSW_ACTL_CH_ACTIVE 0x0080 +#define SCSW_ACTL_DEV_ACTIVE 0x0040 +#define SCSW_ACTL_SUSPENDED 0x0020 + +/* Status Control */ +#define SCSW_SCTL_ALERT 0x0010 +#define SCSW_SCTL_INTERMED 0x0008 +#define SCSW_SCTL_PRIMARY 0x0004 +#define SCSW_SCTL_SECONDARY 0x0002 +#define SCSW_SCTL_STATUS_PEND 0x0001 + +/* SCSW Device Status Flags */ +#define SCSW_DSTAT_ATTN 0x80 +#define SCSW_DSTAT_STATMOD 0x40 +#define SCSW_DSTAT_CUEND 0x20 +#define SCSW_DSTAT_BUSY 0x10 +#define SCSW_DSTAT_CHEND 0x08 +#define SCSW_DSTAT_DEVEND 0x04 +#define SCSW_DSTAT_UCHK 0x02 +#define SCSW_DSTAT_UEXCP 0x01 + +/* SCSW Subchannel Status Flags */ +#define SCSW_CSTAT_PCINT 0x80 +#define SCSW_CSTAT_BADLEN 0x40 +#define SCSW_CSTAT_PROGCHK 0x20 +#define SCSW_CSTAT_PROTCHK 0x10 +#define SCSW_CSTAT_CHDCHK 0x08 +#define SCSW_CSTAT_CHCCHK 0x04 +#define SCSW_CSTAT_ICCHK 0x02 +#define SCSW_CSTAT_CHAINCHK 0x01 /* * subchannel information block @@ -127,7 +164,23 @@ struct tpi_info { __u32 reserved4:12; } __attribute__ ((packed, aligned(4))); -/* channel command word (type 1) */ +/* channel command word (format 0) */ +typedef struct ccw0 { + __u8 cmd_code; + __u32 cda:24; + __u32 chainData:1; + __u32 chain:1; + __u32 sli:1; + __u32 skip:1; + __u32 pci:1; + __u32 ida:1; + __u32 suspend:1; + __u32 mida:1; + __u8 reserved; + __u16 count; +} __attribute__ ((packed, aligned(8))) Ccw0; + +/* channel command word (format 1) */ typedef struct ccw1 { __u8 cmd_code; __u8 flags; @@ -135,6 +188,10 @@ typedef struct ccw1 { __u32 cda; } __attribute__ ((packed, aligned(8))) Ccw1; +/* do_cio() CCW formats */ +#define CCW_FMT0 0x00 +#define CCW_FMT1 0x01 + #define CCW_FLAG_DC 0x80 #define CCW_FLAG_CC 0x40 #define CCW_FLAG_SLI 0x20 @@ -190,6 +247,11 @@ struct ciw { __u16 count; }; +#define CU_TYPE_UNKNOWN 0x0000 +#define CU_TYPE_DASD_2107 0x2107 +#define CU_TYPE_VIRTIO 0x3832 +#define CU_TYPE_DASD_3990 0x3990 + /* * sense-id response buffer layout */ @@ -205,6 +267,64 @@ typedef struct senseid { struct ciw ciw[62]; } __attribute__ ((packed, aligned(4))) SenseId; +/* + * architected values for first sense byte - common_status. Bits 0-5 of this + * field are common to all device types. + */ +#define SNS_STAT0_CMD_REJECT 0x80 +#define SNS_STAT0_INTERVENTION_REQ 0x40 +#define SNS_STAT0_BUS_OUT_CHECK 0x20 +#define SNS_STAT0_EQUIPMENT_CHECK 0x10 +#define SNS_STAT0_DATA_CHECK 0x08 +#define SNS_STAT0_OVERRUN 0x04 +#define SNS_STAT0_INCOMPL_DOMAIN 0x01 + +/* ECKD DASD status[0] byte */ +#define SNS_STAT1_PERM_ERR 0x80 +#define SNS_STAT1_INV_TRACK_FORMAT 0x40 +#define SNS_STAT1_EOC 0x20 +#define SNS_STAT1_MESSAGE_TO_OPER 0x10 +#define SNS_STAT1_NO_REC_FOUND 0x08 +#define SNS_STAT1_FILE_PROTECTED 0x04 +#define SNS_STAT1_WRITE_INHIBITED 0x02 +#define SNS_STAT1_IMPRECISE_END 0x01 + +/* ECKD DASD status[1] byte */ +#define SNS_STAT2_REQ_INH_WRITE 0x80 +#define SNS_STAT2_CORRECTABLE 0x40 +#define SNS_STAT2_FIRST_LOG_ERR 0x20 +#define SNS_STAT2_ENV_DATA_PRESENT 0x10 +#define SNS_STAT2_IMPRECISE_END 0x04 + +/* ECKD DASD 24-byte Sense fmt_msg codes */ +#define SENSE24_FMT_PROG_SYS 0x0 +#define SENSE24_FMT_EQUIPMENT 0x2 +#define SENSE24_FMT_CONTROLLER 0x3 +#define SENSE24_FMT_MISC 0xF + +/* basic sense response buffer layout */ +typedef struct SenseDataEckdDasd { + uint8_t common_status; + uint8_t status[2]; + uint8_t res_count; + uint8_t phys_drive_id; + uint8_t low_cyl_addr; + uint8_t head_high_cyl_addr; + uint8_t fmt_msg; + uint64_t fmt_dependent_info[2]; + uint8_t reserved; + uint8_t program_action_code; + uint16_t config_info; + uint8_t mcode_hicyl; + uint8_t cyl_head_addr[3]; +} __attribute__ ((packed, aligned(4))) SenseDataEckdDasd; + +#define ECKD_SENSE24_GET_FMT(sd) (sd->fmt_msg & 0xF0 >> 4) +#define ECKD_SENSE24_GET_MSG(sd) (sd->fmt_msg & 0x0F) + +#define unit_check(irb) ((irb)->scsw.dstat & SCSW_DSTAT_UCHK) +#define iface_ctrl_check(irb) ((irb)->scsw.cstat & SCSW_CSTAT_ICCHK) + /* interruption response block */ typedef struct irb { struct scsw scsw; @@ -215,6 +335,10 @@ typedef struct irb { int enable_mss_facility(void); void enable_subchannel(SubChannelId schid); +uint16_t cu_type(SubChannelId schid); +int basic_sense(SubChannelId schid, uint16_t cutype, void *sense_data, + uint16_t data_size); +int do_cio(SubChannelId schid, uint16_t cutype, uint32_t ccw_addr, int fmt); /* * Some S390 specific IO instructions as inline diff --git a/pc-bios/s390-ccw/s390-ccw.h b/pc-bios/s390-ccw/s390-ccw.h index b39ee5d323..11bce7d73c 100644 --- a/pc-bios/s390-ccw/s390-ccw.h +++ b/pc-bios/s390-ccw/s390-ccw.h @@ -52,6 +52,7 @@ typedef unsigned long long __u64; /* start.s */ void disabled_wait(void); void consume_sclp_int(void); +void consume_io_int(void); /* main.c */ void panic(const char *string); diff --git a/pc-bios/s390-ccw/start.S b/pc-bios/s390-ccw/start.S index 5c22cb0849..aa8fceb19d 100644 --- a/pc-bios/s390-ccw/start.S +++ b/pc-bios/s390-ccw/start.S @@ -71,6 +71,26 @@ consume_sclp_int: larl %r1, enabled_wait_psw lpswe 0(%r1) +/* + * void consume_io_int(void) + * + * eats one I/O interrupt + */ + .globl consume_io_int +consume_io_int: + /* enable I/O interrupts in cr6 */ + stctg %c6,%c6,0(%r15) + oi 4(%r15), 0xff + lctlg %c6,%c6,0(%r15) + /* prepare i/o call handler */ + larl %r1, io_new_code + stg %r1, 0x1f8 + larl %r1, io_new_mask + mvc 0x1f0(8),0(%r1) + /* load enabled wait PSW */ + larl %r1, enabled_wait_psw + lpswe 0(%r1) + external_new_code: /* disable service interrupts in cr0 */ stctg %c0,%c0,0(%r15) @@ -78,6 +98,13 @@ external_new_code: lctlg %c0,%c0,0(%r15) br %r14 +io_new_code: + /* disable I/O interrupts in cr6 */ + stctg %c6,%c6,0(%r15) + ni 4(%r15), 0x00 + lctlg %c6,%c6,0(%r15) + br %r14 + .align 8 disabled_wait_psw: .quad 0x0002000180000000,0x0000000000000000 @@ -85,3 +112,5 @@ enabled_wait_psw: .quad 0x0302000180000000,0x0000000000000000 external_new_mask: .quad 0x0000000180000000 +io_new_mask: + .quad 0x0000000180000000 From 86c58705bb186cfa73a03851047da2c2c37b9418 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:28 -0400 Subject: [PATCH 09/19] s390-bios: cio error handling Add verbose error output for when unexpected i/o errors happen. This eases the burden of debugging and reporting i/o errors. No error information is printed in the success case, here is an example of what is output on error: cio device error ssid : 0x0000000000000000 cssid : 0x0000000000000000 sch_no: 0x0000000000000000 Interrupt Response Block Data: Function Ctrl : [Start] Activity Ctrl : [Start-Pending] Status Ctrl : [Alert] [Primary] [Secondary] [Status-Pending] Device Status : [Unit-Check] Channel Status : cpa=: 0x000000007f8d6038 prev_ccw=: 0x0000000000000000 this_ccw=: 0x0000000000000000 Eckd Dasd Sense Data (fmt 32-bytes): Sense Condition Flags : Residual Count =: 0x0000000000000000 Phys Drive ID =: 0x000000000000009e low cyl address =: 0x0000000000000000 head addr & hi cyl =: 0x0000000000000000 format/message =: 0x0000000000000008 fmt-dependent[0-7] =: 0x0000000000000004 fmt-dependent[8-15]=: 0xe561282305082fff prog action code =: 0x0000000000000016 Configuration info =: 0x00000000000040e0 mcode / hi-cyl =: 0x0000000000000000 cyl & head addr [0]=: 0x0000000000000000 cyl & head addr [1]=: 0x0000000000000000 cyl & head addr [2]=: 0x0000000000000000 The Sense Data section is currently only printed for ECKD DASD. Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-10-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/cio.c | 235 ++++++++++++++++++++++++++++++++++++++++ pc-bios/s390-ccw/libc.h | 11 ++ 2 files changed, 246 insertions(+) diff --git a/pc-bios/s390-ccw/cio.c b/pc-bios/s390-ccw/cio.c index c43e50b00b..339ec5fbe7 100644 --- a/pc-bios/s390-ccw/cio.c +++ b/pc-bios/s390-ccw/cio.c @@ -85,6 +85,228 @@ static bool irb_error(Irb *irb) return irb->scsw.dstat != (SCSW_DSTAT_DEVEND | SCSW_DSTAT_CHEND); } +static void print_eckd_dasd_sense_data(SenseDataEckdDasd *sd) +{ + char msgline[512]; + + if (sd->config_info & 0x8000) { + sclp_print("Eckd Dasd Sense Data (fmt 24-bytes):\n"); + } else { + sclp_print("Eckd Dasd Sense Data (fmt 32-bytes):\n"); + } + + strcat(msgline, " Sense Condition Flags :"); + if (sd->common_status & SNS_STAT0_CMD_REJECT) { + strcat(msgline, " [Cmd-Reject]"); + } + if (sd->common_status & SNS_STAT0_INTERVENTION_REQ) { + strcat(msgline, " [Intervention-Required]"); + } + if (sd->common_status & SNS_STAT0_BUS_OUT_CHECK) { + strcat(msgline, " [Bus-Out-Parity-Check]"); + } + if (sd->common_status & SNS_STAT0_EQUIPMENT_CHECK) { + strcat(msgline, " [Equipment-Check]"); + } + if (sd->common_status & SNS_STAT0_DATA_CHECK) { + strcat(msgline, " [Data-Check]"); + } + if (sd->common_status & SNS_STAT0_OVERRUN) { + strcat(msgline, " [Overrun]"); + } + if (sd->common_status & SNS_STAT0_INCOMPL_DOMAIN) { + strcat(msgline, " [Incomplete-Domain]"); + } + + if (sd->status[0] & SNS_STAT1_PERM_ERR) { + strcat(msgline, " [Permanent-Error]"); + } + if (sd->status[0] & SNS_STAT1_INV_TRACK_FORMAT) { + strcat(msgline, " [Invalid-Track-Fmt]"); + } + if (sd->status[0] & SNS_STAT1_EOC) { + strcat(msgline, " [End-of-Cyl]"); + } + if (sd->status[0] & SNS_STAT1_MESSAGE_TO_OPER) { + strcat(msgline, " [Operator-Msg]"); + } + if (sd->status[0] & SNS_STAT1_NO_REC_FOUND) { + strcat(msgline, " [No-Record-Found]"); + } + if (sd->status[0] & SNS_STAT1_FILE_PROTECTED) { + strcat(msgline, " [File-Protected]"); + } + if (sd->status[0] & SNS_STAT1_WRITE_INHIBITED) { + strcat(msgline, " [Write-Inhibited]"); + } + if (sd->status[0] & SNS_STAT1_IMPRECISE_END) { + strcat(msgline, " [Imprecise-Ending]"); + } + + if (sd->status[1] & SNS_STAT2_REQ_INH_WRITE) { + strcat(msgline, " [Req-Inhibit-Write]"); + } + if (sd->status[1] & SNS_STAT2_CORRECTABLE) { + strcat(msgline, " [Correctable-Data-Check]"); + } + if (sd->status[1] & SNS_STAT2_FIRST_LOG_ERR) { + strcat(msgline, " [First-Error-Log]"); + } + if (sd->status[1] & SNS_STAT2_ENV_DATA_PRESENT) { + strcat(msgline, " [Env-Data-Present]"); + } + if (sd->status[1] & SNS_STAT2_IMPRECISE_END) { + strcat(msgline, " [Imprecise-End]"); + } + strcat(msgline, "\n"); + sclp_print(msgline); + + print_int(" Residual Count =", sd->res_count); + print_int(" Phys Drive ID =", sd->phys_drive_id); + print_int(" low cyl address =", sd->low_cyl_addr); + print_int(" head addr & hi cyl =", sd->head_high_cyl_addr); + print_int(" format/message =", sd->fmt_msg); + print_int(" fmt-dependent[0-7] =", sd->fmt_dependent_info[0]); + print_int(" fmt-dependent[8-15]=", sd->fmt_dependent_info[1]); + print_int(" prog action code =", sd->program_action_code); + print_int(" Configuration info =", sd->config_info); + print_int(" mcode / hi-cyl =", sd->mcode_hicyl); + print_int(" cyl & head addr [0]=", sd->cyl_head_addr[0]); + print_int(" cyl & head addr [1]=", sd->cyl_head_addr[1]); + print_int(" cyl & head addr [2]=", sd->cyl_head_addr[2]); +} + +static void print_irb_err(Irb *irb) +{ + uint64_t this_ccw = *(uint64_t *)u32toptr(irb->scsw.cpa); + uint64_t prev_ccw = *(uint64_t *)u32toptr(irb->scsw.cpa - 8); + char msgline[256]; + + sclp_print("Interrupt Response Block Data:\n"); + + strcat(msgline, " Function Ctrl :"); + if (irb->scsw.ctrl & SCSW_FCTL_START_FUNC) { + strcat(msgline, " [Start]"); + } + if (irb->scsw.ctrl & SCSW_FCTL_HALT_FUNC) { + strcat(msgline, " [Halt]"); + } + if (irb->scsw.ctrl & SCSW_FCTL_CLEAR_FUNC) { + strcat(msgline, " [Clear]"); + } + strcat(msgline, "\n"); + sclp_print(msgline); + + msgline[0] = '\0'; + strcat(msgline, " Activity Ctrl :"); + if (irb->scsw.ctrl & SCSW_ACTL_RESUME_PEND) { + strcat(msgline, " [Resume-Pending]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_START_PEND) { + strcat(msgline, " [Start-Pending]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_HALT_PEND) { + strcat(msgline, " [Halt-Pending]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_CLEAR_PEND) { + strcat(msgline, " [Clear-Pending]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_CH_ACTIVE) { + strcat(msgline, " [Channel-Active]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_DEV_ACTIVE) { + strcat(msgline, " [Device-Active]"); + } + if (irb->scsw.ctrl & SCSW_ACTL_SUSPENDED) { + strcat(msgline, " [Suspended]"); + } + strcat(msgline, "\n"); + sclp_print(msgline); + + msgline[0] = '\0'; + strcat(msgline, " Status Ctrl :"); + if (irb->scsw.ctrl & SCSW_SCTL_ALERT) { + strcat(msgline, " [Alert]"); + } + if (irb->scsw.ctrl & SCSW_SCTL_INTERMED) { + strcat(msgline, " [Intermediate]"); + } + if (irb->scsw.ctrl & SCSW_SCTL_PRIMARY) { + strcat(msgline, " [Primary]"); + } + if (irb->scsw.ctrl & SCSW_SCTL_SECONDARY) { + strcat(msgline, " [Secondary]"); + } + if (irb->scsw.ctrl & SCSW_SCTL_STATUS_PEND) { + strcat(msgline, " [Status-Pending]"); + } + + strcat(msgline, "\n"); + sclp_print(msgline); + + msgline[0] = '\0'; + strcat(msgline, " Device Status :"); + if (irb->scsw.dstat & SCSW_DSTAT_ATTN) { + strcat(msgline, " [Attention]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_STATMOD) { + strcat(msgline, " [Status-Modifier]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_CUEND) { + strcat(msgline, " [Ctrl-Unit-End]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_BUSY) { + strcat(msgline, " [Busy]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_CHEND) { + strcat(msgline, " [Channel-End]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_DEVEND) { + strcat(msgline, " [Device-End]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_UCHK) { + strcat(msgline, " [Unit-Check]"); + } + if (irb->scsw.dstat & SCSW_DSTAT_UEXCP) { + strcat(msgline, " [Unit-Exception]"); + } + strcat(msgline, "\n"); + sclp_print(msgline); + + msgline[0] = '\0'; + strcat(msgline, " Channel Status :"); + if (irb->scsw.cstat & SCSW_CSTAT_PCINT) { + strcat(msgline, " [Program-Ctrl-Interruption]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_BADLEN) { + strcat(msgline, " [Incorrect-Length]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_PROGCHK) { + strcat(msgline, " [Program-Check]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_PROTCHK) { + strcat(msgline, " [Protection-Check]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_CHDCHK) { + strcat(msgline, " [Channel-Data-Check]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_CHCCHK) { + strcat(msgline, " [Channel-Ctrl-Check]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_ICCHK) { + strcat(msgline, " [Interface-Ctrl-Check]"); + } + if (irb->scsw.cstat & SCSW_CSTAT_CHAINCHK) { + strcat(msgline, " [Chaining-Check]"); + } + strcat(msgline, "\n"); + sclp_print(msgline); + + print_int(" cpa=", irb->scsw.cpa); + print_int(" prev_ccw=", prev_ccw); + print_int(" this_ccw=", this_ccw); +} + /* * Handles executing ssch, tsch and returns the irb obtained from tsch. * Returns 0 on success, -1 if unexpected status pending and we need to retry, @@ -180,6 +402,19 @@ int do_cio(SubChannelId schid, uint16_t cutype, uint32_t ccw_addr, int fmt) continue; } + sclp_print("cio device error\n"); + print_int(" ssid ", schid.ssid); + print_int(" cssid ", schid.cssid); + print_int(" sch_no", schid.sch_no); + print_int(" ctrl-unit type", cutype); + sclp_print("\n"); + print_irb_err(&irb); + if (cutype == CU_TYPE_DASD_3990 || cutype == CU_TYPE_DASD_2107 || + cutype == CU_TYPE_UNKNOWN) { + if (!basic_sense(schid, cutype, &sd, sizeof(sd))) { + print_eckd_dasd_sense_data(&sd); + } + } rc = -1; break; } diff --git a/pc-bios/s390-ccw/libc.h b/pc-bios/s390-ccw/libc.h index 818517ff5d..bcdc45732d 100644 --- a/pc-bios/s390-ccw/libc.h +++ b/pc-bios/s390-ccw/libc.h @@ -67,6 +67,17 @@ static inline size_t strlen(const char *str) return i; } +static inline char *strcat(char *dest, const char *src) +{ + int i; + char *dest_end = dest + strlen(dest); + + for (i = 0; i <= strlen(src); i++) { + dest_end[i] = src[i]; + } + return dest; +} + static inline int isdigit(int c) { return (c >= '0') && (c <= '9'); From 930072d2bf30986e57dac5c5945a32492f288944 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:29 -0400 Subject: [PATCH 10/19] s390-bios: Extend find_dev() for non-virtio devices We need a method for finding the subchannel of a dasd device. Let's modify find_dev to handle this since it mostly does what we need. Up to this point find_dev has been specific to only virtio devices. Signed-off-by: Jason J. Herne Acked-by: Halil Pasic Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-11-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index e403b5f733..d04ea8972a 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -52,6 +52,12 @@ unsigned int get_loadparm_index(void) return atoui(loadparm_str); } +/* + * Find the subchannel connected to the given device (dev_no) and fill in the + * subchannel information block (schib) with the connected subchannel's info. + * NOTE: The global variable blk_schid is updated to contain the subchannel + * information. + */ static bool find_dev(Schib *schib, int dev_no) { int i, r; @@ -65,15 +71,15 @@ static bool find_dev(Schib *schib, int dev_no) if (!schib->pmcw.dnv) { continue; } - if (!virtio_is_supported(blk_schid)) { - continue; - } + /* Skip net devices since no IPLB is created and therefore no - * no network bootloader has been loaded + * network bootloader has been loaded */ - if (virtio_get_device_type() == VIRTIO_ID_NET && dev_no < 0) { + if (virtio_is_supported(blk_schid) && + virtio_get_device_type() == VIRTIO_ID_NET && dev_no < 0) { continue; } + if ((dev_no < 0) || (schib->pmcw.dev == dev_no)) { return true; } From 7b361db37b18a75860decc0a85e0194936401d66 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:30 -0400 Subject: [PATCH 11/19] s390-bios: Factor finding boot device out of virtio code path Make a new routine find_boot_device to locate the boot device for all cases, not just virtio. The error message for the case where no boot device has been specified and a suitable boot device cannot be auto detected was specific to virtio devices. We update this message to remove virtio specific wording. Signed-off-by: Jason J. Herne Reviewed-by: Farhan Ali Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-12-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 85 ++++++++++++++++++++++------------------ tests/boot-serial-test.c | 2 +- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index d04ea8972a..d3a161c685 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -58,17 +58,18 @@ unsigned int get_loadparm_index(void) * NOTE: The global variable blk_schid is updated to contain the subchannel * information. */ -static bool find_dev(Schib *schib, int dev_no) +static bool find_subch(int dev_no) { + Schib schib; int i, r; for (i = 0; i < 0x10000; i++) { blk_schid.sch_no = i; - r = stsch_err(blk_schid, schib); + r = stsch_err(blk_schid, &schib); if ((r == 3) || (r == -EIO)) { break; } - if (!schib->pmcw.dnv) { + if (!schib.pmcw.dnv) { continue; } @@ -80,7 +81,7 @@ static bool find_dev(Schib *schib, int dev_no) continue; } - if ((dev_no < 0) || (schib->pmcw.dev == dev_no)) { + if ((dev_no < 0) || (schib.pmcw.dev == dev_no)) { return true; } } @@ -136,56 +137,61 @@ static void boot_setup(void) have_iplb = store_iplb(&iplb); } +static void find_boot_device(void) +{ + VDev *vdev = virtio_get_device(); + int ssid; + bool found; + + if (!have_iplb) { + for (ssid = 0; ssid < 0x3; ssid++) { + blk_schid.ssid = ssid; + found = find_subch(-1); + if (found) { + return; + } + } + panic("Could not find a suitable boot device (none specified)\n"); + } + + switch (iplb.pbt) { + case S390_IPL_TYPE_CCW: + debug_print_int("device no. ", iplb.ccw.devno); + blk_schid.ssid = iplb.ccw.ssid & 0x3; + debug_print_int("ssid ", blk_schid.ssid); + found = find_subch(iplb.ccw.devno); + break; + case S390_IPL_TYPE_QEMU_SCSI: + vdev->scsi_device_selected = true; + vdev->selected_scsi_device.channel = iplb.scsi.channel; + vdev->selected_scsi_device.target = iplb.scsi.target; + vdev->selected_scsi_device.lun = iplb.scsi.lun; + blk_schid.ssid = iplb.scsi.ssid & 0x3; + found = find_subch(iplb.scsi.devno); + break; + default: + panic("List-directed IPL not supported yet!\n"); + } + + IPL_assert(found, "Boot device not found\n"); +} + static void virtio_setup(void) { - Schib schib; - int ssid; - bool found = false; - uint16_t dev_no; VDev *vdev = virtio_get_device(); QemuIplParameters *early_qipl = (QemuIplParameters *)QIPL_ADDRESS; memcpy(&qipl, early_qipl, sizeof(QemuIplParameters)); if (have_iplb) { - switch (iplb.pbt) { - case S390_IPL_TYPE_CCW: - dev_no = iplb.ccw.devno; - debug_print_int("device no. ", dev_no); - blk_schid.ssid = iplb.ccw.ssid & 0x3; - debug_print_int("ssid ", blk_schid.ssid); - found = find_dev(&schib, dev_no); - break; - case S390_IPL_TYPE_QEMU_SCSI: - vdev->scsi_device_selected = true; - vdev->selected_scsi_device.channel = iplb.scsi.channel; - vdev->selected_scsi_device.target = iplb.scsi.target; - vdev->selected_scsi_device.lun = iplb.scsi.lun; - blk_schid.ssid = iplb.scsi.ssid & 0x3; - found = find_dev(&schib, iplb.scsi.devno); - break; - default: - panic("List-directed IPL not supported yet!\n"); - } menu_setup(); - } else { - for (ssid = 0; ssid < 0x3; ssid++) { - blk_schid.ssid = ssid; - found = find_dev(&schib, -1); - if (found) { - break; - } - } } - IPL_assert(found, "No virtio device found"); - if (virtio_get_device_type() == VIRTIO_ID_NET) { sclp_print("Network boot device detected\n"); vdev->netboot_start_addr = qipl.netboot_start_addr; } else { virtio_blk_setup_device(blk_schid); - IPL_assert(virtio_ipl_disk_is_valid(), "No valid IPL device detected"); } } @@ -195,8 +201,9 @@ int main(void) sclp_setup(); css_setup(); boot_setup(); - virtio_setup(); + find_boot_device(); + virtio_setup(); zipl_load(); /* no return */ panic("Failed to load OS from hard disk\n"); diff --git a/tests/boot-serial-test.c b/tests/boot-serial-test.c index c591748aaf..24852d4c7d 100644 --- a/tests/boot-serial-test.c +++ b/tests/boot-serial-test.c @@ -114,7 +114,7 @@ static testdef_t tests[] = { { "sparc", "SS-4", "", "MB86904" }, { "sparc", "SS-600MP", "", "TMS390Z55" }, { "sparc64", "sun4u", "", "UltraSPARC" }, - { "s390x", "s390-ccw-virtio", "", "virtio device" }, + { "s390x", "s390-ccw-virtio", "", "device" }, { "m68k", "mcf5208evb", "", "TT", sizeof(kernel_mcf5208), kernel_mcf5208 }, { "microblaze", "petalogix-s3adsp1800", "", "TT", sizeof(kernel_pls3adsp1800), kernel_pls3adsp1800 }, From 9de6cbb152bee3917e58ad00633eddafb40d6678 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:31 -0400 Subject: [PATCH 12/19] s390-bios: Refactor virtio to run channel programs via cio Now that we have a Channel I/O library let's modify virtio boot code to make use of it for running channel programs. Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Acked-by: Thomas Huth Message-Id: <1554388475-18329-13-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/virtio.c | 57 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c index aa9da7253f..35278eaee1 100644 --- a/pc-bios/s390-ccw/virtio.c +++ b/pc-bios/s390-ccw/virtio.c @@ -14,6 +14,7 @@ #include "virtio.h" #include "virtio-scsi.h" #include "bswap.h" +#include "helper.h" #define VRING_WAIT_REPLY_TIMEOUT 30 @@ -89,33 +90,20 @@ int drain_irqs(SubChannelId schid) } } -static int run_ccw(VDev *vdev, int cmd, void *ptr, int len) +static int run_ccw(VDev *vdev, int cmd, void *ptr, int len, bool sli) { Ccw1 ccw = {}; - CmdOrb orb = {}; - int r; - - enable_subchannel(vdev->schid); - - /* start subchannel command */ - orb.fmt = 1; - orb.cpa = (u32)(long)&ccw; - orb.lpm = 0x80; ccw.cmd_code = cmd; ccw.cda = (long)ptr; ccw.count = len; - r = ssch(vdev->schid, &orb); - /* - * XXX Wait until device is done processing the CCW. For now we can - * assume that a simple tsch will have finished the CCW processing, - * but the architecture allows for asynchronous operation - */ - if (!r) { - r = drain_irqs(vdev->schid); + if (sli) { + ccw.flags |= CCW_FLAG_SLI; } - return r; + + enable_subchannel(vdev->schid); + return do_cio(vdev->schid, vdev->senseid.cu_type, ptr2u32(&ccw), CCW_FMT1); } static void vring_init(VRing *vr, VqInfo *info) @@ -257,7 +245,7 @@ void virtio_setup_ccw(VDev *vdev) vdev->config.blk.blk_size = 0; /* mark "illegal" - setup started... */ vdev->guessed_disk_nature = VIRTIO_GDN_NONE; - run_ccw(vdev, CCW_CMD_VDEV_RESET, NULL, 0); + run_ccw(vdev, CCW_CMD_VDEV_RESET, NULL, 0, false); switch (vdev->senseid.cu_model) { case VIRTIO_ID_NET: @@ -278,18 +266,19 @@ void virtio_setup_ccw(VDev *vdev) default: panic("Unsupported virtio device\n"); } - IPL_assert(run_ccw(vdev, CCW_CMD_READ_CONF, &vdev->config, cfg_size) == 0, - "Could not get block device configuration"); + IPL_assert( + run_ccw(vdev, CCW_CMD_READ_CONF, &vdev->config, cfg_size, false) == 0, + "Could not get block device configuration"); /* Feature negotiation */ for (i = 0; i < ARRAY_SIZE(vdev->guest_features); i++) { feats.features = 0; feats.index = i; - rc = run_ccw(vdev, CCW_CMD_READ_FEAT, &feats, sizeof(feats)); + rc = run_ccw(vdev, CCW_CMD_READ_FEAT, &feats, sizeof(feats), false); IPL_assert(rc == 0, "Could not get features bits"); vdev->guest_features[i] &= bswap32(feats.features); feats.features = bswap32(vdev->guest_features[i]); - rc = run_ccw(vdev, CCW_CMD_WRITE_FEAT, &feats, sizeof(feats)); + rc = run_ccw(vdev, CCW_CMD_WRITE_FEAT, &feats, sizeof(feats), false); IPL_assert(rc == 0, "Could not set features bits"); } @@ -306,16 +295,17 @@ void virtio_setup_ccw(VDev *vdev) }; IPL_assert( - run_ccw(vdev, CCW_CMD_READ_VQ_CONF, &config, sizeof(config)) == 0, + run_ccw(vdev, CCW_CMD_READ_VQ_CONF, &config, sizeof(config), false) == 0, "Could not get block device VQ configuration"); info.num = config.num; vring_init(&vdev->vrings[i], &info); vdev->vrings[i].schid = vdev->schid; - IPL_assert(run_ccw(vdev, CCW_CMD_SET_VQ, &info, sizeof(info)) == 0, - "Cannot set VQ info"); + IPL_assert( + run_ccw(vdev, CCW_CMD_SET_VQ, &info, sizeof(info), false) == 0, + "Cannot set VQ info"); } IPL_assert( - run_ccw(vdev, CCW_CMD_WRITE_STATUS, &status, sizeof(status)) == 0, + run_ccw(vdev, CCW_CMD_WRITE_STATUS, &status, sizeof(status), false) == 0, "Could not write status to host"); } @@ -323,8 +313,15 @@ bool virtio_is_supported(SubChannelId schid) { vdev.schid = schid; memset(&vdev.senseid, 0, sizeof(vdev.senseid)); - /* run sense id command */ - if (run_ccw(&vdev, CCW_CMD_SENSE_ID, &vdev.senseid, sizeof(vdev.senseid))) { + + /* + * Run sense id command. + * The size of the senseid data differs between devices (notably, + * between virtio devices and dasds), so specify the largest possible + * size and suppress the incorrect length indication for smaller sizes. + */ + if (run_ccw(&vdev, CCW_CMD_SENSE_ID, &vdev.senseid, sizeof(vdev.senseid), + true)) { return false; } if (vdev.senseid.cu_type == 0x3832) { From 3668cb7ce864ee9351d5d20a1ec6b427cd0b3be4 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:32 -0400 Subject: [PATCH 13/19] s390-bios: Use control unit type to determine boot method The boot method is different depending on which device type we are booting from. Let's examine the control unit type to determine if we're a virtio device. We'll eventually add a case to check for a real dasd device here as well. Since we have to call enable_subchannel() in main now, might as well remove that call from virtio.c : run_ccw(). This requires adding some additional enable_subchannel calls to not break calls to virtio_is_supported(). Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Reviewed-by: Thomas Huth Message-Id: <1554388475-18329-14-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 16 ++++++++++++++-- pc-bios/s390-ccw/netmain.c | 1 + pc-bios/s390-ccw/virtio.c | 1 - 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index d3a161c685..57a10138c6 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -76,6 +76,7 @@ static bool find_subch(int dev_no) /* Skip net devices since no IPLB is created and therefore no * network bootloader has been loaded */ + enable_subchannel(blk_schid); if (virtio_is_supported(blk_schid) && virtio_get_device_type() == VIRTIO_ID_NET && dev_no < 0) { continue; @@ -198,13 +199,24 @@ static void virtio_setup(void) int main(void) { + uint16_t cutype; + sclp_setup(); css_setup(); boot_setup(); find_boot_device(); + enable_subchannel(blk_schid); - virtio_setup(); - zipl_load(); /* no return */ + cutype = cu_type(blk_schid); + switch (cutype) { + case CU_TYPE_VIRTIO: + virtio_setup(); + zipl_load(); /* no return */ + break; + default: + print_int("Attempting to boot from unexpected device type", cutype); + panic(""); + } panic("Failed to load OS from hard disk\n"); return 0; /* make compiler happy */ diff --git a/pc-bios/s390-ccw/netmain.c b/pc-bios/s390-ccw/netmain.c index 5189c0fc39..f3542cb2cf 100644 --- a/pc-bios/s390-ccw/netmain.c +++ b/pc-bios/s390-ccw/netmain.c @@ -476,6 +476,7 @@ static bool find_net_dev(Schib *schib, int dev_no) if (!schib->pmcw.dnv) { continue; } + enable_subchannel(net_schid); if (!virtio_is_supported(net_schid)) { continue; } diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c index 35278eaee1..fb40ca9828 100644 --- a/pc-bios/s390-ccw/virtio.c +++ b/pc-bios/s390-ccw/virtio.c @@ -102,7 +102,6 @@ static int run_ccw(VDev *vdev, int cmd, void *ptr, int len, bool sli) ccw.flags |= CCW_FLAG_SLI; } - enable_subchannel(vdev->schid); return do_cio(vdev->schid, vdev->senseid.cu_type, ptr2u32(&ccw), CCW_FMT1); } From 69333c36dc85b84b021766747cffc2b53df93ae8 Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:33 -0400 Subject: [PATCH 14/19] s390-bios: Add channel command codes/structs needed for dasd-ipl The dasd IPL procedure needs to execute a few previously unused channel commands. Let's define them and their associated data structures. Signed-off-by: Jason J. Herne Acked-by: Cornelia Huck Acked-by: Thomas Huth Message-Id: <1554388475-18329-15-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/cio.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pc-bios/s390-ccw/cio.h b/pc-bios/s390-ccw/cio.h index 1637e32070..aaa432dedd 100644 --- a/pc-bios/s390-ccw/cio.h +++ b/pc-bios/s390-ccw/cio.h @@ -200,11 +200,14 @@ typedef struct ccw1 { #define CCW_FLAG_IDA 0x04 #define CCW_FLAG_SUSPEND 0x02 +/* Common CCW commands */ +#define CCW_CMD_READ_IPL 0x02 #define CCW_CMD_NOOP 0x03 #define CCW_CMD_BASIC_SENSE 0x04 #define CCW_CMD_TIC 0x08 #define CCW_CMD_SENSE_ID 0xe4 +/* Virtio CCW commands */ #define CCW_CMD_SET_VQ 0x13 #define CCW_CMD_VDEV_RESET 0x33 #define CCW_CMD_READ_FEAT 0x12 @@ -216,6 +219,12 @@ typedef struct ccw1 { #define CCW_CMD_SET_CONF_IND 0x53 #define CCW_CMD_READ_VQ_CONF 0x32 +/* DASD CCW commands */ +#define CCW_CMD_DASD_READ 0x06 +#define CCW_CMD_DASD_SEEK 0x07 +#define CCW_CMD_DASD_SEARCH_ID_EQ 0x31 +#define CCW_CMD_DASD_READ_MT 0x86 + /* * Command-mode operation request block */ @@ -333,6 +342,20 @@ typedef struct irb { __u32 emw[8]; } __attribute__ ((packed, aligned(4))) Irb; +/* Used for SEEK ccw commands */ +typedef struct CcwSeekData { + uint16_t reserved; + uint16_t cyl; + uint16_t head; +} __attribute__((packed)) CcwSeekData; + +/* Used for SEARCH ID ccw commands */ +typedef struct CcwSearchIdData { + uint16_t cyl; + uint16_t head; + uint8_t record; +} __attribute__((packed)) CcwSearchIdData; + int enable_mss_facility(void); void enable_subchannel(SubChannelId schid); uint16_t cu_type(SubChannelId schid); From efa47d36da89f4b23c315a7cc085fab0d15eb47c Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:34 -0400 Subject: [PATCH 15/19] s390-bios: Support booting from real dasd device Allows guest to boot from a vfio configured real dasd device. Signed-off-by: Jason J. Herne Reviewed-by: Cornelia Huck Message-Id: <1554388475-18329-16-git-send-email-jjherne@linux.ibm.com> Signed-off-by: Thomas Huth --- MAINTAINERS | 1 + docs/devel/s390-dasd-ipl.txt | 133 ++++++++++++++++++++ pc-bios/s390-ccw/Makefile | 2 +- pc-bios/s390-ccw/dasd-ipl.c | 235 +++++++++++++++++++++++++++++++++++ pc-bios/s390-ccw/dasd-ipl.h | 16 +++ pc-bios/s390-ccw/main.c | 5 + pc-bios/s390-ccw/s390-arch.h | 13 ++ 7 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 docs/devel/s390-dasd-ipl.txt create mode 100644 pc-bios/s390-ccw/dasd-ipl.c create mode 100644 pc-bios/s390-ccw/dasd-ipl.h diff --git a/MAINTAINERS b/MAINTAINERS index f7976aa43d..be53b7d3b7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1181,6 +1181,7 @@ S: Supported F: hw/s390x/ipl.* F: pc-bios/s390-ccw/ F: pc-bios/s390-ccw.img +F: docs/devel/s390-dasd-ipl.txt T: git https://github.com/borntraeger/qemu.git s390-next L: qemu-s390x@nongnu.org diff --git a/docs/devel/s390-dasd-ipl.txt b/docs/devel/s390-dasd-ipl.txt new file mode 100644 index 0000000000..9107e048e4 --- /dev/null +++ b/docs/devel/s390-dasd-ipl.txt @@ -0,0 +1,133 @@ +***************************** +***** s390 hardware IPL ***** +***************************** + +The s390 hardware IPL process consists of the following steps. + +1. A READ IPL ccw is constructed in memory location 0x0. + This ccw, by definition, reads the IPL1 record which is located on the disk + at cylinder 0 track 0 record 1. Note that the chain flag is on in this ccw + so when it is complete another ccw will be fetched and executed from memory + location 0x08. + +2. Execute the Read IPL ccw at 0x00, thereby reading IPL1 data into 0x00. + IPL1 data is 24 bytes in length and consists of the following pieces of + information: [psw][read ccw][tic ccw]. When the machine executes the Read + IPL ccw it read the 24-bytes of IPL1 to be read into memory starting at + location 0x0. Then the ccw program at 0x08 which consists of a read + ccw and a tic ccw is automatically executed because of the chain flag from + the original READ IPL ccw. The read ccw will read the IPL2 data into memory + and the TIC (Transfer In Channel) will transfer control to the channel + program contained in the IPL2 data. The TIC channel command is the + equivalent of a branch/jump/goto instruction for channel programs. + NOTE: The ccws in IPL1 are defined by the architecture to be format 0. + +3. Execute IPL2. + The TIC ccw instruction at the end of the IPL1 channel program will begin + the execution of the IPL2 channel program. IPL2 is stage-2 of the boot + process and will contain a larger channel program than IPL1. The point of + IPL2 is to find and load either the operating system or a small program that + loads the operating system from disk. At the end of this step all or some of + the real operating system is loaded into memory and we are ready to hand + control over to the guest operating system. At this point the guest + operating system is entirely responsible for loading any more data it might + need to function. NOTE: The IPL2 channel program might read data into memory + location 0 thereby overwriting the IPL1 psw and channel program. This is ok + as long as the data placed in location 0 contains a psw whose instruction + address points to the guest operating system code to execute at the end of + the IPL/boot process. + NOTE: The ccws in IPL2 are defined by the architecture to be format 0. + +4. Start executing the guest operating system. + The psw that was loaded into memory location 0 as part of the ipl process + should contain the needed flags for the operating system we have loaded. The + psw's instruction address will point to the location in memory where we want + to start executing the operating system. This psw is loaded (via LPSW + instruction) causing control to be passed to the operating system code. + +In a non-virtualized environment this process, handled entirely by the hardware, +is kicked off by the user initiating a "Load" procedure from the hardware +management console. This "Load" procedure crafts a special "Read IPL" ccw in +memory location 0x0 that reads IPL1. It then executes this ccw thereby kicking +off the reading of IPL1 data. Since the channel program from IPL1 will be +written immediately after the special "Read IPL" ccw, the IPL1 channel program +will be executed immediately (the special read ccw has the chaining bit turned +on). The TIC at the end of the IPL1 channel program will cause the IPL2 channel +program to be executed automatically. After this sequence completes the "Load" +procedure then loads the psw from 0x0. + +********************************************************** +***** How this all pertains to QEMU (and the kernel) ***** +********************************************************** + +In theory we should merely have to do the following to IPL/boot a guest +operating system from a DASD device: + +1. Place a "Read IPL" ccw into memory location 0x0 with chaining bit on. +2. Execute channel program at 0x0. +3. LPSW 0x0. + +However, our emulation of the machine's channel program logic within the kernel +is missing one key feature that is required for this process to work: +non-prefetch of ccw data. + +When we start a channel program we pass the channel subsystem parameters via an +ORB (Operation Request Block). One of those parameters is a prefetch bit. If the +bit is on then the vfio-ccw kernel driver is allowed to read the entire channel +program from guest memory before it starts executing it. This means that any +channel commands that read additional channel commands will not work as expected +because the newly read commands will only exist in guest memory and NOT within +the kernel's channel subsystem memory. The kernel vfio-ccw driver currently +requires this bit to be on for all channel programs. This is a problem because +the IPL process consists of transferring control from the "Read IPL" ccw +immediately to the IPL1 channel program that was read by "Read IPL". + +Not being able to turn off prefetch will also prevent the TIC at the end of the +IPL1 channel program from transferring control to the IPL2 channel program. + +Lastly, in some cases (the zipl bootloader for example) the IPL2 program also +transfers control to another channel program segment immediately after reading +it from the disk. So we need to be able to handle this case. + +************************** +***** What QEMU does ***** +************************** + +Since we are forced to live with prefetch we cannot use the very simple IPL +procedure we defined in the preceding section. So we compensate by doing the +following. + +1. Place "Read IPL" ccw into memory location 0x0, but turn off chaining bit. +2. Execute "Read IPL" at 0x0. + + So now IPL1's psw is at 0x0 and IPL1's channel program is at 0x08. + +4. Write a custom channel program that will seek to the IPL2 record and then + execute the READ and TIC ccws from IPL1. Normally the seek is not required + because after reading the IPL1 record the disk is automatically positioned + to read the very next record which will be IPL2. But since we are not reading + both IPL1 and IPL2 as part of the same channel program we must manually set + the position. + +5. Grab the target address of the TIC instruction from the IPL1 channel program. + This address is where the IPL2 channel program starts. + + Now IPL2 is loaded into memory somewhere, and we know the address. + +6. Execute the IPL2 channel program at the address obtained in step #5. + + Because this channel program can be dynamic, we must use a special algorithm + that detects a READ immediately followed by a TIC and breaks the ccw chain + by turning off the chain bit in the READ ccw. When control is returned from + the kernel/hardware to the QEMU bios code we immediately issue another start + subchannel to execute the remaining TIC instruction. This causes the entire + channel program (starting from the TIC) and all needed data to be refetched + thereby stepping around the limitation that would otherwise prevent this + channel program from executing properly. + + Now the operating system code is loaded somewhere in guest memory and the psw + in memory location 0x0 will point to entry code for the guest operating + system. + +7. LPSW 0x0. + LPSW transfers control to the guest operating system and we're done. diff --git a/pc-bios/s390-ccw/Makefile b/pc-bios/s390-ccw/Makefile index 12ad9c1d58..a048b6b077 100644 --- a/pc-bios/s390-ccw/Makefile +++ b/pc-bios/s390-ccw/Makefile @@ -10,7 +10,7 @@ $(call set-vpath, $(SRC_PATH)/pc-bios/s390-ccw) .PHONY : all clean build-all OBJECTS = start.o main.o bootmap.o jump2ipl.o sclp.o menu.o \ - virtio.o virtio-scsi.o virtio-blkdev.o libc.o cio.o + virtio.o virtio-scsi.o virtio-blkdev.o libc.o cio.o dasd-ipl.o QEMU_CFLAGS := $(filter -W%, $(QEMU_CFLAGS)) QEMU_CFLAGS += -ffreestanding -fno-delete-null-pointer-checks -msoft-float diff --git a/pc-bios/s390-ccw/dasd-ipl.c b/pc-bios/s390-ccw/dasd-ipl.c new file mode 100644 index 0000000000..0fc879bb8e --- /dev/null +++ b/pc-bios/s390-ccw/dasd-ipl.c @@ -0,0 +1,235 @@ +/* + * S390 IPL (boot) from a real DASD device via vfio framework. + * + * Copyright (c) 2019 Jason J. Herne + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#include "libc.h" +#include "s390-ccw.h" +#include "s390-arch.h" +#include "dasd-ipl.h" +#include "helper.h" + +static char prefix_page[PAGE_SIZE * 2] + __attribute__((__aligned__(PAGE_SIZE * 2))); + +static void enable_prefixing(void) +{ + memcpy(&prefix_page, lowcore, 4096); + set_prefix(ptr2u32(&prefix_page)); +} + +static void disable_prefixing(void) +{ + set_prefix(0); + /* Copy io interrupt info back to low core */ + memcpy((void *)&lowcore->subchannel_id, prefix_page + 0xB8, 12); +} + +static bool is_read_tic_ccw_chain(Ccw0 *ccw) +{ + Ccw0 *next_ccw = ccw + 1; + + return ((ccw->cmd_code == CCW_CMD_DASD_READ || + ccw->cmd_code == CCW_CMD_DASD_READ_MT) && + ccw->chain && next_ccw->cmd_code == CCW_CMD_TIC); +} + +static bool dynamic_cp_fixup(uint32_t ccw_addr, uint32_t *next_cpa) +{ + Ccw0 *cur_ccw = (Ccw0 *)(uint64_t)ccw_addr; + Ccw0 *tic_ccw; + + while (true) { + /* Skip over inline TIC (it might not have the chain bit on) */ + if (cur_ccw->cmd_code == CCW_CMD_TIC && + cur_ccw->cda == ptr2u32(cur_ccw) - 8) { + cur_ccw += 1; + continue; + } + + if (!cur_ccw->chain) { + break; + } + if (is_read_tic_ccw_chain(cur_ccw)) { + /* + * Breaking a chain of CCWs may alter the semantics or even the + * validity of a channel program. The heuristic implemented below + * seems to work well in practice for the channel programs + * generated by zipl. + */ + tic_ccw = cur_ccw + 1; + *next_cpa = tic_ccw->cda; + cur_ccw->chain = 0; + return true; + } + cur_ccw += 1; + } + return false; +} + +static int run_dynamic_ccw_program(SubChannelId schid, uint16_t cutype, + uint32_t cpa) +{ + bool has_next; + uint32_t next_cpa = 0; + int rc; + + do { + has_next = dynamic_cp_fixup(cpa, &next_cpa); + + print_int("executing ccw chain at ", cpa); + enable_prefixing(); + rc = do_cio(schid, cutype, cpa, CCW_FMT0); + disable_prefixing(); + + if (rc) { + break; + } + cpa = next_cpa; + } while (has_next); + + return rc; +} + +static void make_readipl(void) +{ + Ccw0 *ccwIplRead = (Ccw0 *)0x00; + + /* Create Read IPL ccw at address 0 */ + ccwIplRead->cmd_code = CCW_CMD_READ_IPL; + ccwIplRead->cda = 0x00; /* Read into address 0x00 in main memory */ + ccwIplRead->chain = 0; /* Chain flag */ + ccwIplRead->count = 0x18; /* Read 0x18 bytes of data */ +} + +static void run_readipl(SubChannelId schid, uint16_t cutype) +{ + if (do_cio(schid, cutype, 0x00, CCW_FMT0)) { + panic("dasd-ipl: Failed to run Read IPL channel program\n"); + } +} + +/* + * The architecture states that IPL1 data should consist of a psw followed by + * format-0 READ and TIC CCWs. Let's sanity check. + */ +static void check_ipl1(void) +{ + Ccw0 *ccwread = (Ccw0 *)0x08; + Ccw0 *ccwtic = (Ccw0 *)0x10; + + if (ccwread->cmd_code != CCW_CMD_DASD_READ || + ccwtic->cmd_code != CCW_CMD_TIC) { + panic("dasd-ipl: IPL1 data invalid. Is this disk really bootable?\n"); + } +} + +static void check_ipl2(uint32_t ipl2_addr) +{ + Ccw0 *ccw = u32toptr(ipl2_addr); + + if (ipl2_addr == 0x00) { + panic("IPL2 address invalid. Is this disk really bootable?\n"); + } + if (ccw->cmd_code == 0x00) { + panic("IPL2 ccw data invalid. Is this disk really bootable?\n"); + } +} + +static uint32_t read_ipl2_addr(void) +{ + Ccw0 *ccwtic = (Ccw0 *)0x10; + + return ccwtic->cda; +} + +static void ipl1_fixup(void) +{ + Ccw0 *ccwSeek = (Ccw0 *) 0x08; + Ccw0 *ccwSearchID = (Ccw0 *) 0x10; + Ccw0 *ccwSearchTic = (Ccw0 *) 0x18; + Ccw0 *ccwRead = (Ccw0 *) 0x20; + CcwSeekData *seekData = (CcwSeekData *) 0x30; + CcwSearchIdData *searchData = (CcwSearchIdData *) 0x38; + + /* move IPL1 CCWs to make room for CCWs needed to locate record 2 */ + memcpy(ccwRead, (void *)0x08, 16); + + /* Disable chaining so we don't TIC to IPL2 channel program */ + ccwRead->chain = 0x00; + + ccwSeek->cmd_code = CCW_CMD_DASD_SEEK; + ccwSeek->cda = ptr2u32(seekData); + ccwSeek->chain = 1; + ccwSeek->count = sizeof(*seekData); + seekData->reserved = 0x00; + seekData->cyl = 0x00; + seekData->head = 0x00; + + ccwSearchID->cmd_code = CCW_CMD_DASD_SEARCH_ID_EQ; + ccwSearchID->cda = ptr2u32(searchData); + ccwSearchID->chain = 1; + ccwSearchID->count = sizeof(*searchData); + searchData->cyl = 0; + searchData->head = 0; + searchData->record = 2; + + /* Go back to Search CCW if correct record not yet found */ + ccwSearchTic->cmd_code = CCW_CMD_TIC; + ccwSearchTic->cda = ptr2u32(ccwSearchID); +} + +static void run_ipl1(SubChannelId schid, uint16_t cutype) + { + uint32_t startAddr = 0x08; + + if (do_cio(schid, cutype, startAddr, CCW_FMT0)) { + panic("dasd-ipl: Failed to run IPL1 channel program\n"); + } +} + +static void run_ipl2(SubChannelId schid, uint16_t cutype, uint32_t addr) +{ + if (run_dynamic_ccw_program(schid, cutype, addr)) { + panic("dasd-ipl: Failed to run IPL2 channel program\n"); + } +} + +/* + * Limitations in vfio-ccw support complicate the IPL process. Details can + * be found in docs/devel/s390-dasd-ipl.txt + */ +void dasd_ipl(SubChannelId schid, uint16_t cutype) +{ + PSWLegacy *pswl = (PSWLegacy *) 0x00; + uint32_t ipl2_addr; + + /* Construct Read IPL CCW and run it to read IPL1 from boot disk */ + make_readipl(); + run_readipl(schid, cutype); + ipl2_addr = read_ipl2_addr(); + check_ipl1(); + + /* + * Fixup IPL1 channel program to account for vfio-ccw limitations, then run + * it to read IPL2 channel program from boot disk. + */ + ipl1_fixup(); + run_ipl1(schid, cutype); + check_ipl2(ipl2_addr); + + /* + * Run IPL2 channel program to read operating system code from boot disk + */ + run_ipl2(schid, cutype, ipl2_addr); + + /* Transfer control to the guest operating system */ + pswl->mask |= PSW_MASK_EAMODE; /* Force z-mode */ + pswl->addr |= PSW_MASK_BAMODE; /* ... */ + jump_to_low_kernel(); +} diff --git a/pc-bios/s390-ccw/dasd-ipl.h b/pc-bios/s390-ccw/dasd-ipl.h new file mode 100644 index 0000000000..c394828906 --- /dev/null +++ b/pc-bios/s390-ccw/dasd-ipl.h @@ -0,0 +1,16 @@ +/* + * S390 IPL (boot) from a real DASD device via vfio framework. + * + * Copyright (c) 2019 Jason J. Herne + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#ifndef DASD_IPL_H +#define DASD_IPL_H + +void dasd_ipl(SubChannelId schid, uint16_t cutype); + +#endif /* DASD_IPL_H */ diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index 57a10138c6..3c449ad496 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -13,6 +13,7 @@ #include "s390-ccw.h" #include "cio.h" #include "virtio.h" +#include "dasd-ipl.h" char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE))); static SubChannelId blk_schid = { .one = 1 }; @@ -209,6 +210,10 @@ int main(void) cutype = cu_type(blk_schid); switch (cutype) { + case CU_TYPE_DASD_3990: + case CU_TYPE_DASD_2107: + dasd_ipl(blk_schid, cutype); /* no return */ + break; case CU_TYPE_VIRTIO: virtio_setup(); zipl_load(); /* no return */ diff --git a/pc-bios/s390-ccw/s390-arch.h b/pc-bios/s390-ccw/s390-arch.h index 5e92c7a27d..504fc7c2f0 100644 --- a/pc-bios/s390-ccw/s390-arch.h +++ b/pc-bios/s390-ccw/s390-arch.h @@ -87,4 +87,17 @@ typedef struct LowCore { extern LowCore const *lowcore; +static inline void set_prefix(uint32_t address) +{ + asm volatile("spx %0" : : "m" (address) : "memory"); +} + +static inline uint32_t store_prefix(void) +{ + uint32_t address; + + asm volatile("stpx %0" : "=m" (address)); + return address; +} + #endif From 2880469c95e42f8a5b0acbe8c4808255cc6c9e5b Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 4 Apr 2019 10:34:35 -0400 Subject: [PATCH 16/19] s390-bios: Use control unit type to find bootable devices When the user does not specify which device to boot from then we end up guessing. Instead of simply grabbing the first available device let's be a little bit smarter and only choose devices that might be bootable like disk, and not console devices. Signed-off-by: Jason J. Herne Message-Id: <1554388475-18329-17-git-send-email-jjherne@linux.ibm.com> [thuth: Added fix for virtio_is_supported() not being called anymore] Signed-off-by: Thomas Huth --- pc-bios/s390-ccw/main.c | 45 ++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c index 3c449ad496..a69c73349e 100644 --- a/pc-bios/s390-ccw/main.c +++ b/pc-bios/s390-ccw/main.c @@ -21,6 +21,7 @@ static char loadparm_str[LOADPARM_LEN + 1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; QemuIplParameters qipl; IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE))); static bool have_iplb; +static uint16_t cutype; LowCore const *lowcore; /* Yes, this *is* a pointer to address 0 */ #define LOADPARM_PROMPT "PROMPT " @@ -58,11 +59,15 @@ unsigned int get_loadparm_index(void) * subchannel information block (schib) with the connected subchannel's info. * NOTE: The global variable blk_schid is updated to contain the subchannel * information. + * + * If the caller gives dev_no=-1 then the user did not specify a boot device. + * In this case we'll just use the first potentially bootable device we find. */ static bool find_subch(int dev_no) { Schib schib; int i, r; + bool is_virtio; for (i = 0; i < 0x10000; i++) { blk_schid.sch_no = i; @@ -74,16 +79,39 @@ static bool find_subch(int dev_no) continue; } - /* Skip net devices since no IPLB is created and therefore no - * network bootloader has been loaded - */ enable_subchannel(blk_schid); - if (virtio_is_supported(blk_schid) && - virtio_get_device_type() == VIRTIO_ID_NET && dev_no < 0) { - continue; + cutype = cu_type(blk_schid); + + /* + * Note: we always have to run virtio_is_supported() here to make + * sure that the vdev.senseid data gets pre-initialized correctly + */ + is_virtio = virtio_is_supported(blk_schid); + + /* No specific devno given, just return 1st possibly bootable device */ + if (dev_no < 0) { + switch (cutype) { + case CU_TYPE_VIRTIO: + if (is_virtio) { + /* + * Skip net devices since no IPLB is created and therefore + * no network bootloader has been loaded + */ + if (virtio_get_device_type() != VIRTIO_ID_NET) { + return true; + } + } + continue; + case CU_TYPE_DASD_3990: + case CU_TYPE_DASD_2107: + return true; + default: + continue; + } } - if ((dev_no < 0) || (schib.pmcw.dev == dev_no)) { + /* Caller asked for a specific devno */ + if (schib.pmcw.dev == dev_no) { return true; } } @@ -200,15 +228,12 @@ static void virtio_setup(void) int main(void) { - uint16_t cutype; - sclp_setup(); css_setup(); boot_setup(); find_boot_device(); enable_subchannel(blk_schid); - cutype = cu_type(blk_schid); switch (cutype) { case CU_TYPE_DASD_3990: case CU_TYPE_DASD_2107: From 4a33fac54665558b7f7a68135c4a4c602eceb72f Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 12 Apr 2019 09:30:45 +0200 Subject: [PATCH 17/19] pc-bios/s390: Update firmware images s390-ccw.img contains support for booting from vfio-ccw dasd passthrough devices now, and s390-netboot.img is updated since there were changes to the code that is shared between s390-ccw.img and s390-netboot.img. Signed-off-by: Thomas Huth --- pc-bios/s390-ccw.img | Bin 34568 -> 42608 bytes pc-bios/s390-netboot.img | Bin 54944 -> 67232 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/pc-bios/s390-ccw.img b/pc-bios/s390-ccw.img index 450a076dc0ad9f2babc7f2e3005ea05a1082a808..ba054828d35d72fd1ed5521a48f43f593a1c291f 100644 GIT binary patch literal 42608 zcmb<-^>JfjVq|~=W(F|^1_r1A`v$1E1B6}#;X5!`FmNz9Fz_?*F)%Q&F)%Q&LgX2s z>dsw7?c<&GXVaJ;#VLZ~4Wtl6gTe+Ju8a%}jGPP;6hIOn4};Q# zpZ?|>9rp_pYG(BD*37wl8!CsBVut&h`2mt5HbDk(crq|Bbb;IkOP+j!3=BMAbu7@} zg{Tulh{K)J0@4H*XKny%V_;yAVJOTj$YDrLN;XO?PG-o-FHg=dN@d6_$Vq}J0nzJW z?t$0`*T4P+NR|QS4!HZ)e?Sq3yKnsuRB@R5Ho!apH6JbxV?e|i7eKXdgych*z0AhY z^tut6&!FMLY@G5s|APU8%7F_E40dam|Nd{_*zjT3fu{@%f^8sD9FBq?7%mBXSh<1Wv4a5v!-Ur%3@i*hj~y;>Ff=%`epuT|^gj@r$MfE{(zmtWb=&?ZqB>rWb9R$ue zIT|p4Q;YilU;E?1@hk~eFTGrkfpK{YgC59ykci4X5UKnb6pjpaAU5yEP+9)v{{^_o+c8c4zaC~XG#v?m(~%qlC>^mamjk7v|8pVcfBo!FYKKgXD7S|Nmb* zGjK5Q>||na*~!8nw7i9Z^ZI`Q1_1^~0YQf4?*D&-^fIu!7$`6-|MK_Kbq0_eC^a&8 zZrUWk@S5QR1B2=`rVngjmOfZU>#+di4h9u4+wBrN;|2i+FxvvGBMq$M{|A=WAQnW2 z)MEk0A5a~Jm)IFUKy}E2b%5=d@)u?Y`#-Qw)hP@xGuZzzFflN~OKDJLgvJMz=TLdD zI!HOU{P%waP{|ZL-BIAf8c<2L{00NVV*yak+R4IDxxC~5e^7}O-M(}m1EYj2D2GNU zOkj|(SgOvz@YvzQ0tOZ_#Q>0q&?XZHhD}Ng1|0?rGRy54n3uOO@GZCd|9^Q41J^{x z4;z=4FbFQc#lX0H+yA@EH9)#Q1~9Pf6k$+$o&QmQfrb4GDBjfkA`}Y1>CKg=lVGMi7FfW*FcDgg@IlIv|8KzM zNXh?$AU(^!{C@_@KmWlcyKp8XkNw{amAe5-IH33n=7~@cU|24}z!0Foup>Z#VGsMi z*Y7_RFsLm5{XYTZT7K;VPyhe__j~yl1_qEWc6Nw7D9uEDgsD=V@cKQ-hdad>l-U14 z)vf;!0I6rJJ9n}$EP|<3O54f8unAFN$j~zgMdM(6Y!N9XL{{IJ& zfX)C09x2zIG7KWGt-&f7KzZr4F-XN@g#|1O6COK&?c`t(*vZ2nyj=VL|JUn3d|+VN z$-$t={sj{MpnA^QclqykKOQS6Ffgz<8hqpc*{!e}RC6#+0EL2e|5fIX90Cl>TNp$@ z{<2yCQo*+l#I|k$vsZxFc6&f!@xN#)!$+RwC5&;70w2CV7WlA%LB)~b!-2;PA677^ zEZ1bX>8SL9;jzMp9SkbVGZ{4;(>^df7Dy0hn6MmFYLqatgYE2mtdPLK!18!Qf&>H0 zCA|+f1sFi($0X+i1q=*UR~{>TIKZF+D(@y)Ethm)n6%1(0bGKM|NrCY@ZkW+Zct5p zN#MhY0*1!{4xrM1DY*0x0ELtDst+fe76>pnKmBmR>45;_a&KtqZ~p%U*i?RyzZ5J$ zsf0lokO-f#qr!)qV3r<8)XoXy=YKyQD}YN*u&Z`5F(@r>Vdy*XfPrD+=jAgQ z7#}|X<);Gy3=9(=FZX0%SiX|s|Nh4TT+2%sl)yGiGBCV$VPIk4U2e%B;ONr8@R$LV zvm75fFic(b091l=gUea-|3AP!XIcK`e=}H(%43574h9yGsXJL1qCqZ`e+o)L%k{xI zPYaxvS-_<(3s|qpP8Nnhs2gn{vdg6z1i&nn<&q4HJ2@C6moH~v2df6z2S9vFJ+Pb& zh;1bYW`pa0TQ9IWhUH)WO#;bTN`d9T^^4_4Fk1;EZfgco_xH{6FaHvt`5oS_w44cQ zS1#9N5CEmL3AdJeGBAR4%7udR9D@$1yq$0!l+Gr~~ z;nYqRhUJ%7Kb)EXYUzMloUD$H35?Ex8({T4tmXM_Dq{oN@)ib;*ZChA7+4N``Tzg# zCUD(5AynW3$8webUxfshmoNl?>iFfB3<{2c3GB{{3mEn?1~41|Wt#5gB@B+xwDZLnWTV1AK(56(H@n6u~r`Hp8AC|oR0fY}p3 zY?~!uQ=&lmeM0zD#s#1f#uy}Hm4oD7ujLgCj8a8l^IbqCDBO;{;P7IVP+$J#e-%id zMKIX(L=fAu0nCmDv2F6fHmQPDrcPxNU|g>9|LoLI0lDQR|4&X`B_O}N@umx3ZRnfu>h#V0>#P{R!~V~ac%id2K&bbA9gUXOzkiLxm_Hr zhGn@lg94bPvV1m!!t(9^--ArE%tiR=&x5H96Bw44{C~OMc>?3|mj6#5D@Fy*w&lDF3XHquwW4=^~R@w!Y{rM6n^n_ z%Oe;V0v0qt>eqk?3;_ZK49izAFg%_B3hA9J4C#+Id|1K2vQvs-qmUq|Pw-e_0|UbZ zu)TAa%W;6heCFeT4NMFykpUmJEN@{9+$q9f_1FMZ+P-#XU}E5v5(*G0V2Ci8z#wfD zU^0PWBGU%R8qMYd~NFLx5reLx7?I z1K3oS*PB7I5z++=p#m2emLK`Q0pw3}aj>7ksm5X{m@N+yw>Ad(|M$xX=>!H8zm)u+ zvr~k@>$M2TG;k{lQkPu;h3;d;1_lOjdsxN?>{oCuGJ6U#m+LdgT#HsP`z?rV%?P$* z`{MvmjqzF(WMTy41O_2c=*)+OScG5!D7~eFYz2k;YZZ_>h|Lj94Ga-R6Bw3rF)%K- z`oRzoRKOr*7{JuP5MVTcLCAP{$^Yi%tN#B7>4oS(whNSICsm-jTlxc%yF0;doepxV zg(jFi5yZAy3U+t&@^%0JFE9BY6d~~82&i=_Vb~$Sz|m>Iz##^%cP*SGHZI-9z|gUQ zfeEa;C_>=F0d$p*6+UcX5PNONV8Fn;Q-r}A(G!zih2$1zunB=6`z>_9Y;O?TDjw_> zQCJRTVBpk$Yye7Y5sn26j}F|bHEODjJ%fTl;2#|EJOq_nb>GbH5%I2M3Y$?`A% zz_DU}7;L%>Bp*c>ILIvj@;~PB29QZYyO+28?^?c#{XfWV^TQn*7`VZzS(ktLKl!o4 zM+Z>71uC!Ezp%4|YcTM5gjwtI-~TOkvM_80^&del{&hxT5lAO5C{L6yR)ET@-c64M zK;>2Ff(snWk3d>%rpH05IIn{2v)}`>!TmnV$6!-tg2a2LLE2FY49m3{7?+Fx|Fe^Y zK^+vkz2F#>-h>E8Q*iyy39kPw!0km&aQ$xyuK#}(?i6Ft0-5q!3Dl+mm!BM9n>yev zlM<-Qs=%)40J*>%Tz_&lf!LPQ!0LU#_IpBHhG^mKWP#K-y*41-(y2%;3j&F7CV2rBy8# zM9=Hp46F>i&@wY|g9QT%*hG0y+Z|*>&9~)}3JjB0p@g3n$mE)rAa_U$fody8pXECk z7$}g{E|(u_PEInkog?oQ62MgF#7?>MHXMc=B;`x0MhHxAi(sv0o3*dsgV8*=~sYB zo}G#eP0L*v4MDn0HiG0i_Ji2w&0zLU5ZmG`*tBMdX^$NQ*cm2(`pnBq7`{1L76?FV z9}dSr1BR)q0zhrnSJ0a02S|TS8qV+mkCI3=fnxE05lF2`KG@z`5ZfF)4#H6mVq0{B z-Qc+V%l|k~$oANPQU*9zg2K;v(kcfAaC;h(dckdy9xaerQo$h8A!*LU0VKlV2Qtyj z6U_Diu`Oc2rVD}CDg4VN4NzhelsZ89%Pas?ll3+%U&+7_5b$9MI6PFAXD~?YWMR+* zn;^QAiNWZx0jTG*yoJFN!+VG!HQ8_eDd zVw+zCyQh8mmw!ukvM^{lIyf-yWMMD^sgU>sO4AHZ5O+OZ07?^}+I0CYNQ+it&!!*- zt(_7KNz1?dYkzD2N{5dd3|JT@fOJcJTrR=zUqnb$V!6w|Bajq6l_7v-dCR}OAbIhF zAk+Vyg!DteuE^NQ!q5bglNN@g*?+s1fB6UQlNg7B#MzTUZZ-p#aqO`mws|2~C#a3M zF2dl$25^Y7EVp7{3^4exVRVXGNPqE|RzdO6~t|gWO_t5+uTQ9mFu(J}iffxG+LOE(SCT z(G>=^sTpLtDL5|KY9A|H@Mo9+NhQnW7#xq1Oxvh`wvS+gqBJ( z7&w7axF*9AP=C^K!G{%Kv%vB1B?l^*9tVI%H zxp-2M(F2C2+Op!18y%Hi1(n>opMD^cC2q1ydO= zlrAq}Xqw7!p=@~xL)KKr3*~UV8^C(^fz+Ga2eZNPZ+Zr-x9RbP3$>71NbHQ{CAhAs zV3mu(y3T^xb3ts=jbL5rAa>^hFuQdp3&SNazZcA}-O0jWw3CHF7sUJB_1NK~0t3tI z^^g`2Xl&oe?<(_!f(@V+EV%t^0v?-Z%>~UjEdTO9f0EIMCCk73X9SsPV!ZszzndVI zp%2&<;QHUB2+R%viJNAC?D-}K8*4VS0L!_7piM^&BV_7+$=}ydef$%7c3W2H+gb@&TmI!~>*<e_v-#W!R9kyo3QXYn*?fOf=4(y!14EW zHE1-TqXpbH0gtyKYTD(R3`}4jmV!(cn}_7XZjcDeOt1@0!0gFj_HwWtu^=CsSVdeA zV2Hf10n}fC_^|-2vJR{gJPyNB0cQ7rbvZ>|*uVlybp}yjwHaV}a81vW1ZJ0k)#`xQ z9u-bD$zGj{UDn+s63UC?O4j!=- zZ9$|N{X(#=dQeyygL4IQC72B^E50~`!zkz~^8!fQ2R!@JZh^&oaGGq_2AL)rf@HoU zNQBuRY^EWY?FnYbf@1QEFxY&3P_Az0h1&v|eG_0{FjkEO`L=~YaJeP}Gf2muE)eyAz~xgWNG03h<$B<`Kg;EM3>={GZIGz&j^%pb(Hd}%*BIPmWCG7W3)_QS z`mX^T15zON=b*<9A15%dyzc*)0G<<4U|=u;mu{b3!K&E7s_fBJJ!N1JS_x_|GMoU( ze+9Q-g%*H(^&b?PN5P`ta@{Bj#QqE()n&^Dhg-@2w8sh?CxH5N3=9mQ`P4X&T}JUB z*E3mOWxl{O0bb@9#eqe^=KyHGx2XudfP2vRG1uu$p{{4FaoEa zPurKTWLN^WWtZcs4-BB*%1#yrujP^q@>kg}Fo4G=Il%KG3;Kxn}PAQ zHn>Nt)BtXee7dy!%Rg|vC?vc5%YSf-_)Ej`FaN=F=eppTYsMasZw;@4%>>W;8qNay zJOC1E%fI|j1(hBkd6r2Mo0fn1zZIlI7aT8);PHRM>mXT1aQnir8m!xJ`IrB*9}6&W zFo=OvFV(!n&A_0*!2S`Gqh8avfq`Y}e*>Q7B@8^1 zm<&L_8Oi~H|NsA=4!FR;5dezmrwj~? z3ZQwEWSx0pxG6y`9S=7-T?xSu-cX;R1t0fB-{)f&kC*2nLn_0|5bt00FK|3Ji{$ zlo%8`6c}W{I>B=XA9~o?LH$z0oaGi!6RI5y1Q?cEFt9in2yiX8U|@()2w(^>5a3yE z!QcQgZIc3nB5YO|H2?4+3*;JjJL&=hgP|vA?h{nU?POt?5~^^4XSoVP7IuatQ`L zDe2|A7}y*H6c{EkC@?u}P~ey(rNA|bL4i9$-~z`c1qLCw%bx!K|3Ma1r$c5K2ggCI1gE_h8_4a#Y~k$-*Gzyw!kV`IrAY zLALK?VVDYW!SXNv_b=D{{~yil9Sm$Bbs+z&zP|j+|Fxiy0nOx1Wl#X+d#>f)42;eq z3Jl8y!RshM>xhiN?a43y|Nl<{sb*(~^tV9qYOr>>B!dFDg$7Df3JeUZK{NVBlR&xO z8ZxH0T!KLX6dn-*7g(VF-_WsvL1y``|KKwA{p`>UpmA*mad1cnFfbT^^XTfypxIR9 z_7bAk1oahoZkhcHw0#U}&t!t)6;xC0WMNqR*x&*KgW~cd3@po6^8Z*aA;17CCs#*1 z2uxs@#4v#g8Y7ahR0m4Q??XT>ERajUedQJgE@#CF{7#A+U~RQuj}<0xFieEFV7b=+ zpOAP1tqA~)oq|FLF@9>u37U5UwM;fEI0}3?3ZA7@xypV)U;)GOoeb-i&ty2}xcdWx z*foG+wyr6G-(x zZ>YHu3=9qdAJ#1|VQ2)e+j{!{|GSr085!CxXe@7G0Ie?4UTz7^)v1paKCFZFcS8j( zs4Q<`SiAfP!=&YM4ARS6{y$zW$56k#grRqNAp>Ys@HQyFEwB0yu2l`PUyFm*0lXFj ztpIQ``~a@;{@n(-=G_LUjV1q2g7VYj0MOXfYlaUS7*wZx|F8|r(g(E&CNX?C3ib)8 z74i}k^2;k37?x)+FhnReFo0{Wx!^DX&u{))<7n{V=yFd6R`6KyQw9d%bdY`j7VH#Z zP(vNb29?sxEsqTf7#LU}b2G1n8Q2+kcXBd_?Brk&el5tr&%g^>n*bhTRoTS$VF$>5 z$O;0m3st9aAiMRk!U6^c&9{{U>9*h%JIhx3qWB5 zTIUO@c>;E}F*t(lV_{HP?#aLn9!&?YE)&`U()Sl!|LbRh+0U1M`L}VW2t$!X(rarI+j2?pNZ0(G91KEB#sB}<$;t3h$YuFTh6UiZOACXjkO?^M6F{j2w3gy= z;04fl4Qm2}6KI|ow64>K|Nq8JRRGT_ zgYp@;cPMyzrwD_Pqrpan*Mgu>0;Rm=91M&?N{-hqFa$96gL=3>mzVsX1Bx%b8{kl| z1pBMxKX?XP?;KbRlv|cKDuGrpg4*kk9YC!|P`UFue`5f+=FntdbX0<@-4vX#{LBBN z%MHORG9)FybA21`EmdP+kZ_U^>U3aW>J(sLmllx{>J(sL-YLQ$2u@j`v~3NVp~Pak zBm?8}FaMoDD>RpX`CkOCXkQ@2bHo zN?RBdm)~RHTz-VX6y&o%hjy|sST6TuV21U31#7|L;P#)f510*Z|LJ*x*{hb{W?)>t znt^5c_W$59COuoQ_&mtU$^ZZVIzoHL{g65pWVc2=SPi&t(F3;)|FmJ)>j+i@p4m4R z0JBSk?k+F+4<4h|lLd=pFaPp?Hz=Kh#wRvdfLb*h44o7l7{IML750Ci_KF^OCEf;2 z@G6)Ij}-zK7#d*fu)y*XV11Gh`4EKwP`?_wQVUU^>2-qABq)R-HHVYIhXnx+AC@=- zd|0%Tg<;Z8CWf@-EetG^6h16mzLQ~r$ASrlp!($T25?yfor}`|x7YvhgF->~0oV`V z_P-9;KW{ESb^x{MURyDMW-=QXydxAQ=s;#Omq&u?2?hpXOK5mPM;k!CXJ-eQXTO8} zAIK;6TOKP27&A-&`SJ0G4;2h5U|$+8|NWogdi;k49~L+od{`hMw8_)}v|>-7Lx4dB zT!#Lb%J5<3@)GbWyA_~SngI+9O*@$x1YawH#}`BxlpcThSi!);{%^Skcoo49aO)5} zKdHyMToRmWf*va4`c+jcqJZhP>^6?dAvY@fkEuCzy>CUiOV?|7r6Vt;fdC&!7ifGw zKtO?Ic_0Jh@)8E_#|j|#dk82nfn*#66qvyNV`2XXi4RaJ+Qj-{0k|GvdHex1;t350 zaQ_6e>W84$_d#O~pjrY{&c0S+5MtnA|MyrxL4ko~CksQ+ zV*vpshKZ2^3LMK@7^DM0G-xFw$dBL<1C=bG7DRx80JvxPSOC<20hM761_C_n|3I;c zFD9EA7+gSRfON3`L$tGWLG7#vupdt7s<+ zc%*6~Xte5e3&?cn7#Nm+Vc_4%!mtebpQ z!Vn5dg`g7RU&%zq4gAYX{yzut{`*EKEYJY4{)q@GLE64u3JmhE!7Xr*yFqPVM11IY zfP z^Z)mdRA)u`avf-5gX#50hGiZHs=yFR2B}WHE1DJV#et^P*vH$S` z(3%_2jK1T951T$Ks+_5gn?W(K(sfrWDd z1QyQODbFAnAb_M#5T+00o<9e`dYr+v>@Ehz<@*>wBeN$VA+TRwfMNL&hHs$$)RxVl zd1F6r34rrI|vAX zO6Lzp7$yc7fJX!@9vgf(#xN05J~|qM>r@E_l?aCq2Rsx$91{+CtpqBEA1i?NQ#dLa zfL3a$>=a{g1+6vS*#I(4K!HI86hjUQ1`LlC1SA+(KqZcYfdbQG1px&HmgOD{j1CS8 zAUOrl7=i*5NHwTL3Q!PW0qs=NSU`b;fhE8T9NWS>Sr|N)M>0TK z2fCn8Vh93{j&m$u&ENo&+ZC zd^qB;;lp}(%zw1`^rHcu4|wrm z`IrCh%RLyFz-z}H3>X3g3_xj$ZTXjf*Ffr&|H0e`HU~UTq-6tUgJ<1JU%PkoU9UVTb1GNg^E?fh0A!KY(8N6QOJGh^r1zuC|9X$V|{t|5ReCRkd zcs?33W1$T0ReV1La-Sx6Z14LX5L^8MSV!CPFaN-+e3Zenc;CVO1kHUQx$ofh-|Bln z>=#ALzx)HQ3{nQq*Ly^h^@W?#C{RC{L4S^N=;>OFX%gX{7*9(B=@}m ztZoX3{laVcmw!Bv5CH9UT5iF>6rdmg8e;;5sLJv$|8${oX2HM+%7Xz03Jd`b3Lsg< z$8ySSayAD*KqgFhimZGzw z0fUp`0%1@;X}SCVUk3~r7;C03mt@G<$-zXjyu*YzJPAoB*A ziJ%f&bIS7H|2{za*5LITDxf|BD4dmbLGsp*uV{H1AaKc zpz@gE!vzMF*TJ9_^vgXN_?=rn+;D0T;Mm!~;Oy-3;l|?yA8s(HI5`M#J{H)(#V}#H zAj84sEe!FXx z{@CHe4+fPGh6@tQH5mSjC@j}w0L`9**E?tmKUVk%8ZU@sV0awxk%IvgSH{ar7%U;Z zQU<=q4j&sBSYFqE^Z?ajAQRQ{mVfyVp3zoP1Es}pCeU#xvE@4%9)Nn8p!WUp-H@g_B75{@xi^w3#0mmd}Qm z_VpjcG|(kz z=^&Gw8UlEssyaU`2btzHAwU$W&cQ*FVS-abfXM49AhpmPb}k>5J2?c%I%|Gd?xYYP zvy+7ZloEb`YXn&Al>7&;_||Y({^h?RD9ja$mVf!r3tRUHYP0NQVYmWa_2{;fg~1e* zZXXALR!KfqxER5}@;Kq64`{vY|Ns94Kz-JoEDYYuEkUbA|NsAO46YM6QCldWu!gi& zKymUf7o1<<>+{ubF8}>s;yUYxO&>N%h)ojuu*u2b!)7Ok4_iVLz+jVYvhYbAZ5yE6XJqm;wYoJb4@d${hg-A0B{L z&Z<040IkqGAi%&_br`Zz{J;hV#;W$m8$fvt)Pi+1_yC@B)39+2_^@etCIcHN*8f+4 z*P>3C%CLcZc?p96sGfc7@G*h`+LL;n51NGrwaD1Nfa+0*9EcB*L9`2(|NeL3u>xrP zD?&gqBS7H8j{W+I76A$$_UvR~XoiOP*5xe>r@^XN_RA~ISYE=gKSJTd9$_JHDIx+M zrvUGGc`N|RxsD1SLGyx)RgsSsK&8cF0g$gBD}30)AmS+S0n|QWtP*`J0P^Ew1yG5w z{0l>MgusVApuQ(Vlf-J#jmwWPFiR|y-nCPNVbx9%hTY3E8Mr{>Xspw(bAI^t;Tvc@ znWMo+2FHL8-@xXxJQe`uYH*3hvXh5FqEO+(j!7CHzCE7s5!9Z!@KJ$59M8ia^ks+za5HVLz5zGP*b_ePDp@;r;Nq z0hB^PyLdr;IIvydbo1B%oU*{F4rFe{D~EywOpufcZVRYf=l&@0Q2^BX0{Ib?Pai9M zOkhw65cufv*xi21I>^ID17u;Zo!ZfVDK>ll9NBi zOj__!U=q_up~np$0~l16OEPhQR|Tsq*I?oZQ23YuZj~5(%mJ0|Y7$_7f%@RyJ6RYa zmuG_Z1poj4cjjY(25yE4LV}MK8bEbq38NUeWN%^cc`XiVae&($ptjBj$WBoCP)CCg zA3&wHeDo#Jj}8J1;2pREA6*`U#_m}z349a?fbDHgK`MFEJ_gbC8k&4Tq^rN2{W6=2eYh%!gmB+#j z3=9(k1QxI?U%|i(TJ>Pg`?vtK!t^>TDDE%Ge{^VIc&xB70z4XGx!jY14Y^PF3pASs zntNf|$->|cvipZDxLgB`CxP>`0&;#12A9*ImE)k=5>o0Vfc84?w+F3$V9XU-wY-Ia zBP8L1%{Mh(T7KmJ9MH(e@-P3tFPCFrUS9I=9rh6X*R=f0|9H?02dKvcYAGEEU|=k}zdVx> zJQx3G3aDSexc-0x17q2h#{v$_3=={GE=WLkTv|RB0B!XO5x5|-T#kViv@d#Upo75j z5(Z^x>(dpK230zjYclXVE_4vwYY5(`_6Fr3dm@|$A!x!I2ZyJY)l9c z*q8!!6(ihLEz2_*6hQv{;|X?E4yea^pn!p~tP+%$%Sx7iVc-S(Q)>BY2FAw<4NMFZ zB|$p`wH>7j7^Z@HawY#iBAL4Z>|3klnGE0_)E`-pxePl%^=p}agn|G^fWpQZSWH~< z|K@)CjWd?FFl>ns*f=9p;DW$%6^3;I0vl(5%U@85$MSdqs2%}@E4b7H@j-oB2}aOf zrJR5TA2l8afW|$RuV7#bSn$yX-1^955O}Q&s+R%;K5{{87YWcslRtpR{}sS}#D`VOzx)HQ+EoFk^`FO< zfBEkMTAdpr@ZrdEIR>`nCI4!cfBDw~N)^k${0{>8L;>7a`~V*RR|e0hd;s_V6u|30 zJ~S-<^6x2HnF!q-FLT<_;KL74dX%{cDH9z*9xB5`kg)mp?vWQ4Nsl_uw5AN|(TF@cLf`aC`9m8xMsAOyC;( z$RF@pY6b8}$b0bq4<+zQsQ0%(=E%PT>%9W4)sOrUU;gF)c5wX)9+6r89 ze)${z8fl;V|NoyazXtDMhl~e2>C6jLC&+5dsk zdVqj}#&QWJhDi*H9svT1ptQmmAfT85Qv+(v!1mFD+W#Pzf1ZrXd=4Dub0C|~LeTtV zkZV9O400VvHHv+9VE?o*NKE2abcz7ES&juXFDkWMg+XDt923Lx5(ZE{HgpnDbcXo@ zsx}4eN09oG|F4!yFeoCcSq|C7D6?D+REscxTP!H@5&{e$bHHXuFz}$tF^D4bv4z{G z?h5Ad89 z3&Ud27&j=6J{^3_07?VkQ#AO%y>`vze4sH_$PRiXaOr$M2{d!`*x{oF^z4WEkagam z)oP#_K*%ok|Nnok!&C$6(*!7dSn>FRfB=KYat(&$*LOkV?g0uP);zu-02|A16jG7^ ztyB%_5@6umDZ?<)(cr_1<(>=zps_Mg5ApE?P`eWB=Hnppe};t9rXU7TYYn{GcX=cu zXcpE5B=>2Xqrr#ep#6=@BN;($1@J2MPpd(4pF1KHE`W3BCx39A0~!$r%@m;d?Xkj# z<>2!e(8QNZF#eYkf~@=j*#PQ|{ttr2BWxc3%b-5*P8No(AY1Lm^ZYC3!VdjjOWR^zrM?$z`(Qg8Uq99+zpW4x0iQ{Fo5P3e0Q=iSc63V zOD$If&F3@lf_VQPZwg}21+8(~DZv18*%eUfp$OhRcdvE%m;dh(>(>-l!gA-wBcK(M z3ZNA&qPWu=C}lxbrLq45wX-2?kSP1VrLzD3KNe7QU}yl(2`DTt`Tqj!p4s6F7dQlz zmTUe0-=V-Dzx>Ppd(iON{~-c0zNMH8cSi$gwGmif{Z-}*92-C`vYyJc0W{9THkE&) z6KvK35?Ua;U+)L)$YKBTSYV?w!-VB};CyYf{0M^{bj_;Tay`;OFklmRR#0@pCXQl1Je_>lK!pAF*vtck z5jJsW1x06U;xPN+J}-u?_WWA}DxZV|m%A_+b}2A`*4`L1OaO%%xHSwq69_yn#J~Y+ z147JZ|MxiHVgclo4T#9jb_S=NEDWk(83XJx%QYDPOMzna94JO1^Ms)BX9ZP|t0Xpo z;_icbzy<{!Q0@ltKD9g+Sb(TSy`k}Aca`}9%R(2}o(J&y#Grcyzk;Qcsx`ZJT`G>Hb|#DxP|xnD0o~9T!Mf~qVM2c zH1bOz^NJvmJ1t<LQ; zAx$=b+XNZ2S=vU-FTld3eaU!cG>3B|DiIHZN~sXnqVj zafjt`0cbX6XBz{koMw9sS-r)?0BRlNfMz`*bG$oQ7y>{$qIb42ID=NMBB?9+|9Sb$ z|Np_Ob~qR&?Nnq~vHS>w6=(($q~9rG0|UgSog56B5Zji|{{J6pqZ@et2G45|&`jKG z6Hw0>w5AE74m{tdvXhgca3>3c_;M)*11HH1jLX&k-vjlY9z*u5KhdrD*MQ{2!2R}D;F%#gaEtR5xHT*VuB&c? z*5^RRSK<2@lyt%4tK1*9eAprZS~)X$Ckq2;-PP7mhYLK*k1&LS&CLbb`KlVkmIL=R zUX_B_Qk@|2+x8I*A9jFd50|q2uNU#?bYKt{V_C|@;1s~{VMoA(4|^v4|FBC!YPlxE zmgO`4{|AjKZ8BB>t#+BvF@Zq_)CQ5#1g|wd!f;w>=JFEIj@JMGZ<$C)Z8F^eQn#RE z0fUSXXon_fd=T9FfsCiV?gzD9*uQ}1dP@H94K)C*mH!XkB`qcda@{KhkW1x?!E7lI zTS^bazV&4Jm;Xz5iZDom=d-}8lEjiPiGH{wz_8qtVZm}qhHH*49~eL*UZ6R9u>{bn z+e-o;4sL**VWbKkwB-^k3=RSYpi@N*9Rv(Ov#E@w8laJQ2LS`NP4Nr{o0J&9 z=7LYY5qK=1!o)D~u>j^uCr~Q|yvFG_$mYl3byN~U%YXmZSZ?wE|8nSV(@iGeR0Tfu zY+A#Im79#fe6S6VKY-39gO(@oeO?MXm;e6vV-m}U1>l`IQ+Bd2ECtV)fYz6rIxP6G zC}6^eCCj0!As;V5UOguA2o#Gi!2AEC4uaX>{eKewK$YCc2fGM7FC_x*gTFWhPM@g7`~!Ft>K~(o z({erV8b}pTt-;*z*x=(v&{wnRpuOCX!C`p`zs7PyQ2oO9b-5vf!15BlL(2_8yO#L+mm4y$EHB}US#Ahg&&8+t zSm9$QSa;|05`nzsiXh#*@0Tlrbo1_6t_af2+qqm3q?{GZ z!awP;!N)GJDP7A;1Z1FYh=90(Ulrm8o~sbMc@}}j-lj5q;#l4yumUvF=B)UM1JYLi z|I5+f69=fApTs2qTE)ZYsPKtn`AS9x(E3@B@1Ng)Eby@#6tkek|~@7i=GB2A72al*+`R z{#wZ>K(T!eVEY{4_E|p`_}B-wPk;e5V~%Q{1jY7=g6-pg+sFJ^;A217J_#)LDNt&k zEO`I-vkNf$o}GRy0NQQ$SO7Fnh7o_D6;&LR#GmYbkpG^|fZI3uvB1X}44_#%4J`fx zwIM0954>jN8K^A=?w3D{ek|~DAvin@u-Iooad?7P-#^oU`%n3?z{f?P`sAYp7W*71 zw$Ba}o=-o(?0fq9vB1YAVE;K_vCo5I`-H*v?SR|2`LV#qrC|F!u-F$sseLlwUjI{2 zKOP)^Ppcmbd|U>$F93^u5ft0E7wkXKSQf-S=f?scL1Q%nA0r_4Eth2AfRr!LGRy#6 zW;%n*OapM4=>jS%3mBZ8!89YNoxi-AfoZuoq}3xc18j#FJPZYwdw@^VQhDs~r2;hO z{>1_`<_@ZdB`ZOz_yi-DcQWxTuVgsk*zth@Tz_t0U=WN1ne*h+Ybgdn1|IMZmj>{> zH|PxH1D8ZUaZCWUh9tnVr=*h>D;4<*Z zKF};Ds3i(%tAfUu!1ix?Ed^Th1RkdW*?$5YPUaB%msc`?$`%Qbn?ZAxmjphN=LYcJ z>L)dzdJSxA`D-bZ8PW@Aw%*vu!a%XD65us+PaNU4+B{AG%``*&2C@OZh3p!5Ku8JAZ>+vU*km6#9q5g*(Z@c6&zBCsfPguur&%Ps!@kTTpU z#vr&;jKTP^0BBz-_{1?5@Z7%&3aME(oW#|4r()i+X{i7IF<*3#`XUH zat!#y;TRae09IcG>ZR{wVQ_-0WK@9f=-H{rV7Od`fycBWP5NJqBbB)b>4Kn+dd1!0rXlhe-HL0<~@|89=EPzk7lm z7@UGXae!ujmxJ7+@Nw00MGl7LC2Vh(E3$y-UCR|gqiSsJ%N0R82-rfFD>5`JFJV)J zwAEPOLE37pyOG=*L9~0p=f6l;k?mfCkECwJPz!uw!^aKFOV~keOo!zqEIg19W4;M#-7+tOv~HP;A+1|x z2T1FdnHSQ!Wx9oMIp=z!UEU2&l?)WPd?UC;y>WR7+bu|3Go?XX&SVU6ITHiK<&0+` zE@zwtaXDi;#N~|Oy=IU!%O&mVrt*P+ESZ^;qEJ8qh8k@QNl+h8KtyFsLsN zDlx#P0II<1j|R|L#Hi&3sIAT59Q-lSDL{a6c{gf#A%1yzCu0$)!~xm&@Dr>adH5E5 zrZQ;O^96YQ%m+{}A5vF28hik|1AO)#=+tFUZ2)fZW2p^LTl{FXfjD@4=izp^t)TM- zH-K7jptYFGH5q=O*a~Wyh=W(+P-bf?*snEkTg#W{f$sz0TAl~K4*;~zDs6empBQjI z)6CJ~W5@DH$hv|bzM$UaAODFB9~UhzVVwa=VdCLn^Flyr)Y0J+2PC!`_%LGI;S-0W zqXW3C6bH3tKxI=9NEcWoMp^menlp0%gA+6OJO$8<`5TCPKx<0i?qOa2<;QA>jhx`K zXuxJ6my<{(wAf|BCG@|Q=p{7N^cuKJ4L@-#H$*G{4Hy{2W`NdV>||jmhO1;WKra6s zm#h3Mb7m=EaAFC7oTc~W{~M_3u5dlLOphbNbY~ox4`f zeNEWyfA8WjeGNg=Hxps{DjcRa5;VPv2-C}Pm>vc<9XUlJ1XAShSnMg1?HoMZkxfS` zfrXQxZZCqXL@oCnm#h5N!r}JAa6Pz8*ClNFukScapFzlUp>IT(eguc7cf;z8gM5%F*pdpxjAg!=`T>Awk^{*xJp z>2Yw=k>dd=mW4oPK@zA&Fqd7-$&4N z4kAns!C^WgAK-Aiz$GF~SHxj@1R=Kz!1_~oLgwQ;9Hy(nO-D{UNGVew1e%BO)QBIq zU{9G$pRk&alrjb6p&^4OWqxeL?slgAgiPmuNrdTsI85&$X!-^sOqa%Cx+6i;O9`9) z;RO!U`3Ra0+KU3pt9Ziw!&Y?DAtl1sEfD%!ENE1B`IoQa5dPNyDE$SJrx}kEbj1`R z{I~<%6;RWs5Hvl42-90}m|lt1bfkR2FG4t9eDKDeFBsk6rsF6v_+YIqJbC(q1P-?& z`d!F*8Y$QD850hf_nXj7hs5@`kI?W(^uKV~(o2Lb2hnYTn9e}3KH!73cJYMJ`+RiM zA%6HV1xoLS(yu_{am&B_6ov4AIzZ^}=1@8bO7}w4ecu7$e>(}Kzd-3p5c+pBg#PIb zrOTl7LJ0la1(H+Vr$G4Mq3-zp3c~*(3bE%W)SjP@A>zOIptL2FhU)*-59RNI===2) z!v77m_g@u+|IZytGeGFSXCd^z6A*p>rb76Cq#*R~#}M(~8=(AJ2>rJSO8Y};8Hl<+ zFCqLt+n_Ww9R37A`LYoDAJo79Vf)X5FHO$~0 zN_^5Fk>fFs6+Wy4?fm})Iz4;+#~-kHP$uw-^K0PyKRJ9sYOjIs|6~K7wtO74Tl^fT zt;z&G%l8`i{BI6RusZPh-)!L1mXGT!|MGt&)PL~(-F$l|F)=VqVqjpL#K6EbiGhI` zbUNOY{R}M2IT$1y7#P5}V1ai(f=3*eM?y}Kd0UHcKksLdUB~`JEMO2=x{TrfR5k_< ziLT`(3~W<57zCELFx~*Uj`t^6)k|js1`cqkqsbuPWFP>&jg_$)w5oZ^eg>6k4h%fg z92i6=o94PTm8p z8DwH`RQT}0(cvSXW57qQ#|5DM!2tyyjs!4#So2r_v@7_r0%-p*@9$gcPU z2@DL90ic~xg&V=ERzdrL7cj8w6l3Vv$-=O0c_jnK;|9?F?VT(PiOat*fbQd2z`(Lx zngO&27qqV%bPq7gR1Wazg;&8l>{%u{fY%TIdJJ0E2iof>0NIK5|FNTZ0K-(!ZI&hf zLHDMJ!DqL#r|b`4iP#Xpv2+{5|9}l2)-2z_!1%ZUG|K_r&&~qci+f4*BbNe$qrgW# z@VpLq?e8Ul4?icscHxGA*9E|C82$;FXaDdMeV!fSQ*asy0)^*Qc1MMe;Qez~nIJb` zg3`-l&{_i)h>wkUMSu z|A-X$uwr@3zu(|6fW&vnf3C*~(3Fx5Is<9>m;Z{OFy>hU)~h1AM?{6qgnh`qB70b9bw;FqsZmY9V@XMBZb3+>KU-f%Yg);7N;{Xw#@@+M8a~Q zJCB(d7$g!H7y_6W7&1V#0~9bYxcVrB_y>80_$velxH@~fc{)3Wc>4Px8Gy`mPRs*E zR#9qVib6(eVoGWe14ET(fDZ#G7C;Pz+{EGHBrh?3Ljc|2_h=J{aDh5Rl*bPnrLBR@%B?<-=QD^`; zCc#QQgZ&lC@^eaaQx#HDi<65o3rg~fFjTqbD1_t}Wx|q5a$-qhPJTL4ssNjnnVXoN zs!*JoT#{d;U;uI_YW{$DA`O(J74q{GJcIppEzQgf6mnBjG7}ZKFl+_6t)M6~H?gP^ z*>Nu5jFewgsZdgsnyOG-QdF8;Qd*R%P?njWlUS0OpNG{XNdC(&s$^h*+YA>|NJ%Xy z$xz72%*`xONUcatO-)G!9EenIYF?>AN@7VO#40@nQ0W6I5>isr5=(PRASq0bfdOPiL1IxZBmpX<7o{ea zq!uZZWF+P(4T?U{Wn_iR%jmgZsw0u1VhSI#8#H3VE zaqHvaqX#a@!No&yW;&<@0eRmuFD)NhGJvJZQj0*gF)+9|2D>O^7K7`TwEQAaU4tS9 zaw#Yklqi5?OENNx6%vb*GZ;YS5yVZPumV{FHW!qaLB7aOQz*$uRWK|rQ7B4H&M!*Q zV{mry;X?J3XMm3pR2o$HK#Vg4*`|PESY{r`Sumw~MA(gFvL491JYA4KkZUMxE=JXb zoPWX~1)dKmm=R7jVt9J(@<#>+&pb$Ggp}F&X|VDU94n9%g;h;)Y7tZx*)B*h7=Q{q zP~EiuTzjh zPGV(#X$dSn>nJ1^7nkOO90yHv3Xm8MD9TSSO3a1jXxE~m{33Av0x3n6^vj2&2T1s4 z<|zb(Fu3LxlvFC9s)K|L%T)|&R0~?(zMpo z)rSk{sK(pb!+8l-$`0zbN_vncReDAyR3v3^0FhnybK<(vXP&hJs>Eqz<=Rtm9U|`^c(jW@dZUpiFGcYjl3b25L;26|y1t*_^ zk|Lv0V38?XvSW;RHs>U+%i%Z}k!N9=q|Ns9VApZaV|3P5|V)B64AnYuCuj%?WliMIc z5JuxSTNtr}grOK5qznv<|NlV+>KPaq?4fKB1yc;`7cet0Fo4v6T>(;nhGG2%c~GVW zskKtbgqHaX40-tq#i@D4so;hYgN{ORX;CW3IxB^u)Z&7CkPHK;fuLb%pvk}hi9y}s zUGsl+3`6ge50XYa7|^#i=Ew1y%}q`3k|Ve!;E~&!hMbq9QY|ur#x%6596g z^a~9242ndAOj>?XZej@*eeg6NUyzudT8u>nI4y$P-^JkYRY*=ONKDQwsl=jI0Ui=1 zMU{}Sfrd7$hpWeh8aEKvgJTG8E|;Q$Up}~w0<}UxZ4%_r@znQM02gD>`VtyiNjcdd zvz@`^nH4mtfGg$P%;Ma{lH?2qn4GhV6S(E60GEN(Xi$?tX$De{`Q;;9o>-QcnFDqa zD3YK__dhhfGlBS^^bMkI!L<~K#Q>{=;56tCs03))^&eDVfs~zrfU~8UCHaZq*e)(e zOil$i(m^Q&+`%s{$u9zB?4r!PbOxx(#G<0aN^nV5oST@F15R$q`FYSL0K7E>Dte(w z07VScG)^rpW+*OB&Vcn*%QH(d6q1u|85l~iiYCJgMuk*RV}gr8L7})fGX*jRk_=`t zfW(qB;`8!BtmKlS9Np5q%o0fRoWV1%B(`NVopgcgF>`(PHJKiNJK$F!7;fcvn;cu5}`gQ zwYW4lRW~3tFC{ZC9i$#?0zv?45<&oKB0>OU2c%fg1)HA=GQ|Zvm;@I9xvaFfAT=)~ zH3jOH;1Y1z3wOC=PAb&pV4s6~)g@rF0wA4ekiO{P)a3lUl*FP+2oG!q!YQEQ3SzSY z)Ob*sfJTe*K*5Bp$2UI()C~rQOf<+5x}kZQCAzM8VE08kl@?cm{NW7s8B82vIz*X5 zG+2vsMrv|4NG+J>T9KSu0I>|@8mNARa~Kq&q3uLnu&rRvf}%MyKM(9a&%EUPBG6cy zu1{)SdPxSzWVjB9RS+{kLuHxydAe}>p?2wl5(uU!*fzLYu(fH4$*B-mz{Q<25;OBM z^U~q^6%-Vb3leQXDO5qBpeVI0J~_GEmI359P!koxb4|`pQE*8tPEiO3S3V#YC}^bR zmM9pR=q6Q`q=H*LxK$Y&Ayk1(hM3}II3lx94{-q_l0f|MKC6#dBM7tK2W)^@(s9-`23eh0f!ng|2{$-$S zhlq>Zf}GTnR9%<++{8?XXh3RFZem_)UWqQagoVZ$xYwl{Qk0mSt?QNx$%C$WDZ2S- zy3Un3pjh-T05#q6i*$X9!8y$@UpENc9o2OM4Jv`nam&m})eR`h2SqN(D+m0;y6^0F_9YDW!=y3eNeZc_pA2Q-G$BfQ-sw1(%}C zvQ!067qA3OCMUmKA-OUKJkXF@Tnv(f>H&>8f`t{-6f!cwil8!((pWz?wYV5GHix7q zEw@BBB^6wt_L^W&^0uTg_>4Sl%KAU2q`eZ<0T3(J&-yN+P(+36=5>D zV1@b$8JW5u$0NA_#8m?ih(g>RZ2+?eL&gxNj1fo%F*;pTng_{2o-PW`&gER7S`Uu- z8UFwOj$UVCX{W;4tFU^OA7lcux|GD?6y3~%94iG>n}a|Dz@WY*td$1}u%g6VE(U}S z&j24oaPtb*=G0U0EQWMeK?CWa_DD`nC3p}Gl;-S_^`hDaHVMNf@Tdp0!X?@Y2Cyzr z^$i{lgtmqWx-7LKHMtZ#i2%}?3@Z6R!zoC<#c;6^yafyo8&K$i`wOse2ZcAZ{SJ{E zB}YSGGz3ONU^E117XtbJ|9yp+&SU|h!DF1rQ?G0gai#(Y%>b)irJ;OoC@le{MWHl| zf{xQLgZic*F%A&Hz`y_+Hv=(Y-Bg%<7!8tR$p8QUH;BOj?d#_M|Nj%jXMj!uL)lcM z*6qM*7&CM{ilGZK`o{vg=b3?lfkgr8GO#Yt*f5yE0QWA0%fPq*%3uJEB_ivAiI37m zg#dUg1QE_e>A|9i+4uqj1A{TVUC3;l!oa|g!@$766cilbXB!cr;O!a-qMZF*TtRf8 zk6Vy!L<9pvaEN0_XfTKw7OW8D8XN#(GB7Z>J3Ct`Xt?`@YATrMS?HN67#SFv8kiXw zD`*6zrYLwMmO#XHjV(1981#xWic5-05|bG8GC@NY1q^z5`6a1(>3OAkNu`-NDY}^{ z3}9YHVsQq8UP@(Nab+%qE-7NrD@x5t1PL{ z3@bP1LCa4@P<%q>(LlNpfkB3Wfk7T&F&8vNFeouFFgPQLD?oHGNHH)lcp}6dBOu}&pty)eh^s;4 zgQ0|>FtZ?sAvG!4D6u%1A*(dEAig9&-ZQ`_J~_WWC6ysPwIn_#KQW~su_!k_GcP5z zf+0CSuedZfHNH4GryxEvuY{o>F)uTjp)9kgBr`w0IJr18zBsj{v;b~ePJVfOc4|>x zY7RqYK~56HNf5Gs2ZUw-MG^x8Gt}n{{h*+!2gyMs7+^uz56KG*Pz~^?=)V9`%m5dM zg;YN*N}%SzqNx7?ntEsm_P;=ggYAa}VgCn&1c(I-!hUE5f;tEurxUKm8;{x5LgDFcJR zn*#z24C=EliF{ZtzyJ~vjo-<_unEMH(mSBQz~HODlZ7D(#1~w&T#|voQQ^b#<(3Q_ z&W;HTPL2v(%cU6@mz)3pbzlJlgUeTtx_ZH~#|{e^7+65cms>K(I0p!vaSC)`fGcfa zU~pvs$v$Ri0IAbtU~6V#=2$Mzz_`4H!E$*81H*v{3=A%x92-9Di4d5;u-t-yd8yd{ zpCaa+1`L8?!T|yk7G3=@{?F)%G}VaQ&t$MAo73B#=A8Vrm9 z3L6_D6a*v!6gGBvI0(o&IBab32oR8Q2-w*6SV16{VFK8(b3zm@aDW_J->JYL2y(5M z&|?FT(ID44Dty@U*uWr7Zgq<3@PBS zQvd&Lxh6xya!Uqq)QF@Dl`n7kKLccf$i(G(;Lx&&P?!LgRk;BYSAHv0ue7{{p$5d^ zT@Ui7+FTHuG1kLCAP?eKuzOt}JAm8{a&d^m1rCsdI~^DV#YC2xfn#~kP7wxUkkdiB z87@f_E&uX=1=zfBkcdF*V*yYkKUNT6VPJWzuyF!|%JMJ&j~!5`XJAm11}R}wf%{P# z(~tk>!2Aet)MEi~1jsRfB7kkV94G?*cR?%xB`)Waj}-(M7$!hO9xGg6U}$)3Ai%>Q z2C_@|*77g^i$Ts)HU9+?5HJIA)C9q70}xyDA&AXziyaIOd|+U_eFP-#a`b=!1LIw$ z$%oi&`2K}mXAoczU;t$c2F=Ml9FmfPj|CWaFsOh<)Go0zZh*>4OfKLMV*wd9c?O3@ z{p>Rkflhd%q?d{@Fo0rA!egmA1A~}i zfWia@3876U7Z^4vF*tNMFvu+TV_;t1!oauO@BjbhEeu=}885IdFJTZ|zKelz`Lh2H zmTNFD>=a>8dhGB~fPv+8{f8S2EbLznxIblJ5M3yww7lj2TaZ76ouLVYU#NV!3j@bv zfeUO56D7r$m;ApAO4ZB1{J$?$zI@65ugkyuzXI|F$b(LQCcooUjP*Y7^#A|={>#5G zFo0y(*|3THQ{}2EvV^nQ-vM?-ysa4tr zD^-*VC+Bdbd8slk|NZXAV+92U1{Ozyj~pNa6}dsVm~jHgA*#+-nLly}Ff4Cj5CK`M zG6|%D?;MD&Y5``S0I@YHC(Ceq>8RR1R!CrAV0pYDL4twhlHP}#0t^Ql7#Nz24-_yk zsJcB?_;7$h1(e;Iv?fpD4$*1^mm3Nlpa5eK2AQNH4mLH|QQ^Z)Fv|}ls{Vhn0FRtd zinIOZS4<>UN-D{%8JJ0pw_qIFIV`-+~;E9X=E=uz-teP|PYYFesZj zCL}0;RPYvnbSnP_v$H^Kb!o6EBFn%0KLFw@L)689*_RGHWvFN14Fhr1*})1KmVfy- z3B*_C0UP20W-kS^oj`20&meVw-z@*~F98~>@R~!}clq!C2Fo=W1VCZgaB8_H10zVM z+)_}~G3bCYYQug|m@B95WMSyq$-=a|r20JnV?nrA;Y}Z2)lu{YU-8`E> zK2~l7v)6#wYO!Er!a$B}aGuJz0F(sHK|BX=4Wgp-80=(-^VvTxuV7%5Dgs-i1Ih_- z%UHo7!YZM@{LB9$kjYB6U|Z@dK^$dMFb7;>si}ailm#n|oysJ@xLoD`$*G|Na?4Bp zADy~NKz@12|AYIP1(=qX{NL+1K|mE`Li1OUqZ*hVD|}eNpz>JY1E`EDU|?u|3rhG( zP0M#O*grP-u!DhRYKH;H6VhNcER&T4gzE(u80`0fOi`9c_0Cj=2@K0i{y*LCJb`g} z%m0Uu6(%q<$br1qybalVvi+d2gaoDHDX?GIK;|jE0J9lDY}Jh*fBjhnjz>)f3CF+! z24}_whP{k(0U%2@gQHOW|F_VD3oOfzFt9Dp`u~6Vm;cKrpAs8T0$@WD_Bcpr#!l7d%#MV1U#yG7}K)P&f_N#17J=WCmt4 zg4imXzy_>-9PnWQ1Iues1||mH2*wEvLZBE}Jb9f^noUy-nz1HG#^!^J;#v(dO7Rw$ zy%fY&2?HDJKRG~HK}hkl#KxuD7#KP>Fff6Yv`uaj*0+C$WQ-@h+Gh4`Qnb zfQ{h;zXYjLECB1!dF%j6JrM>DGRwdGk9fRc0~Z5}(C+0e z|J#=DV*d|O*|C9v8?0J%`IrCwlM_Uw>J>DX|Nd{WlZ9dPP8NnDkcGeYNGt*=;#~mB z6O0ug>uV=H762vO&;=JbmLGv6B>9yfRh&=3M(+l*?}OON6Tqf)g4nfX%OxEcCaqFn zSgy^$xLo}IpPeiW>YxN#n+eh(UH=IY4)WlP&AAt(QE@9snsYmdt(*ZiFmb0CgBHk` z*Gdd547}i+!~vG6099AZ_5Xt!f#CW_J_f9N21vK!JTMzlh&qGS+khRQ338g-q*VqC z%dO#V0~bQIG9WF|pi&ZAp~;(oL^#34xnet*4KB`=!NvJ6k;(f+rIKpdmPar!fOD}* zK){C$%Xcy`fYNo&D{$bkyxz^g3U>Dn2A0Q(8!Q-Dz^>s3RlOjW=3HAYsem3>S|GRP zoCH}Z9W>cWOr7!B4&VD?E6Td4|cS~0}5 z$!y|AVL5I%9c&5GD0K)Fx&OiCsazmPgacfaDjWl|+rh?KfQ>a={^frh$g4Fnll#P# z>T9?_N~9)(41gpzIVF$?M;u6#{5LQ=0>oD01RKBvVh6upE@^-gf}j`yC1Cl#pmL@* zVfji1hJb(%OTfNRS)Rclv6F>C6Kn$OP9_GU#|9rZFsLlIZ(;BRxhDEJ$Reo#gfnFC zgG4x_LDtEGi!=^V5L@vW$dDiVLG6v`HIwrsRJkASWMS9=QZ4B+d7gx#zdlHv{JiBW zARPqu+snWF13O!GI!G1!Ww87YFdN)DQLF=-S-$+sza=|a7_=N692j@9Fn|j%Nt?-h zlJbmyChJMcFi9Iu_K}qHlVAkp*ncO%?I&>J`Vp~{g`o*#qI57MlK<^m{^j2?kWI4Y zAaQnZr6v!~zwEgnaYZ?>t>Tk+NQSt{fQw#saM3H@3)TQGdKJJ$@AtdVG#&6^1p^Bx zoUK+oRt2SONICU7|HBVZdttJdlq|ck#G>V2{z^~IlTxlfwUdbf)cAb@3go1Ikhzi) zpg{hA8{`C;pXfZ5>MQz0L$J7f75p8ty;FZi&5fo1uZf3=fkq|Kz{SwUUxDkHG!evm10|G@0d z$vM*Y(wQI;%V(gv@o@mC6};S%0bGMt{+oPETCQFWoD$eP!H#;$z`*7L(jemjHo|WC z7Y4BLl{Y~SuDk{+c#zAT<(do(U>R^Wmk0uR1(N8c!Mz|hS&(UR)4-;QgV^#XKwkNF zaQQxP_oKc86!tPD5ds1XkpdSAU?qd}A&?fuT~!zqvkIdJf^ zf`eba6s$S#@r4Vukj}I?hvX%gHtAxpE^xnB4&1_HT?sNlJ_@WW7{sou0kaE1-83*i z6U@)t$--c?lZ8PS#QR;}_SoU00t3tI^`KTN`rCxHlr|rN9jeRxyyFvOmE}1wd@MgJ27Gfnr>`@GA3$7(~BI z>HtU`%O8+B*^gj0xK@x`4%V~a@r4bUkZ2IILPUeqQgDBi?z>OnWa0`cJ zKFBh;0I+6HkY8jUM_dqKh`g`?)b;|`;!=KKr*wf-%7TY~SXw}AIa9DMy~qn2SYSy> z3f#+NDLJ6PpwA$C8LTl6tXLkbSOUa0e~QS!ju$>G0Ew0VpL{}Isa^&Wa&C|$1?uL4 zB&A%z23dpZh@C79Rv@E(={$D$D8az;dj5wG;J&;91B2viusU^+9$6PKTM^8@1k&;4 z^ztwNy+EBaXfn9Kz#wG{_VWTz
  • z2hN4i@w~9fMG8CW%RND6FfRiYCy>Gd(SU*# z7UkgLTl4@TlqLPay1GF&%Yai1b1R4~YX)|OKFCIy-&dIzFo07GxB*ixg2nuBCQxZq z&JHq86sZ|4DGw51js_Vj15TdIp&+&_8_0ss|G^gA0Hy7+mvC$JA`~`&2WVu@O;V~d zk~CfZ``-hQIc!Hj&9?pWiV4e07^Z?n_CrTO=7Bl_W#H8JdFA9aN-6X z;39_!T;#}r`x#8&B1Z;XpV;{*nl*Zm(8V6hIWrT+i_3~n{aYyhj$ zMpN~afkEggs8VJC7ua9HaVB&J@U$)iju<8@u4E%PdV0TpgUIq0hVI7-69gDoKoMGc z5F{_+1+pDdy@-RyPME+A7#S9jEE7N2qyr$Qd@|n|xu5|QK;Y~s4zA-E!GR_H2c($s zCs^?+u;RtbS28RCTQhC)A{E>EcCfeILpLCL4B~BIU57xrB*6otjC(+A>0Gc0X*n%mGu%g1A_ts`$uqYzW*TuF@^|o*2kZqEMW5Pv4OzkyQ<1OQ||2) zX8^ZcWJ)LVs7c!DESF?pY!+nZ0ILJLKspK9R=WR@1JqV}4K@|j8j(%~6<6RvIR^#? z>F~)>YTBN29)soc8mJ8`EwJ1Is?>k7fV!@{00V;w zbTkE|24pZ&arSa@h`JHe9RA52>cNbGlkcckG0k2+IYPrpM9O!$B!dE|j|29J0t3VB zn#t=l3?cJkyO8b(Y~oRf7lm6&F0PY%$OW$c)or|H8Z zwH7p<1sb(l^kMQbO&umF+sR)vbyOA|0(t(Q^Huf>0t*;GO@4=f59^kfFf@WEC!YTQ z|6X>oi&hAe6!+v=TGEV9CU4U+WqdgKjh2*@p{@;hxu`O~2a7bKP+VE|_{NpPq0pWyN@|G>4Rq{L)D9dq6{Aank{1tls;&dHN> z>?9WEc^KX=@o>CfHq}7^l#SMbBqiQYexaksWT-w_LU$RH5cA{%x;kPK6=2<#QyDIR z8Z6Vm3iBp2=^2ZjSpMbz(dCBV;bln)@Tlm5b(6jHGNdHLmfvJxaGd#pVfju5a5eYm z0$8K;eEa`vVDY*T{|Ab4@e`mFIR7VfvOpnV@=<+$2k|{%(U%a>5QPBn5ISVk z8f1h19rk}975XJR(!_G}cVE%@V4Gc0)1|Jp# zIDA;*Xz=01q=pacmRtSb;Slg)QGmgR4bvJvtd+1@?#Zyi(d7ffCSwPXDFz(|3^GEd z#>-0>I3g4jz-8F~9toqJA`Hrp9X>{Yo2eF{N(2--?Cjv&2cGE@SKKUP=*P$;Ry#Sz zXd9E5>SPsTB_=)N%^t=sj7)zIOztz0uHXM50y1VVnhL6Eb*n(xDnMa_4k$!)yO(P+ zGE4$h@SvGK_5a|epl;oAONJ+)c+jn09?1xH(cgXG@|o)@;|9YE;Mma(UEadLi#B1g zAKZca0tx5kTL1q~)-g4y2RR%(;th7Ts352nGl}KHI#7ikp`f4xN&x?-2`MdaVZ6Ou zf`K7GV53Wbzy^)woeVn5Eg6g*J2o&Z_hf*~3Pysu!~g&Pje9Jx!I)vfa`2oI?c_7|$=41I?Tfx)TZ!}b7&51=tFjpddMLOWR)JeNl@u!8j{f%U{c7WlB4K`umK;hX@0 zg>!bwGYAF!|29_sef!Sbl_27!-)0T6oPw@OXp5 zhb4{)AJ#h-eAu!43quoRq|QU&!$t>z562`7K~s2-1vYSiCgwgIVVLMBCBP71Ai(h0 zK|p{(B|t&I;<3SpV+<2t8-uGwV~5GV%`ECY7#JKJ3>Y3O2uLunI5;RUJysAFEZpaZcqz)L^_8jiv{Ss2`xM>2p1 z9{<#WJjqZLA@E_z;|UiS7&w+oF)#)Qd|0x4HG@NaB&Z*?grUX5;KMSIe;o`y9C0xC zuztCO0E2_Vhhq*3A2vRo02=2G5cqHaY5}Or=C(W%GV=Au7nDN&CxWsxA7r?~0mK%8 z3_yU}H-fK0QS{=;@-P2=muoOEEw^A`2vFF-5KwQhL11|V0~^R-P)b;CiAV*16hQ|6 zdk!*;_Xfy-AK(^@Fu3dVgAXJwxF00`V#o3?|G?ueyhlL30*QSGcbA1ff!W~hvfxS( z`^6&Ukf?_Q1vnI3{x^V}vD}iu(9z+;y5*6eX_o*0f8PT+1JcssoeDDI`$dq|!so&4 z(;&7WxO@Ae^s&Q73-BxwXuRYLNQM_I^L+=1Eesxe`wpJ|5KIQ?eo-H@{L8tU5L?h6#D3wi{L4RZeZm_8;(wn2k`o5^o4Q46 zIJqZOota_Lo5|ZkorGSm{Q#X|6t+CDfPt}O)#Ou_B}^rjlO3#zOoUE?$}FKj%YXm- z@K^zqR3O80pvD5omAt_qd2RDc0v~QDFdW#xz*zij@-r(R1);T%9X>o@PbWcFmt-!#* zUk0+`S-!Kt2GGdF{K*q+44AaECLglt7X^bU_Z|22D9~PENBm7v=%g<2zXx zyq8;orX>FV|IIvkgRQb)CMYry#k!#Fgfg@IM44i^71(s~mjz3i%%|02vPwD!4qE#ZgV9;Ok=nu#f@+W5Lb? z4GfI=ER*dV! zlLNFc=l}n27bd54 zB9oI`%_MSfEze}I0V|UQDP!1iK!AZU_x$7yu4kBd&rGg!Q{&*h4U+%)ck&uHHv={= z5a$DUz>60={r#Z<6o_1gAn}J<%fI|<2Jw0GK>VMFCM&tCxN(6;(?5V`w0Vs{5+A@5 z+Fa~lcEIv4|DL`UXE0#keJ#jfz`*lZ0F(nkd4adq(cr@mP$aT_p4{UeCBg#^;14{A zNc_Myxz0swvVaFyJrB4r|LL^+%RgU`BY3WY%>QY%{L4RZ5y!b6l#Ft-K%(!#6H`1% zU^aN1kFyJ!)Igns3(LR!0T14Bf=lW5;I#@o;8BzJ;La*%F<9pw4}}FxdzCl(EkE){ zRTE?>XDwI)+_2^WH+$Z%0*gn3#g|Op=V8g{Klz7;Jfr<&3D0;&{>c@daZI1?Prl)K zow0cG2Cr;J+sQKCava>NK|1aWO?L9`Wt=|wg14f2o1?*p<)GQe<&lg`pgapw@d-RM z@i`(w;erN;|MB!>37<(~Y+GL6Wl&(?S$d6uVfigcqItV`@&O+cC9b<5b@xn{fBF9& z(YfRn1+^PNG9Mi#i}}{@aNUFneE73@p05Za4_6>uBxmwIzY-oUxSkI>lkNN~nBKpi zyu-h%o^v8dp#*p`_5I?24GKEo68!1^|DQr03oL*)>KKB-jk*#Br>o2tSQfg#2JaY~ zK<;=A9;D|4cVu3JixyV!Snr*rtIQV|7J9%{XM`X9yDGDDQDR~g2ljndQKjQ7(|{OT=sHY2I~fwuK$)#76>#H zy$4;v12%>I3n-OxvP=#OoWsPick-9O7cmUJAU&_Z)1DmQ=zj&y94sav@!Pt~zx>~} zd?o|K@=gZE?B%_+TJOp}Eg#U`)oUp6^v!n}G$X=ZB%2Jk9h$N&EqJYZm8I`RMi zk{t{TOa}k|FRfr;U@G|kf9V7U1}4zLstpVbOe_BXUwVUqf$7Kp|H~p67??Hw|6h56 zfq^;W|NjjN3=GU4{{P>0gMop?;Q#-fI~W*PGXDSH!@Nu!q5| zowtLhle>$no3n?bm%WdzpVfu$025OY*ZFg2&zwGW^2Bkz1&mJIkR%BrLFUH%|GyZ# zfK~z|0>z-^)FuD_F9va?7~Eh+fw*7{lAi*VS7Pvi$@`o?#|L(s4g&+jmjD0f2{14) zGcmZe@nJg6i*EvG5jxnodIpe&C;$I10% z49r=$ELfq;%yfZ^ZvmqZMEA)P$B#k1-@?Ga7y`*TY+(%bI9&LFiKz+^c*pn_FnVw^ zZ9^4t;%2&qF2c;b2vx{~o2doi2DmSv9=TKh|Nj~d1_rh*xP0=0iJA2Rs*zBSFqkkf zFvC|Nn3GU|?XX_hIlsPdF!-n3;s2T2CVD zbzxv&>iPeFdjbOklN5s+s=ffe7fj5I`%v_yah*JM`pnsL=aEfeU;vqO<^TVk77Psa z%oYr8=xNuB?*KE?JFXMRnp0o~pFlE8gn@xs=Kuda8Vn2!pcD;GQVa~BJr>L+Q1K21 zH>4aPh52$Efz|7>v1yP z30?R)n3*O)A_yL*A`A>{BLDv%0d0$L0VT>ZaGC_|h+)(D|NjVR|HcbQ+F*Kc`V`21 zFqUCpVDtI^|L6(^28MbW1~-t4PA~<4+yj;eWgoU2r~(I&!Jty!i|+(e0!pR;WjVGf z|NkEaZNGqQmV)T_VPIfe^Z)-b&|J$W>~Rk=jp-EE`FdFW1W7v~h)9QflYs$LN3i|* z|Nj_hXUjB*k6`s&3MjRs*AY4l4D34p{~rhCjx7vsh*~iKq&g0gI>1Q-T>pV97513_ z|Br*3OGda{{DO&@xdNgWJ++H4FtE@0|Nk^7DoP-M047x!7}&S`|9=LQhTb3?&UA(g zR4qGkGeGTU@L^zJzw`h98Bn|K7`FO6fNujcGouE=da(1Epnd_l2P^=#m4N|d9*55V z|K~xwnsTsv2;4%b4?vg*i2^T3Dg)J+$Wh?Kz`#-S|NnW=elHyP*$b3NccGf+#*HIN zZ5S9h&iwyhe}RL6f$<717hYgyW?X}6BCLS1VPN3o`Tzgo0tN<-0POn1_#Ut_vl(F5 zFT=pV>GQw-|4mSw$st-g$SoPpoc|E{D1>|#7r3e8#GM4L)?fh+PNbZ3{{O!T%67!4 zcVS@QJX8Pw|1D7BxP-8PDGY7{7uX6X?l>4D3dZmQ%RnqQVPN1A`TzeGXjfwb!kjdS zF>c&m5KV5}K1iAv7#Lz07`W@pDs2Cg;#|KA3c576!qSX_sJf$PZs|F?HAFw`^5Kx7M0F#}4Kux2aRQjiXi zhA;pB-vJdUO^5>N08m2aVPN1^`Tze8s8G6voc}=;T^&fznbTlh zHT7Ju_DKMD2%HT|W1xbEJLmuZyPzntLj)}}hTOQ3qXFbl69xwEIsgCP1v#`1*`Wnc z=lOAafvo3S!05;Ab@tr(dT1U4X}t3P|J@7*1_q)uLiO1&F!1pF|9=;hhJ2`M>7*pSbi3dfU;JYkgn<`=ZSz((dVVfkmKrzk0zyR9=3ElI^ zd|>kb`Qn10WCOAYiWw(tR$tK1D5wky6k||`gC&@iCm&y!#bjhL*?3X3SZYRmT2W$d zYJ5gY5rZC#H@Rm~_2l-&0+YoT8*qAnwJ|VxOb%FVKKb}!0Y;C>GZyO$LcI&~HY3O6 z8;kV?LAj3sDR7D>ODsVI0xU2XEha}S(G!C0=Y`RJGbYbiq9+sr4JsHd&@uVO68U;) zXB51jn1O)-*WP7_a}j%;!TdD1{~`O485kJSKrIp^ng_OV6s&;(zF8Q|Enr}ff^EeH zZ)#>>VCVxiZ$UX|!eoJ^$}yElRzQ24{h&e;)b4@O3<+Sv5Cqht;C;z;nEIg>Ff>f= zU#b!V+LR2_0JQ*SZWBbDSqap&Kp4+}k|a8jEC97tkkmCz=3i#7)MN@3ngpd`6fEQ! E08`vB9{>OV diff --git a/pc-bios/s390-netboot.img b/pc-bios/s390-netboot.img index 2c6886efb8320ff711589119ffbf077e68ec8784..aa90fbccb142470924ad5f0de5a4c6337916c63a 100644 GIT binary patch delta 23541 zcmZ3mmU%%h%LH8}My84S_YL2gFn|G!769=W85kJYKnw{7qZt?&I2afhWCR%) zctCviC5I-xPkzrVUjHA=gUc>~IE207{{xUJB=HUqpS|J#3lI$#hq{ct z;r|B|@g;}Awz4<;hq)Z;A&6Vq8^IoBUGV z3~Dg|o5bG8qyZ9yJE+A_VDm#3dG5&%7&#{ia4F|4UHu z7lBCj>;nc24E2+B1(X>ZCI<>=GAX)Ft`U$@$ea(7W=#QWI0S#nIrSMu5Vn zn&p}d3?NbUbrMa>_5S}~-ojuFR-XkDO|PG9ASB0;*8%3_Ob!!Laf;&I$;6+YB(Bh0EeSOk;EdQRm)Wvy+O`cuBpHfAOO)CmpWNSM1?83Y_gMx z7gK`DEB-i;1fV<)nZjG^RB`;G+!4sFV+r{lw)Y!XiUa?%0fzeT^0c=B5;p98w5=^o`CVvxG;>d;=wt2FGghYMzez3{q z%fI}63UX5RYcPNI@-Khyg86^I{K?Bb85ov-`Bx1V+X)uyT>jodMv>B1FFOD5>w4)C^~tHR25TG z_+$ZTCz0fd%Ox2UcCs)mTK@Zg0?6M@I+N3+4J9XsZn(g({0M{ia!CdTkTuB#U|HeG z8>Eexl6@yXkyc`AygiviMwZcMvYL!j{p@|1g5krCEahTw3SjuKBVfXZJ(K=_*d-ygT$5qT@)`gCJJu_GVAy1=z_3Y)VM50Q z2ASm=|Nl#AhAMz9JS{YHc?p9V$dbPwB&0T(ZUCuU(6N9)MrhUY5{9DX8Vn4eO!3&^ zV*~@s>;8`#3@q$lb}})zEHC-LH`L$)$MPfpcY>WQ4>IN76EIs9%)SR=*QYN9B_o-R zn?0!XYU8(k9pa7ln+xhKPd<&q5799=#zI0}AXIIw|%Az6R9B?HGL zfe!~aFgy-$U|^W=T9tu?f#-3+1qOx&XQdAZofHHZoLN5{bYc)-T<*=lxLlfnX}S6T z7a&WM-h&-5Z@DHz{Uu1z@&Et7=@Oa{e}H@h_lpRF)MJ4UDQlEfI=#Iw8GHM!39XXTT)ST)cfA~EbQmzmOl$bCa+4954>tMnhmQdaEbRY4CL{7hYUlFb|9(tj`LIA@(@qwK zDLYvhmOgg4z`>xnT!q2ZVZnz*0TVtfS-z5Cf#a$V438IpW$KSGh=77U5ghD)w}9A5 znjr7~T?b;Pm4cFhbi`wW3k(b_652Z(7_63iGB7#@d{_Ze_0RdS!^a8+me=#aPWl4U zlXwy&^S2MgPGSbLJ3#C-f3R6b%fI~J&CdQh|APU8N=-Bg2Hp0v|UpOxVf9V7$D9fo-`w1JCjn<{~k}<%SGQ%UhVsA{0K`d;mGbbGZ})ql8ufga@_~?3+_i>q`EgUjF6(>&FHk zI~XQRZ1~u*yhK2IxgmqY@)CZH<%SFj%S-saE;nQlSYE<+Xt^N+$MO=s{^f=YEXzyy zVwM{+FfK2t=hJ+w@UauDy>oeqK;Cjikaphp%N0S|dG{<=1Zn5(T&@Vx&KtH|5u}}0 zdATCUsr*NmD}s#SpY+(^V;9(zuH_{HGEhfEKperZ3ULI_Rfy$0i$F!-REAF+%j;VN zRxFohV02de#IanF;s5fI|Gyj!K5^{iU=W_fCBQI=L4eUw;Se%Ga6y2f!I|}Aq7#Dv zgR{%WL?;IU#^u@!jP=NQKRE+ze92B01`RMP|FOWw9*{*J85mSR^(~S`D;YUJmVq2~ zNq~3@QS7rlQ2+G*|35Zx3(X%3eC!2V$brQ|0g5dY0(*-EX5a7s^^XNU_JJ)Fz+#~U zr4}YV2U+<04BW!wj|D#VgYA<5+gA_DOt83H$*4fFg?qpjf(jpS6#njiEbwsx1IuH9 zj|yN5QSH;Hr`W=YU<+g5J`8^>@Nov%LJcey8lc%%PgE*JPBckrU<*OX5#mGn#{wT0 zf`ilmi-i_w7NXf#k3CzUSZEEh@Yfrd4}U#>Ebws=!^Fn|A1$z0=z!lsH2dnI`4GiI z0kDNz;1;fbEbws&*oO{SEc77BLNxp8K@~&dQ?P|Ka0|;H3w&G(w$KBMg#naU2)1t* z*glt?EDZJFIQ(V*Sm5I_u!R9wER3Mo!hW!YJa7-PKNk479Asbp#|Vgp%Ox2&AVny& zG&2AfpU&Xo(*Rt2f{V=p24`nb@mT;aK9^TBFfA8{w44&NK$iTx1+(Pm)#V=l|L+uG zP9Dp5>JcCmcIIFo3Jm4Ip8#LEBzSF$gm7 zJQldX!O-wn;KKn1mF1QU2QGCor(;G%T?+bxSW|C7@U{`7?y*Q$Cv+aJRCl*TJGq;u)Ku*&~irw5Z%ArQ2<28 zfDD3KzXxnJfo2QX!66_&$I49twRbETKuH(jpn41!F>pBtJ1{r}f8u~UNa5qE<%%2( z%S+haE>~m$(Yux_GJ)v!<%*0TI%K&bL&NeCHbqGDjrAR*`Np~%$-xoW99)my&;q&m z`ya52d9b>e7zZ1CTm@2pxkg@PI*OxhKO5 zM7tQ&69<(Xj~zbjU{Hb8B@LM62B-DW!2Cn6KitNi<7}$0)F{CXo`4a=~gqk@zeC${r$)Etz^}`p`_59;M zvEk#Q*BGg`pU(lF@)+IV05nI~5rmm#h3Mb7m=E zaAFBy0FMfN`TquLyenK2F5?Zb8xL^^)O2Sorq^>UZ{d_8WPH>&B8(TnVm!z47LM!q zOs@xx2u1BBZ2aH5IE-I|&v>xut$0kY{{;!0zpHQ<-$>ASA3VmxOfSb_dRRT&c;s~H z@QDMK8zO%}lO>+q@Ru1qH$c)2hd{$|G9v}^(q97?;~V-{c@E*bFmxG z{*$2b;e?D|uJR`fhv`*d)9XQFKez%1RJDNWN3=YPoT>hp<1n6J&_}!hIldmVNI)^3 z1&8qjgFa$2$awsw|Gwy4kF~&ILj*mJWEqh~I9dLV#hxtL&Vhp-6!MDoNW*Th;!=Tu zA;JXe`Xabe)WX|w)p7_JA|@kpkJ|A(7iFUi1wrHXM}uJY?U4&!GKH2wtK zc-*ERftX&8Ue>ea5;T4a5ynr17>{9k7@FzzkP%O$0w+9_aLoQv!V$CDaN}|0sc>1S z!E%1(5X=4aTPd*%T<2(IWrfa zbk!k_XX(aiJk0b&T&CB9CLAC`0Y9YCjR!TM!x})1>km!Kzx>LAT8AiraKv=z79yhJ zDmW_YF&djJp!s7+nTX5yvxJTR$qY6g)ATshu!6RKk=y@aU!j48r^WNx8b?ec3LxZk z1d3^BRD|t;IvzX+v>aUjV^l4lIhI@;*0iP6c7$1S;dhCveb+zyW{l|Au)9cZ?TFk0&SdI9M~^`%UP^gG^Zd<=aPS6d<~02y0MniAUH1FaJ;`1{{~Gyg!I; z4a9f`f^|a>tPPAOjNa!%O-Gb}5MTV50;Tsu=~tktdHI*0q7eR12MGP$97-oa>0XGs z?>iv;ZzrMj7bramLjP`t&_BH)bp6jV2;=8M2>sgyl40JbK=|LG4*C8H!v7%(vFIn% zqMwf;;=lNyv?Y{=>i^Xb#5Ah`>L0D9r$&|DJ`=|4u+O{+kNn z|B-^wzaK-ye{X>DYa#UCCMfL>rDY)M{=9_n|7?TO;E#v@L|bv2}TA7fe(wO zDSTLnw&F>JLFusrXfe~y1_m#QdTmf`wmg!75!B3ldl$5*mw^M+@CzxB2h3gw zW~+hN!C>?2IW8>!@_!}Nhw$}{!I9vS;P?Ls$egH?!G{F_4j+~{1bhIkKAZ#|h6b%e z1^I6$!vc>56AT@9e_(jL0lcaUx~R)v3uN7^_aN*1Q^4$(Aa?KuuoEXgcDNA9z`*j_ ziot+^cV`2GcZ9+Oo!7>oc~5wVgg$oo2=e*-4%sz zK+Dk}i`hU+-GUh>_sE3sAl7;YUEMrI){l`X$Z+x#xou2=Hz&`LS7J)uzxjZ?3!}|H z5kaNL4j%;=Sh^G#zK{*HurokmgAU00Wbw%kiXu)) zUzS@kJUQUNz>v(oJd%-Nc_t)A|2BX|bbbomZ1259*NC5_x*ZChV zOrE6ZR1aB{2C^&Q=bO}%p z&{%HCU<{hO1{H40BN;$WU;yd(tMJ(1!vfHX6a!<13Cra`YtI;;FPCHBSl+^TWw{&! z4R&_1*&O^?WSwVKakVh`_=* z0Rju>?38B^3=sIRxgM@j5T+62gFgqrnw*!fW?)&qi-B?ZKG52;|0f|~uwPyPw5;|U zXhlJl@bZ-m3}7b}d|2<;@L|XDFAVJ-0v|Ry2z)s9*a75V3B$(%4jc?DOU)Pc{;sOSi2nPk7#{vo*3@ia&;HVM? zuP~2f0IzlVqYLsNL(t<17Z?~gmak@T0D1GVKt0I4ksAaUmM>vQ0a?FXLx91-;KQ*1 zg%1ZD3_fgpJOMPz=CI+z5r+*Q*26>32s#hL0t@!ve?h6{e>5nTz1=}!`kV*E_Ll;) z*}&|xpj^qaVfmN;?#n$Gn3hX0)H4J)7%&6~7ziwnU|?JR<=-`sLT`1L3qhKnftStt z^MKipK=S@8!3Om}d;^IjuqRypFMvDI(9z+;I#4c$J8=!jiICNxUf;l`?FO&T@p}p8 zYy}%#12#JTvBO6TaJGRLhhBHVsuqD%`Aq?{=YZJ$$zUCB%fI|v4&r-(7l%FrFAnwV z1FHis4)q5w4rMV~{^j3VusTqYTL1r96^P>(4pvwKV*6Wz4U}8{R@$T%fI~NfrP zJ0G;^0a6k|NJ4;B~1K}9Vg z`&SsjW!i7;W+bu&;K703Kt+ts*X+mr7mOOC%k;e|NkJ%J*t*} z`Tw4s{dN5Z3kDTvKsN7WVc2qD0t2J{YLFqnrc8Fz=wY&DoqR>ZnK6B`gr*9QZ_iE^ zhH8)r?$0NCXgV?3vrq2PlxI%`73Oj8CU4SIi?VyLQ=9?3Yr;3=5;tfc!vTT+|Noy3 zxWK>>Ai%%^GW!W=j*_uikQrRN`&zud|B-`%<-ivP1}6E}_hDNSd@Z3BZvv<;@ztAb zq?J+c%L*#n;{BHY{vWZEg&`T7U{pY*+)fsT%HM8*KtDWqB;1z{W69Qfzt2|0~Niz#WcjphSSE?0jZUj?s2v^4U6hfwl~j&*aJbwC$LD zrcM5*t;FP$JXuLcMbF25`IrAU(6q@yqcM3W3&SMHq9IVq^GTjuqNBzX8#;N8&IuLo z-=N5Iow;1&|9?<_6l?~(E$2NG6snWo=*qDj5MW@mWt_Z5SC7NH5X7;0Hu;fmjKm{v zkdGexJ1FX7QznP$y18q4bF$%~`Vib^_#3mpIQf2);LP3EcKtVtO zq)J2rl=VSdkU(;_ypvfhoD^(sF1KU^&F{D@k7RHLDHD7LGR)>6NRiEf$!Qj9Og3vK z_gPpmS+`9-WnmRy{R-rK-*cdid)9FW8WG+T#(j57kY7(iQi z7_C7o88jbn5MThU<66MLpmJaX1EaO&O@hmoR{eQbwy^VC~RN zc?>QfS6P1m*Y7P1r2zsTmIMe~kN}Imj4%Kdq8C6503AS^rL1EnD_Qx7SnmXtuAnCE z@=k^&AU9d9ncQThB4sTJGTU&*@-P2EJFYZA)h|e&Rqx~jR$@%njgxO!8Sz=?9=O23 zXq^RiX6j@KYb7Qt-^n)CQcTv#lOwFN_S6HhsS?f)nXRRb>EwtQ{(POzMgCNLo z@yno%ec<>qU|_UnnS9S$g~=^+GK-C!jO9s4UbxDvz_Ng0c?qM0V}ZgOP{@TPO^&ni zW3rt*d7X_OhgB2EdselRpV;V$S*1f-ED;V40s#)-Jx+{P(UUc7)tIbYCkNPS30dkd z_k@(|1|cAgmg-6#0ju6KiSa8FC~Pg=CkNPj*jO5%CpbP=P|R3L!9owzTz#AX>J)(t5Cv@q z)c_m7Ir)mciHOA;$A$)-0D}dffej5%+*sV7EaPCtVR01XR_nc!qa2i&%oj|qbI1}h zuYd)K-dvEZdBNmg4h?$d>z8{nfP+L298%`XK|!+o%l}xAA`7Jmg-<&`Ne7g~Eo3LJ zaFh|T;DRL355Ja|Foc0zY~DEehNA|PdCp`erxZDJOGksx=Rhvj-8PAN18A4rZ%2bq zpbC%CZ1v<8Cj$esNsa~%Ixq+6ZUxz8)-j1ez+w59|C1qw38X>-wfUWGCqHp|z~oUl z`IvJVpY_d_mdG`p#r8mnlrv?UNt4sK{8?fh=X)yHkY07qoj2R2&6Lfdb6B zaI%7H1e3MXDz@&bBb{!NeWyCp)-_NjaK=rLSN}Yfdh3 z6Vr0a2TSh(NrQAQZ()o9+2;gqrs;0j$;qG&;#*yuyunR|$?DMLb8b$47LP!cKS-x` zCnyLlZh=}#PTm3xAhm9LKyhPv0W2Q~Vq2U7v*RBtfV-nA4Dq1trxyD`a+Z50$GO)q zIV_)i$6ZOp61>~lEfAz$3*0$%m^N9+LoLGc(SCgahUF~`TbHk6U|Zh8a1ms&jVoxtT^{_DL4H>E%c$bRQ!9ZT6&u(B23?S9+YOV~`IIo(Sx#2* z4Pvsjo?PMUz+|g6d6#bplg+Ei0)BE#mX9YJ_=$*F-Ur#H1!~@DF85&I1G&KR+~g=f zGX;uv z0tYw&XoLJ?Sv6TGG)%Rs9`W~ z0*z=uTDOXz?NnMIy9`!Nik4zBubnIt{ej7xXEI+*43p{A$vH7TOs1D6?~2*OWLiAA zBGyRG%n=mira_MlK;idzgJA%J2s9KKP1Pr#h?Qe_SY?p%R!{xvKC4e^D?qp%u z4BGj;+>?RXQE36FU|=+PHhEH<8$@ns<;kSmtA zFc>d)VNeTIxWKaf2!j$xr|}n%=S}W`(lR)(LP3GWpat@y@w3MQ0+Safcsm*I1F3S6 z0=XZoN*fjO_M(*#wgtU2^!V_slE9PWcSVYpiB$uZi3o8yI)PtOR{5f3YfeuNr%aJ z=j1y{tF&$=N?1oI2rw*{`2SyANW@T57<}Y_00ZOA2yld%77sc|)?8&Q06L z4WR7`U?J1T6F^%wA{ZJNA{-_#NU|(%VdmZ`#;^b+yIXDYr{sA|Hy%x%l;Xr>^nUWS zlnqQqlP7nisxTQfOkS6o#B@VqvQS!%k&)C+&;owwDF)jffI|E4*2e}PHZXv;vAKi% zv+WKj#N@a@zS?$n@`kh^wd*S*1OynwrA366m;3{_pRZ2=N$;Hac!2@~gV=!v1_r~! zlQq(-q^|oTTz%ag;%bo6>zVnBohS~7_aVu+4y@i$i}N%(BxOa4pc{LgWDD0 zb3X(a7%z5SWe;Egm*LCkRL| zh=Dpe;L`2_1LJwW$vqWvVwzIRfBz2v_iI2qvp|c-8P97?K2RaTbe?PSwF(8MbMGhr zsL+zoxFuD#yo!MjluL!Ocd{@vflNR5VzO1`5+@Dtke-kLsQA+WEx=;j$-&?R8uS4v z7Xgo@X@E~~5&Y+P=>j8ID|kFc@Egd-=Sn9lR4H(nwShTllRc^wn9i9^&Z;sH&;Yl* z1UE^6MgG9RKyB`!uuR}b^L449%?pF%$$8U`B#;Jg8DV78OvuuPV*2< z2bJm|E6+ZWng!By_Wfk9YBjyH7a{W$pz!?uXZe?Z`yrVf(YDqs1yv!S2?MhWljl_{ z@SR=!*r0)dK?GF(o$Z}`rrL|?Z1QA@8XcyysgvDmLPXALf`%wS*>ZUX10yJo&ZxV!LWe>1 z%>VPC#H7{*3QPexQ0j41YG8z&6eDa1vgXY6$&YGH>@>t*3o@IDpHU9sFiaY>|Ff7;j{~IcD9VBx4`*Mx{f1o1gz#{LKYyAHS6#;ku zPCs3)@&6ZCMDxqPJ)rK=8Rq3P7)(KN^U@!z>BLSJ2Bfrp`u^mwdNrog8z$G)E9snG zvs~)`7m&WwYa<05K$8hlU|qA8YyAHQHu%dw&>1I;r)N$+Q=i3jI%l#)gPg$W;Qjuf zZDIewBS)uQCZ{!Mahx^)YgL{+qd|-BG$$koKyh|jY4U{zF|pGM%V$9L7k+gDB^!|H zQ-3G_Y0#@b^%N9%r|ty^7=SvuLZGoSknNyK0(lxrZ7C?Vp4#_VVFLq$N`!(UxI=sD z07xw;o&ppS7y=X$K-FV0*!Z>JT&@B(eSLs}0XP;j*^Z$QPiUmdF zFEvo=Hco$BU;rLW0`0W{^{`IqZEkAfVPsO?I(b5K8k2(6%YRVLu?35jPhQfdz@*eV z`51_5p8Ntt1y1H_S71`ooUGHX#-x03a!C6uB_*Hb5(caRpa4-|U_1^QiDFPX0kTC% z59Imd=92|F#4?nfU%P;FmI|n}b^j#502O?4sJ{AaIU{G29<^QVX5(WZLxyfKT!^wReQcOyElUH>} zSt;qjRW>Y_Fc5~SoB&p-4VNogE@2=JmFoe^sZIU|GDUN;RHvNCu|*)aD1t{h1do8c zbZq8ipH3-_V<#OG6dWE02!Kj71qQ}r&C4wr0>Fx=gZwWDZdDwsnB3PXRevn&v4a5% zgBU0Zl?1@jp zl9&`VCJXc`@hM7y+{g)@FH)44?9eMFqzLw^=z`^6z{NI$BE#g2UL_`lPm_Ckb(j?P zOy1RN$E2`(@~_?qCdIx$u~~6o2Vcp4?f;W_ylO41C)&A)g~8AG-Hx4-MnJrCq^c@ zmy_pAE?|;;J(+Kc1e4^w%^FjBn3yztCU2Xj#i5}GvQXB4@{4II5(iqq_KJWC9z@8< zmQGfgzK=*OcXwK&wLgEYvToh&v(g-Pb^WSbfLnAA-t|Cyo1q%J>MW9AvA{Q{f6 z&Fo-gQhh$TVfGIW)g+JxsdbYZ=JYU0^-SiQ>%%0KxjANT7$cM9!O2_ZEn|`_oE$U% z8j~dFWRnH&m?WldR#_;=#3bH1*=2D7lX%GFZHwiY#62coTWrQ8ZZMf|i3yXq{MhBT7HDF8I(}A@7}y&#Un&L_-^AFvdu}G zZJ3$(eotPo?FOz&i35M18EAmXS5E@K4OZ3y7|jaAI5qvMFj@Wyt2fc%oK&>;^O2C zg|x)XoYWMB^30M9h2&&g28I%>qE-q96=`UIi^0Iq$k@cx%-q5Jl%p}06R1vF5ippXn^ zGJwRAGvf2|L9FDGq8#1Qyv!1XlFEWqkQYi)i;7AMN)&=piwpAeic=Mwa`Kb26(&B>Dyp+@wsAGal5=%;p5l(T;NrgJyGp{7IC^t1FGqD70RzOi^ zZemd-NMCetYI1&F3WQe=_HRjPF~~6xwp{T3iS#K3MCnt#SpG*a(0S>OJZ?~LU1Z5xq_Uapplka zqF`jAn^aknTC8cs#eiFtu@OQQDBK{XIOpf3WP$=y!7V2-y%?OxopV!kgHp3n>yyDL z9PE!WNCfK!r52WE7C{mjC=PY~OG|VE5{oiRD&hW#b}cN;EXYmGD}f0yC`5yt3*#z8 z`jl3%3jTMW){e)+mVsmb|8DY|a?rO?FZmYI{P8&Cv}UyxrE zqQi?aOHy?`^D;7%GD|>03<}Zpp1B1@smYnesi5Qt&A&mZg)n8hU`=3$IYSauVp2{j zTH0{SEGjP11$#l)CqEsm+BL5XqAH*$wKz2o63WPSfmA6d)Pu^V%#_l^90lk6(!3H- ztSLZ~NkB$rv4TrcW?8C&rwdpDCX@rqW~?KL2^(%8L5dWU||I{g^WzFBB%_c zVAs!0EiO(>PX%cKnFdOK3Te3|x+$sPk|nPs+CbMl79<5yf+}O7YiJq^HLjp2KV2aa zoOKkE^HV?uLS>xu^U^ZYON$c0NiH)lEgxoDE?A+yLPn-8$n_u#>Y!S@| z7GcO3;*>E0$+$u4r;>byqS8D_KJs)?aCR=|f}VrRz`)P&|NnQ;8ITMNc3?r!8O99% z|3l8={r~^}f3PS61B_;d^7%nDvbxDrc0UpW2|_T6l*4Y1$z^*EZ?@j+%`&;;;JnQ$ zhhDK5ZI}i!|Ns9DyCC$sFbKV21(d!Aq2c0ZCOaHAQ`)%>BD`}6l%55pCqU@k(>BjI z&c;&zAPK_1FG3*=btZ}C)c+@>;gJl)VGp7H#-;wD9Da2Vo8KR+g}4y_16>uPEVe) delta 11243 zcmZ3`%d%iC^8{TchD#Im?;GyTVE_XdEdb&%GB7Z(ffx)7=N5oy_69gT*`85d5YB_i zPA+Ghjzv_S$x`I!ik0D4zrMQY_QTA~SJn81#JwY{C+9HDlALq_WHbW<0|x^GgNz^p z0}qJLzGTv5d1f)GhKoNyTm}XP1qKENQ)Fp>W^u_0P)Vj11_lO3K?VkHkN|tb#mV)| z;gaW1fOrfrBO5N>0P)!y&QE^NEM9*e%!A7=fjES{;rs)TDkSj^5TCu_{0k5b7l*ox zz2W=^6!9gKz_zkCoQJs_>LG|**&8l^Jj%cj40Bdf16V(M!vzU2118=G;juSdf@pw= zw=f(4$*?zE(g4wL2emK?Y<|ch&pr78Bj+RmF6FR;%YXlFIPjE#f#u2p1qOz2K@gj@ z4n%T&KVZPXkUm*gK$$UPa-e`F6EE-N8UZPV(0Y(Gs{=?w*i$gu3e0u@vH1>6-X@^H zm@)affHo7)$;m8&DzYBC4k$1%x_5)*xz-3NEq7tg3sCq}abN)hqx)o6K~tfynIO5~ zzMU)#S|FCX<>Vegzc6o8u)$y#21kP=gcCrd>RzyT$8t*sE)YN1cew@wi=)9ujR1vD zHOn;_7(k-JX%bD#_5S}~-ojuFR^J8^4K|-_ASA~Twgk*knH(mh;^caCCliCx@)E{{ zLLJLn80^8SXM+^*URW-{z!o6zsphf5rwRs<<(3e$6;nXAfNTV*QZ1dlM@YgdOlqeH zgC$5H=>GC$|0jWs0cim{GUzIpuMC#|z5L7nqdQp`G(kMqn#p{^a*Sz{b%o`bcvvR; z32W55WgRGBV023d$-B>8F2TUyBvHTsHo!_~qQNz)212`Ocv?LY@ty-?a=nZnla!mz>00D?rFWJd5A}UPodXt?* zyqLW2OzsoeA?%tCN;sh#KxTV)9&lh_be$Y2YAECyyF8LXd)VX$7V$!OrH^a1Q0#d44c zSk)x3D*4F?Vj>!fDKO!P(_mqqy~>m2<&8GFhX{P>tm3v5E+n)fV0cL{GGXz zg+U#bpq>6q786$!3TXgEs7q^rz(||nSTHeCIGKryq0hVW+9VfSl zYcV;kpS(d_M!?x%xh4anqf!IJ*_xB@h)XbWZkYT{T!|yt0&HvcWCaO{`rzAOj{fp5 zf1iS##17^MFaPrQE|@O?=6f&qWMEkSOEchx|%yIdbzo$X`z#b5v(HF!HdJbm0 zgW0|yHrH8pc6Rmy9~gw3H-JP_P988|5ONjU$<1K2bMkfx7e>v=eEqzC&LOymk$h^j2%Fx7<3pg$OxGlFE3%>h)@6}@&Et-_edD+6k$+) z?C>#yf#vo3j}{Cp>|Y=uhzNVXyPIR&{1}=1EGMsV-^S$oaB_-=5>tZm<_-@RMy9_9 zCZF?^uHXM50utW7iU&3@FvMqpf;2#3gAOP};;WZyGBQkJ5MWp?&A_-^{r?|Oc*W-} zw`6#7z=44wK6`m2Bg28G|NsBp2P(%HxUMp8FucG4*6X^wg@Jdu1_J{qzdd&ND8Rt- zdjE$XAeTbIdAZjA|C4jPOzJ@nf1Urq0OAmzv&(<~x0uB8Vckv^hD{L)3OXPQ{!arJ zqPIciTY$humjHnc8p}Hwbe3B(7&~@sU;w54oh%F*%OgRh|NsAg;~ooaFlLyrT#kWh zc?;wH<#G%h%Uc-FFPCFrT;9Sc4f2T33Q*c-t^c?@0_@6AumP3~N{$X24M3LuwE_8x zQFbQ_!|LS~3=EDFK5Sm@!NA~D@L_v^!-vgK^+G#Y7(ACpGO&X6D1r6FKNk3~nL#c@ zVBwqqfrWE+$}{!0^~XK!8CdKtaIbvB8I93=>})Gq5o5IvP7nw)U~8_h4Xf za4=watRNu4z~bPb!1P!_K!Jf}xdj8GgMk7_Tmd2u@_&GX07rm>01K$teXJm0z@QT0 zut5i6X@Hl21~goRcd{_JgX%F*xcsRFd6Jk_26iD0$+%{o(u>8w^PKr{GI*y5L+Em(RPaX;WbnV|AOpOM!G@WD+4W$y9*FH@0!m6OTb6(McNxU@ zjtBV)EH4Ms7QF756Ny|8F8?x8(_8tn0*P%z5#YjeZ=xF z|0aS3n8A`qz-%Khdmos61gy|y`ImpwKzy(FU^#Gg;q@BKUIh~O2Dd6$43>ZSC%RmN zfqA(F15<#)28IAg2&*jr^3Qa+2LmHGBRgzh2oL~;yaa>d@-P21K_vmCg7De{YNF(P zoy;Py!Ibl0vYxzC2(%3T{XYRzGJvgoxmQa6r<+->DN7F3w-x z!rHq(KJA;!o zq`+GL;RC3^0(s2K8f4>7a1rQr49xxp7XJe>iMeL^m;d0X_fiMTy#UKi0ka>0*)PEA zGM0b&zZ&9CNM*PD_v9D;;*9x|0;K&bcd{_(?qp#ARRUb_i06WeT0+`|kP`RT|K(p8 zSeh9aTtKoQYktoHC6NC=A`~uYfcU@MCT9i67E-|_!H$Zn6Uz1|G|Pm z1ys=QWMOFD$-=PZzyt&IYCB9pfT^f1|co$M0m%xFEiBT$9M^Zrg2hH8)r?$;;p z2y|kywx9ebP@X*$RL)1MO;!q0i?Wv3Db4_Hjd(7<#0_d691!^b|NrTL3k)0q0+3kw zF=4qR17ouwGY2R3I&?TN z$Sn6`U|!zBz_;8F+~MGw$e6&oyo5n$`7Q>=<;(s*02Mor6)rF^G(c;s{0|qvk^l6+ z^Z(zh;J|$Y3Zoy(m+LVwE-(52RH%Hp3j@bvfdn>&iIU>WOa9+kuED^#{LBA)&=`i- zY94nI$IeAT_7E^@j)hy`PSf&jzvFaK9A?_^j4@{rk= z$wJ90Qs#*uvkhk~|MDNyQ_%!?FlISOy`5r@^VeK zE3p2l;hss1Uzxyi{gWT0d)Sy(peHy!S5VBDCc#1v)GmFTZ~>HkpE59rUIW$eU;_dt zr(~Fjn3^~?H0T5vECBTtH9&D=sy=y5h8c$`FUYMHtdn15C@~qon9P@%C1iX679@Ie zL9)g>CU<2v=ox>91&JOwq>SH#f@JxZ|FIxNCRq^*pLT$f4k(G6q)xWTk`XZpf+Wxn zzm}IUgn?Xad~$L|mIjmYmdTT{Qsj&q9SuH%14wt<3IPS0{(%l<`?;NnPA2 z#^4LeZWbz&Z{2d-Zhyk z&xOf6eR4paM22|~NV#7A@-P2CfviLnpmws87#u(qM+B(S*vZ1svXg~jDJWGsZv+*o zAfK3mhb*)ofGsj$V6X=d+vvUlXVMl1KA5=e;>j2C#H4J&BVoE%Fr<4Y<%?sSZ@)m}RAdAgJK^B9vjQ|6q z8K|;hG-ZVJcR?LNuwiwe1Zu_r8vZiS*BPDY)i3)n$aPU4ZnZyWMbH~yygG( ztBf0%F0d>w`44KcGa7*cfYHQY^0ZKuC#>7YVPDMr9n)V&67FG z9GEPNCVQ2IFj*Q+UQi~-WTrj&K$(b`nL5ZeEl}%CbGZitAIJq}Vv}E$nJHL&gw|fm zz)Ji<4q|8ow?qvX7%d)6b||02WU*lKz49myi&T(Ci=@d86`PnWI3|Couw$}#xY?xA zf>FXe;<17N0|N_a=mHT~7LJ=+syY}s%)#ZNS?^|%8a_rQv!9byYIB54&s}A|zyVGG z+93a!9-6$UHcZ5H+VU^|Z?Ut#?*HHb>5rRDoXk_F#bjDM*`m&>-sSIO0gz1x6#oDJ zJsoVT%2J#EOrQ+v@*5_{z~J(BO1%KXV+92U2A0PL3ZQyXfPul~5vZ7jt70@&gUBtn zWMF)3Aix0`VR?XeiplulmDY< z-IF=mjO2`#g2LQz)?)*3!(oGA0D}lL7#R)oCr7l&F*!b;T+!C0VC1^|_digF2jr{G zpb^96o(#;6N((?G1EZndWTkdDCPTK(S?wZ>A`Xv11^n_a|5J7{F&Kdg1B0r`lRA{@ z4KmpOLAp<%Tx6IDiUv@t5Yqf&eFs(=2?{|)bx?N+l++9YmwU3l165q0?DhDukP>*v zt%bpOxeJ3@sKN!7Qfyu_5-cAN=AXN@Y zAoqh+$%B&n|1%&H^nZYC)c*uGK@cprZ@GlAv4g;eWneYWCLicjVbZ@f`B|qClm3Cp zQe8$&`pYK!br~}rn%vYCqj1d;G|&N3d(94H_ce1+t_?6yU;wpwb{I|m*Ja0KH)FD0 zw+@qm?Bty8Ra#e9N?1pL+EWt$|BDNW7)lC*hw23w7_Tmf+yKfROzEKB(bd_L9eTuc zuC_jI_;7(i1uRtic*2JR3@Q-}4Ga+u6Br~}mbWl-?-XNL0FvFFH@T^29@AB=$x6LW zO!~o-(|R{B>3yFp(WkKgasm(!SS_K9iiQ3=46?PN*tG^ z!HUH;yG`_CWZIH4dDSvRkq;H0^_9!P|`c_<^TUb%pm2L0>D{U3{)h6 z#!+<&mdYtGNC|}s1h6buVUU+F>M&ps2r&o%HKQ4)f-dnPnJB=(c!>pOSzrGP{RT|Unao7c=qB|_5jfEYzufWo$>6^$zNvMFr8gC*=WuZrnBLb7tE8K{Bce= z(^#z<7GvWSNC>Vybn^fBz2v_iI!@&H`&GnjElD zgz0q9gPTVBP$2Z~Uk?42wOO<<)4lTR&N;-vBbEGz&j{#0%) z_hew)$-&?R8Ma|y5CM-;s(@SBg8v*ZU0?)hJ#`qQNAMfS$EWsA-mpl4!{{8Cvu^T@ zMG8!(>L&kMWFVl@1Xi|5Y7tmd#bm3+CNif$MXVqwyiXN^LgV%6o>k+L*zBfd^#5_dmP=7;0-823mYA%!M1k+*>&FHS3=AR%CNMCbygE5%i5Jt!)st5&(P28dcJj3)AtEOW zK|>UvY`Hvxfe{o(Cvzs-EtO(AnKL*^Q}3OIn~$fQ8Jjz3zi@&6y#;4l9QLE(P<;pCVVSxm>b zOg^zfPT=_L{r&PW$QvN;Y8Cu9JIK>eV0D1qI%5wEzJFP)An?G+G9-9aKrMvm;WLGAAgt9%p;3 zuz`U=B|^fN`3EL%SS8`5lmyBL$5sXeOn?{;o^BH5 z2BlVoHK5cgt+-tC|9?iOgf5VdvkED+T@dCz(^CI!{WPu9;;lH0Ic z!hkgZ6d(!=jE6fwZj{#nxn4*Q&(COut3;`g;vcJKK z!L5oz2PVsHlBz$n>9Kczm#~N&Z&vK3b{V+ZMo-LO!5QFQfd+cDq!65cHz<>v;GxWey23-b0 zIk49J(TOmfpFdu&!?l1rXkuvvylE_HI>W)&v+ z%*|Ui>o79OKAHSzO9Yeb?#VV=lbB?CCNJ2k#3u`0b;MZ(NA4z$E>2^0}P~Lek(7Kj9Og zc?^&%q`M~VGGmgyx!GdZCq^b|-^n_A3Yf(FC(qj>!6fFidB>g}CMK0Blil`faj2Am zqDFG_4JBWahyg7Nx{(Vf!e~QfBBhzNy$tsucDQsE@ z4iA;(`rxUNn8ykY3=Arck_rrn{95a zU}EA~GWpppFD72A$ws&LG4b5m{OPt0GZT0GWP|%>n7FP?{&#;56W83ydmb!e67HXz z@KB3MxN>sOLk*1$1t8A|Cqilv&*D1D811JtClDaYO9nI}atK)-XI40L^~v zWMU9p-ohZ_s01!z1Xur<%=bu+Y4wZAdXGYwHgIlkdE~=5`NwMy@Pa+ifxDLhlfR&@k~GER#9D zn<*W51`$4R2TEUp(kCGFp>vxPzO%8^Kc5BRKl7oGhB}i(bL#&S((p0_;;uzlHI?kJ#h^3rY*8Oy^)_)Z;t{ zb_D~&x#{+-jOLu@H$V(NKfRun(TuABwqo|e^!;EyY{BZK>Hk?7?U)!YO}A%*(DiJL FiU3IT%kcmJ From 9138977b185266e750c9422e554b0c4e7860783f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 17 Apr 2019 13:31:42 +0200 Subject: [PATCH 18/19] s390x/kvm: Configure page size after memory has actually been initialized Right now we configure the pagesize quite early, when initializing KVM. This is long before system memory is actually allocated via memory_region_allocate_system_memory(), and therefore memory backends marked as mapped. Instead, let's configure the maximum page size after initializing memory in s390_memory_init(). cap_hpage_1m is still properly configured before creating any CPUs, and therefore before configuring the CPU model and eventually enabling CMMA. This is not a fix but rather a preparation for the future, when initial memory might reside on memory backends (not the case for s390x right now) We will replace qemu_getrampagesize() soon by a function that will always return the maximum page size (not the minimum page size, which only works by pure luck so far, as there are no memory backends). Acked-by: Igor Mammedov Reviewed-by: David Gibson Signed-off-by: David Hildenbrand Message-Id: <20190417113143.5551-2-david@redhat.com> Signed-off-by: Cornelia Huck --- hw/s390x/s390-virtio-ccw.c | 12 ++++++++++++ target/s390x/cpu.c | 7 +++++++ target/s390x/cpu.h | 1 + target/s390x/kvm-stub.c | 4 ++++ target/s390x/kvm.c | 35 ++++++++++++++--------------------- target/s390x/kvm_s390x.h | 1 + 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index d11069b860..3be5679657 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -15,6 +15,7 @@ #include "cpu.h" #include "hw/boards.h" #include "exec/address-spaces.h" +#include "exec/ram_addr.h" #include "hw/s390x/s390-virtio-hcall.h" #include "hw/s390x/sclp.h" #include "hw/s390x/s390_flic.h" @@ -163,6 +164,7 @@ static void s390_memory_init(ram_addr_t mem_size) MemoryRegion *sysmem = get_system_memory(); ram_addr_t chunk, offset = 0; unsigned int number = 0; + Error *local_err = NULL; gchar *name; /* allocate RAM for core */ @@ -182,6 +184,15 @@ static void s390_memory_init(ram_addr_t mem_size) } g_free(name); + /* + * Configure the maximum page size. As no memory devices were created + * yet, this is the page size of initial memory only. + */ + s390_set_max_pagesize(qemu_getrampagesize(), &local_err); + if (local_err) { + error_report_err(local_err); + exit(EXIT_FAILURE); + } /* Initialize storage key device */ s390_skeys_init(); /* Initialize storage attributes device */ @@ -253,6 +264,7 @@ static void ccw_init(MachineState *machine) DeviceState *dev; s390_sclp_init(); + /* init memory + setup max page size. Required for the CPU model */ s390_memory_init(machine->ram_size); /* init CPUs (incl. CPU model) early so s390_has_feature() works */ diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index 698dd9cb82..b58ef0a8ef 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -399,6 +399,13 @@ int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit) return 0; } +void s390_set_max_pagesize(uint64_t pagesize, Error **errp) +{ + if (kvm_enabled()) { + kvm_s390_set_max_pagesize(pagesize, errp); + } +} + void s390_cmma_reset(void) { if (kvm_enabled()) { diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h index d8990c405a..7305cacc7b 100644 --- a/target/s390x/cpu.h +++ b/target/s390x/cpu.h @@ -734,6 +734,7 @@ static inline void s390_do_cpu_load_normal(CPUState *cs, run_on_cpu_data arg) /* cpu.c */ void s390_crypto_reset(void); int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit); +void s390_set_max_pagesize(uint64_t pagesize, Error **errp); void s390_cmma_reset(void); void s390_enable_css_support(S390CPU *cpu); int s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch_id, diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c index bf7795e47a..22b4514ca6 100644 --- a/target/s390x/kvm-stub.c +++ b/target/s390x/kvm-stub.c @@ -93,6 +93,10 @@ int kvm_s390_set_mem_limit(uint64_t new_limit, uint64_t *hw_limit) return 0; } +void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp) +{ +} + void kvm_s390_crypto_reset(void) { } diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c index 2c6e35b5aa..7df7be4a1b 100644 --- a/target/s390x/kvm.c +++ b/target/s390x/kvm.c @@ -283,44 +283,37 @@ void kvm_s390_crypto_reset(void) } } -static int kvm_s390_configure_mempath_backing(KVMState *s) +void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp) { - size_t path_psize = qemu_getrampagesize(); - - if (path_psize == 4 * KiB) { - return 0; + if (pagesize == 4 * KiB) { + return; } if (!hpage_1m_allowed()) { - error_report("This QEMU machine does not support huge page " - "mappings"); - return -EINVAL; + error_setg(errp, "This QEMU machine does not support huge page " + "mappings"); + return; } - if (path_psize != 1 * MiB) { - error_report("Memory backing with 2G pages was specified, " - "but KVM does not support this memory backing"); - return -EINVAL; + if (pagesize != 1 * MiB) { + error_setg(errp, "Memory backing with 2G pages was specified, " + "but KVM does not support this memory backing"); + return; } - if (kvm_vm_enable_cap(s, KVM_CAP_S390_HPAGE_1M, 0)) { - error_report("Memory backing with 1M pages was specified, " - "but KVM does not support this memory backing"); - return -EINVAL; + if (kvm_vm_enable_cap(kvm_state, KVM_CAP_S390_HPAGE_1M, 0)) { + error_setg(errp, "Memory backing with 1M pages was specified, " + "but KVM does not support this memory backing"); + return; } cap_hpage_1m = 1; - return 0; } int kvm_arch_init(MachineState *ms, KVMState *s) { MachineClass *mc = MACHINE_GET_CLASS(ms); - if (kvm_s390_configure_mempath_backing(s)) { - return -EINVAL; - } - mc->default_cpu_type = S390_CPU_TYPE_NAME("host"); cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS); cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF); diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h index 6e52287da3..caf985955b 100644 --- a/target/s390x/kvm_s390x.h +++ b/target/s390x/kvm_s390x.h @@ -36,6 +36,7 @@ int kvm_s390_cmma_active(void); void kvm_s390_cmma_reset(void); void kvm_s390_reset_vcpu(S390CPU *cpu); int kvm_s390_set_mem_limit(uint64_t new_limit, uint64_t *hw_limit); +void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp); void kvm_s390_crypto_reset(void); void kvm_s390_restart_interrupt(S390CPU *cpu); void kvm_s390_stop_interrupt(S390CPU *cpu); From 905b7ee4d647111d29f29c55618591366321c47a Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 17 Apr 2019 13:31:43 +0200 Subject: [PATCH 19/19] exec: Introduce qemu_maxrampagesize() and rename qemu_getrampagesize() Rename qemu_getrampagesize() to qemu_minrampagesize(). While at it, properly rename find_max_supported_pagesize() to find_min_backend_pagesize(). s390x is actually interested into the maximum ram pagesize, so introduce and use qemu_maxrampagesize(). Add a TODO, indicating that looking at any mapped memory backends is not 100% correct in some cases. Signed-off-by: David Hildenbrand Message-Id: <20190417113143.5551-3-david@redhat.com> Reviewed-by: David Gibson Reviewed-by: Igor Mammedov Signed-off-by: Cornelia Huck --- exec.c | 44 ++++++++++++++++++++++++++++++++++---- hw/ppc/spapr_caps.c | 4 ++-- hw/s390x/s390-virtio-ccw.c | 2 +- hw/vfio/spapr.c | 2 +- include/exec/ram_addr.h | 3 ++- target/ppc/kvm.c | 2 +- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/exec.c b/exec.c index 2646207661..bb1dff423a 100644 --- a/exec.c +++ b/exec.c @@ -1688,7 +1688,7 @@ void ram_block_dump(Monitor *mon) * when we actually open and map them. Iterate over the file * descriptors instead, and use qemu_fd_getpagesize(). */ -static int find_max_supported_pagesize(Object *obj, void *opaque) +static int find_min_backend_pagesize(Object *obj, void *opaque) { long *hpsize_min = opaque; @@ -1704,7 +1704,27 @@ static int find_max_supported_pagesize(Object *obj, void *opaque) return 0; } -long qemu_getrampagesize(void) +static int find_max_backend_pagesize(Object *obj, void *opaque) +{ + long *hpsize_max = opaque; + + if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) { + HostMemoryBackend *backend = MEMORY_BACKEND(obj); + long hpsize = host_memory_backend_pagesize(backend); + + if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) { + *hpsize_max = hpsize; + } + } + + return 0; +} + +/* + * TODO: We assume right now that all mapped host memory backends are + * used as RAM, however some might be used for different purposes. + */ +long qemu_minrampagesize(void) { long hpsize = LONG_MAX; long mainrampagesize; @@ -1724,7 +1744,7 @@ long qemu_getrampagesize(void) */ memdev_root = object_resolve_path("/objects", NULL); if (memdev_root) { - object_child_foreach(memdev_root, find_max_supported_pagesize, &hpsize); + object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize); } if (hpsize == LONG_MAX) { /* No additional memory regions found ==> Report main RAM page size */ @@ -1747,8 +1767,24 @@ long qemu_getrampagesize(void) return hpsize; } + +long qemu_maxrampagesize(void) +{ + long pagesize = qemu_mempath_getpagesize(mem_path); + Object *memdev_root = object_resolve_path("/objects", NULL); + + if (memdev_root) { + object_child_foreach(memdev_root, find_max_backend_pagesize, + &pagesize); + } + return pagesize; +} #else -long qemu_getrampagesize(void) +long qemu_minrampagesize(void) +{ + return getpagesize(); +} +long qemu_maxrampagesize(void) { return getpagesize(); } diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c index edc5ed0e0c..9b1c10baa6 100644 --- a/hw/ppc/spapr_caps.c +++ b/hw/ppc/spapr_caps.c @@ -347,7 +347,7 @@ static void cap_hpt_maxpagesize_apply(SpaprMachineState *spapr, warn_report("Many guests require at least 64kiB hpt-max-page-size"); } - spapr_check_pagesize(spapr, qemu_getrampagesize(), errp); + spapr_check_pagesize(spapr, qemu_minrampagesize(), errp); } static bool spapr_pagesize_cb(void *opaque, uint32_t seg_pshift, @@ -609,7 +609,7 @@ static SpaprCapabilities default_caps_with_cpu(SpaprMachineState *spapr, uint8_t mps; if (kvmppc_hpt_needs_host_contiguous_pages()) { - mps = ctz64(qemu_getrampagesize()); + mps = ctz64(qemu_minrampagesize()); } else { mps = 34; /* allow everything up to 16GiB, i.e. everything */ } diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 3be5679657..7e256d3d31 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -188,7 +188,7 @@ static void s390_memory_init(ram_addr_t mem_size) * Configure the maximum page size. As no memory devices were created * yet, this is the page size of initial memory only. */ - s390_set_max_pagesize(qemu_getrampagesize(), &local_err); + s390_set_max_pagesize(qemu_maxrampagesize(), &local_err); if (local_err) { error_report_err(local_err); exit(EXIT_FAILURE); diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c index 57fe758e54..96c0ad9d9b 100644 --- a/hw/vfio/spapr.c +++ b/hw/vfio/spapr.c @@ -148,7 +148,7 @@ int vfio_spapr_create_window(VFIOContainer *container, uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr); unsigned entries, bits_total, bits_per_level, max_levels; struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) }; - long rampagesize = qemu_getrampagesize(); + long rampagesize = qemu_minrampagesize(); /* * The host might not support the guest supported IOMMU page size, diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 9ecd911c3e..139ad79390 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -73,7 +73,8 @@ static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, bool ramblock_is_pmem(RAMBlock *rb); -long qemu_getrampagesize(void); +long qemu_minrampagesize(void); +long qemu_maxrampagesize(void); /** * qemu_ram_alloc_from_file, diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index 2427c8ee13..59d92c4275 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -2136,7 +2136,7 @@ uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift) /* Find the largest hardware supported page size that's less than * or equal to the (logical) backing page size of guest RAM */ kvm_get_smmu_info(&info, &error_fatal); - rampagesize = qemu_getrampagesize(); + rampagesize = qemu_minrampagesize(); best_page_shift = 0; for (i = 0; i < KVM_PPC_PAGE_SIZES_MAX_SZ; i++) {