s390x/kvm: Guest Migration TOD clock synchronization

Synchronizes the guest TOD clock across a migration by sending the guest TOD
clock value to the destination system. If the guest TOD clock is not preserved
across a migration then the guest's view of time will snap backwards if the
destination host clock is behind the source host clock. This will cause the
guest to hang immediately upon resuming on the destination system.

Reviewed-by: David Hildenbrand <dahi@linux.vnet.ibm.com>
Signed-off-by: Jason J. Herne <jjherne@linux.vnet.ibm.com>
Signed-off-by: Jens Freimann <jfrei@linux.vnet.ibm.com>

Message-Id: <1425912968-54387-1-git-send-email-jfrei@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>

4 files changed, 129 insertions, 0 deletions

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index dac00ce..eea0742 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -181,6 +181,10 @@ static void ccw_init(MachineState *machine)
 
     /* Create VirtIO network adapters */
     s390_create_virtio_net(BUS(css_bus), "virtio-net-ccw");
+
+    /* Register savevm handler for guest TOD clock */
+    register_savevm(NULL, "todclock", 0, 1,
+                    gtod_save, gtod_load, kvm_state);
 }
 
 static void ccw_machine_class_init(ObjectClass *oc, void *data)
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index 412e49b..bdb5388 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -38,6 +38,7 @@
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/s390-virtio.h"
+#include "cpu.h"
 
 //#define DEBUG_S390
 
@@ -53,6 +54,9 @@
 #define ZIPL_FILENAME                   "s390-zipl.rom"
 #define TYPE_S390_MACHINE               "s390-machine"
 
+#define S390_TOD_CLOCK_VALUE_MISSING    0x00
+#define S390_TOD_CLOCK_VALUE_PRESENT    0x01
+
 static VirtIOS390Bus *s390_bus;
 static S390CPU **ipi_states;
 
@@ -196,6 +200,51 @@ void s390_create_virtio_net(BusState *bus, const char *name)
     }
 }
 
+void gtod_save(QEMUFile *f, void *opaque)
+{
+    uint64_t tod_low;
+    uint8_t tod_high;
+    int r;
+
+    r = s390_get_clock(&tod_high, &tod_low);
+    if (r) {
+        fprintf(stderr, "WARNING: Unable to get guest clock for migration. "
+                        "Error code %d. Guest clock will not be migrated "
+                        "which could cause the guest to hang.\n", r);
+        qemu_put_byte(f, S390_TOD_CLOCK_VALUE_MISSING);
+        return;
+    }
+
+    qemu_put_byte(f, S390_TOD_CLOCK_VALUE_PRESENT);
+    qemu_put_byte(f, tod_high);
+    qemu_put_be64(f, tod_low);
+}
+
+int gtod_load(QEMUFile *f, void *opaque, int version_id)
+{
+    uint64_t tod_low;
+    uint8_t tod_high;
+    int r;
+
+    if (qemu_get_byte(f) == S390_TOD_CLOCK_VALUE_MISSING) {
+        fprintf(stderr, "WARNING: Guest clock was not migrated. This could "
+                        "cause the guest to hang.\n");
+        return 0;
+    }
+
+    tod_high = qemu_get_byte(f);
+    tod_low = qemu_get_be64(f);
+
+    r = s390_set_clock(&tod_high, &tod_low);
+    if (r) {
+        fprintf(stderr, "WARNING: Unable to set guest clock value. "
+                        "s390_get_clock returned error %d. This could cause "
+                        "the guest to hang.\n", r);
+    }
+
+    return 0;
+}
+
 /* PC hardware initialisation */
 static void s390_init(MachineState *machine)
 {
@@ -253,6 +302,9 @@ static void s390_init(MachineState *machine)
 
     /* Create VirtIO network adapters */
     s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
+
+    /* Register savevm handler for guest TOD clock */
+    register_savevm(NULL, "todclock", 0, 1, gtod_save, gtod_load, NULL);
 }
 
 void s390_nmi(NMIState *n, int cpu_index, Error **errp)
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 0171de0..8135dda 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -401,6 +401,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
 int kvm_s390_inject_flic(struct kvm_s390_irq *irq);
 void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
+int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
+int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_clock);
 #else
 static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
 {
@@ -408,11 +410,40 @@ static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
 static inline void kvm_s390_service_interrupt(uint32_t parm)
 {
 }
+static inline int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    return -ENOSYS;
+}
+static inline int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    return -ENOSYS;
+}
 static inline void kvm_s390_access_exception(S390CPU *cpu, uint16_t code,
                                              uint64_t te_code)
 {
 }
 #endif
+
+static inline int s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_get_clock(tod_high, tod_low);
+    }
+    /* Fixme TCG */
+    *tod_high = 0;
+    *tod_low = 0;
+    return 0;
+}
+
+static inline int s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_set_clock(tod_high, tod_low);
+    }
+    /* Fixme TCG */
+    return 0;
+}
+
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr);
 unsigned int s390_cpu_halt(S390CPU *cpu);
 void s390_cpu_unhalt(S390CPU *cpu);
@@ -422,6 +453,9 @@ static inline uint8_t s390_cpu_get_state(S390CPU *cpu)
     return cpu->env.cpu_state;
 }
 
+void gtod_save(QEMUFile *f, void *opaque);
+int gtod_load(QEMUFile *f, void *opaque, int version_id);
+
 /* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm);
 
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 896534b..b21a348 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -486,6 +486,45 @@ int kvm_arch_get_registers(CPUState *cs)
     return 0;
 }
 
+int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    int r;
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_TOD,
+        .attr = KVM_S390_VM_TOD_LOW,
+        .addr = (uint64_t)tod_low,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+    if (r) {
+        return r;
+    }
+
+    attr.attr = KVM_S390_VM_TOD_HIGH;
+    attr.addr = (uint64_t)tod_high;
+    return kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+}
+
+int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    int r;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_TOD,
+        .attr = KVM_S390_VM_TOD_LOW,
+        .addr = (uint64_t)tod_low,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+    if (r) {
+        return r;
+    }
+
+    attr.attr = KVM_S390_VM_TOD_HIGH;
+    attr.addr = (uint64_t)tod_high;
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be