[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH 8/8] HVM save restore: PV driver support



[PATCH 8/8] HVM save restore: PV driver support

Signed-off-by: Zhai Edwin <edwin.zhai@xxxxxxxxx>

enable PV driver's save/restore in HVM domain by:
HV:
* send a pseudo PCI dev intr to guest in restore
* rebuild the shared info on behalf of HVM guest
* set a resume flag in shared info

Guest:
* check the resmume flag in the pseudo PCI dev intr handler
* if set do the PV driver resume work


diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c
--- a/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c      Thu Jan 11 
17:03:17 2007 +0800
+++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c      Thu Jan 11 
17:03:21 2007 +0800
@@ -138,6 +138,10 @@ static int blkfront_resume(struct xenbus
 
        DPRINTK("blkfront_resume: %s\n", dev->nodename);
 
+       /* resuming qemu disk would cause error when restore and skip next vbd 
resume */
+       if (info->connected == BLKIF_STATE_DISCONNECTED)
+               return 0;
+ 
        blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
 
        err = talk_to_backend(dev, info);
diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c
--- a/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c   Thu Jan 11 17:03:17 
2007 +0800
+++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c   Thu Jan 11 17:03:21 
2007 +0800
@@ -183,7 +183,16 @@ xlbd_put_major_info(struct xlbd_major_in
 xlbd_put_major_info(struct xlbd_major_info *mi)
 {
        mi->usage--;
-       /* XXX: release major if 0 */
+       /* release major if 0 */
+       if (mi->usage)
+               return;
+
+       printk("Unregistering block device major %i\n", mi->major);
+       if (unregister_blkdev(mi->major, mi->type->devname)) {
+               WPRINTK("can't put major %d with name %s\n",
+                       mi->major, mi->type->devname);
+               kfree(mi);
+       }
 }
 
 static int
diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
--- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c    Thu Jan 11 
17:03:17 2007 +0800
+++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c    Thu Jan 11 
17:03:21 2007 +0800
@@ -815,6 +815,7 @@ static int xsd_port_read(char *page, cha
 }
 #endif
 
+extern void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void));
 static int __init xenbus_probe_init(void)
 {
        int err = 0;
@@ -882,6 +883,7 @@ static int __init xenbus_probe_init(void
                xen_store_mfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
                xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
                                              PAGE_SIZE);
+               set_restore_handler(xenbus_suspend, xenbus_resume);
 #endif
        }
 
diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/evtchn.c
--- a/unmodified_drivers/linux-2.6/platform-pci/evtchn.c        Thu Jan 11 
17:03:17 2007 +0800
+++ b/unmodified_drivers/linux-2.6/platform-pci/evtchn.c        Thu Jan 11 
20:12:33 2007 +0800
@@ -165,6 +165,7 @@ void notify_remote_via_irq(int irq)
 }
 EXPORT_SYMBOL(notify_remote_via_irq);
 
+extern void pvdrv_restore(void);
 irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 {
        unsigned int l1i, port;
@@ -173,6 +174,16 @@ irqreturn_t evtchn_interrupt(int irq, vo
        shared_info_t *s = shared_info_area;
        vcpu_info_t *v = &s->vcpu_info[cpu];
        unsigned long l1, l2;
+
+       /* add a check to see if need resume after restore */ 
+       if (s->pvdrv_resume == 0x58585858) {
+               printk("evtchn_interrupt:resume PV driver.\n");
+               s->pvdrv_resume= 0;
+               v->evtchn_upcall_pending = 0;
+
+               pvdrv_restore();
+               return IRQ_HANDLED;
+       }
 
        v->evtchn_upcall_pending = 0;
        /* NB. No need for a barrier here -- XCHG is a barrier on x86. */
diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/platform-pci.c
--- a/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c  Thu Jan 11 
17:03:17 2007 +0800
+++ b/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c  Thu Jan 11 
17:03:21 2007 +0800
@@ -28,6 +28,7 @@
 #include <linux/interrupt.h>
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
+#include <linux/kthread.h>
 #include <asm/system.h>
 #include <asm/io.h>
 #include <asm/irq.h>
@@ -280,9 +281,85 @@ static struct pci_driver platform_driver
 
 static int pci_device_registered;
 
+extern int gnttab_suspend(void);
+void platform_pci_suspend(void)
+{
+       gnttab_suspend();
+}
+
+extern int gnttab_resume(void);
+void platform_pci_resume(void)
+{
+       phys_to_machine_mapping = NULL;
+
+       gnttab_resume();
+}
+
+/* support for PV driver save/restore in HVM domain*/
+void (*suspend_handler)(void);
+void (*resume_handler)(void);
+void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void))
+{
+       suspend_handler = hdl1;
+       resume_handler  = hdl2;
+}
+EXPORT_SYMBOL(set_restore_handler);
+
+/* scheduled suspend&resume */
+static void restore_handler(void *unused);
+static int __do_pvdrv_restore(void *ignored);
+
+static DECLARE_WORK(restore_work, restore_handler, NULL);
+
+static int kthread_create_on_cpu(int (*f)(void *arg),
+       void *arg,
+       const char *name,
+       int cpu)
+{
+       struct task_struct *p;
+       p = kthread_create(f, arg, name);
+       if (IS_ERR(p))
+               return PTR_ERR(p);
+       kthread_bind(p, cpu);
+       wake_up_process(p);
+       return 0;
+}
+
+static int __do_pvdrv_restore(void *ignored)
+{
+       if (suspend_handler)
+               suspend_handler();
+       platform_pci_suspend();
+
+       platform_pci_resume();
+       if (resume_handler)
+               resume_handler();
+
+       return 0;
+}
+
+static void restore_handler(void *unused)
+{
+       int err;
+
+       err = kthread_create_on_cpu(__do_pvdrv_restore, NULL, "pvdrv_suspend", 
0);
+       if (err < 0) {
+               printk(KERN_WARNING "error creating PV driver suspend process 
(%d): retrying...\n",
+                       err);
+               schedule_delayed_work(&restore_work, 50/2);
+       }
+}
+
+void pvdrv_restore(void)
+{
+       schedule_work(&restore_work);
+}
+
 static int __init platform_pci_module_init(void)
 {
        int rc;
+
+       set_restore_handler(NULL, NULL);
 
        rc = pci_module_init(&platform_driver);
        if (rc)
diff -r d18c6a3c676a xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c    Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/arch/x86/hvm/hvm.c    Thu Jan 11 17:03:21 2007 +0800
@@ -200,6 +200,9 @@ int hvm_vcpu_initialise(struct vcpu *v)
  
     /* init hvm sharepage */
     shpage_init(v->domain, get_sp(v->domain));
+
+    /* other hvm info need for save/restore */
+    hvminfo_init(v->domain);
 
     /* Init guest TSC to start from zero. */
     hvm_set_guest_time(v, 0);
diff -r d18c6a3c676a xen/arch/x86/hvm/intercept.c
--- a/xen/arch/x86/hvm/intercept.c      Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/arch/x86/hvm/intercept.c      Thu Jan 11 17:03:37 2007 +0800
@@ -445,6 +445,71 @@ void shpage_init(struct domain *d, share
     hvm_register_savevm(d, "xen_hvm_shpage", 0x10, 1, shpage_save, 
shpage_load, sp);
 }
 
+void hvminfo_print(struct domain* d)
+{
+    printk("********hvm other info***********\n");
+    printk("callback irq=%"PRId64".\n", 
d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
+    printk("shared info pfn=0x%"PRIx64".\n", 
d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
+}
+
+static void hvminfo_save(hvm_domain_context_t *h, void *opaque)
+{
+    struct domain *d = opaque;
+
+#ifdef HVM_DEBUG_SUSPEND
+    hvminfo_print(d);
+#endif
+
+    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
+    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
+}
+
+extern int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn);
+static int hvminfo_load(hvm_domain_context_t *h, void *opaque, int version_id)
+{
+    struct domain *d = opaque;
+    int callback_irq = hvm_get_64u(h);
+    unsigned long share_info_pfn = hvm_get_64u(h);
+
+    if (callback_irq == 0) {
+        return 0;
+    }
+
+    d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ] = callback_irq;
+
+    /* reconstruct the share info for HVM guest */
+    rebuild_shared_info(d, share_info_pfn);
+    d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = share_info_pfn;
+
+    /* set resume state to notify HVM PV driver */
+    shared_info(d, pvdrv_resume) = 0x58585858;
+
+    /* set all the evtchn_upcall_pending on all vcpus */
+    if (callback_irq) {
+        struct vcpu *v;
+
+        for_each_vcpu(d, v) {
+            if ( !test_and_set_bit(0, &vcpu_info(v, evtchn_upcall_pending)) )
+                printk("on vcpu %d raise a vir level intr %d to restore PV 
driver in HVM guest!\n",
+                        v->vcpu_id,
+                        callback_irq);
+        }
+    }
+
+
+#ifdef HVM_DEBUG_SUSPEND
+    hvminfo_print(d);
+#endif
+
+    return 0;
+
+}
+
+void hvminfo_init(struct domain* d)
+{
+    hvm_register_savevm(d, "xen_hvm_other_info", 0x11, 1, hvminfo_save, 
hvminfo_load, d);
+}
+
 int hvm_buffered_io_intercept(ioreq_t *p)
 {
     struct vcpu *v = current;
diff -r d18c6a3c676a xen/arch/x86/mm.c
--- a/xen/arch/x86/mm.c Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/arch/x86/mm.c Thu Jan 11 17:03:21 2007 +0800
@@ -3000,6 +3000,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
         case XENMAPSPACE_shared_info:
             if ( xatp.idx == 0 )
                 mfn = virt_to_mfn(d->shared_info);
+            d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = xatp.gpfn;
             break;
         case XENMAPSPACE_grant_table:
             if ( xatp.idx < NR_GRANT_FRAMES )
@@ -3143,6 +3144,43 @@ long arch_memory_op(int op, XEN_GUEST_HA
     return 0;
 }
 
+/* reconstruct the share info for HVM guest */
+int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn)
+{
+    unsigned long mfn, prev_mfn, gpfn = 0;
+
+    mfn = virt_to_mfn(d->shared_info);
+    if ( !shadow_mode_translate(d) || (mfn == 0)) {
+        printk("reconstruct share info for HVM guest failed!\n");
+        return -1;
+    }
+
+    LOCK_BIGLOCK(d);
+
+    /* Remove previously mapped page if it was present. */
+    prev_mfn = gmfn_to_mfn(d, share_info_pfn);
+    if ( mfn_valid(prev_mfn) )
+    {
+        if ( IS_XEN_HEAP_FRAME(mfn_to_page(prev_mfn)) )
+            /* Xen heap frames are simply unhooked from this phys slot. */
+            guest_physmap_remove_page(d, share_info_pfn, prev_mfn);
+        else
+            /* Normal domain memory is freed, to avoid leaking memory. */
+            guest_remove_page(d, share_info_pfn);
+    }
+
+    /* Unmap from old location, if any. */
+    gpfn = get_gpfn_from_mfn(mfn);
+    if ( gpfn != INVALID_M2P_ENTRY )
+        guest_physmap_remove_page(d, gpfn, mfn);
+
+    /* Map at new location. */
+    guest_physmap_add_page(d, share_info_pfn, mfn);
+
+    UNLOCK_BIGLOCK(d);
+
+    return 0;
+}
 
 /*************************
  * Writable Pagetables
diff -r d18c6a3c676a xen/include/asm-x86/hvm/support.h
--- a/xen/include/asm-x86/hvm/support.h Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/include/asm-x86/hvm/support.h Thu Jan 11 17:03:21 2007 +0800
@@ -248,6 +248,8 @@ extern int arch_gethvm_ctxt(struct vcpu 
 
 extern void shpage_init(struct domain *d, shared_iopage_t *sp);
 
+extern void hvminfo_init(struct domain* d);
+
 extern int hvm_enabled;
 
 int hvm_copy_to_guest_phys(paddr_t paddr, void *buf, int size);
diff -r d18c6a3c676a xen/include/public/hvm/params.h
--- a/xen/include/public/hvm/params.h   Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/include/public/hvm/params.h   Thu Jan 11 17:03:21 2007 +0800
@@ -31,6 +31,7 @@
 #define HVM_PARAM_PAE_ENABLED  4
 #define HVM_PARAM_IOREQ_PFN    5
 #define HVM_PARAM_BUFIOREQ_PFN 6
-#define HVM_NR_PARAMS          7
+#define HVM_PARAM_SHINFO_PFN   7
+#define HVM_NR_PARAMS          8
 
 #endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
diff -r d18c6a3c676a xen/include/public/xen.h
--- a/xen/include/public/xen.h  Thu Jan 11 17:03:17 2007 +0800
+++ b/xen/include/public/xen.h  Thu Jan 11 17:03:37 2007 +0800
@@ -465,6 +465,9 @@ struct shared_info {
     uint32_t wc_sec;          /* Secs  00:00:00 UTC, Jan 1, 1970.  */
     uint32_t wc_nsec;         /* Nsecs 00:00:00 UTC, Jan 1, 1970.  */
 
+    /* flag for resume PV driver in HVM guest */
+    uint32_t pvdrv_resume;
+
     struct arch_shared_info arch;
 
 };

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.