|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [Xen-devel] [PATCH][RESUBMIT] add dom0 vcpu hotplug control
* Christian Limpach <christian.limpach@xxxxxxxxx> [2005-06-21 09:11]:
> On 6/14/05, Ryan Harper <ryanh@xxxxxxxxxx> wrote:
> > * Ryan Harper <ryanh@xxxxxxxxxx> [2005-06-07 17:05]:
> > > This patch adds new control messages for vcpu hotplug events. Via the
> > > xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled
> > > when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
> > >
> >
> > Updated this patch to use vcpu_to_cpu up/down info to control whether
> > state changes are sent when hotplugging vcpus.
>
> I've applied both patches. I think there's a flaw in the way how
> vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of
> gross and it won't work for cpus > 9?
I updated the vcpu_to_cpu string creation to include a field separator,
which gets rid of the -1 -> # hack and works for cpus > 9.
I ran into some issues with stale vcpu_to_cpu lists when running the
hotplug subprogram. I would take a vcpu offline, and then issue the
command to bring it back and the vcpu_to_cpu list would not have changed
to indicate the the vcpu actually went down. If I injected a xm list -v
(which always showed the correct mapping) then subsequent hotplug
commands would see the state change and fire off the hotplug request. I
don't know that not sending the event when not changing state saves that
much work so I took the state check out and now just send the hotplug
event directly.
> Also the whole hotplug stuff is still missing interrupt re-routing
> when a vcpu is taken down. To do this, we need an evtchn operation to
> change the vcpu affinity of a port by changing notify_vcpu_id.
I don't fully understand all of the mappings that are happening, so this
part of the patch might be way off. In any case, I've added a new
evtchn op to set the notify_vcpu_id field of a channel. I updated the
HOTPLUG_CPU code to use the new routines when bringing cpus up and down.
When taking down a cpu, I route the IPI irq channels to CPU 0, and when
the cpu comes up, it re-routes the channels back to the awakened CPU.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@xxxxxxxxxx
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 19 +++++++---
linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c | 32 +++++++++++++++++
tools/python/xen/xend/XendDomainInfo.py | 6 ++-
tools/python/xen/xm/main.py | 18 +--------
xen/common/event_channel.c | 29 +++++++++++++++
xen/include/public/event_channel.h | 10 ++++-
6 files changed, 91 insertions(+), 23 deletions(-)
Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
---
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
--- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-21
22:10:55.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-22
14:31:48.765503988 -0500
@@ -103,6 +103,11 @@
DEFINE_PER_CPU(int, cpu_state) = { 0 };
#endif
+static DEFINE_PER_CPU(int, resched_irq);
+static DEFINE_PER_CPU(int, callfunc_irq);
+static char resched_name[NR_CPUS][15];
+static char callfunc_name[NR_CPUS][15];
+
#if 0
/*
* Currently trivial. Write the real->protected mode
@@ -1328,6 +1333,10 @@
while (!cpu_online(cpu))
cpu_relax();
+ /* re-route bound IRQs 0 to cpu */
+ rebind_evtchn_from_irq(0, cpu, per_cpu(resched_irq, cpu));
+ rebind_evtchn_from_irq(0, cpu, per_cpu(callfunc_irq, cpu));
+
fixup_irqs(cpu_online_map);
/* counter the disable in fixup_irqs() */
local_irq_enable();
@@ -1357,6 +1366,11 @@
cpu_clear(cpu, map);
fixup_irqs(map);
+
+ /* re-route IRQs from dead vcpu to another */
+ rebind_evtchn_from_irq(cpu, 0, per_cpu(resched_irq, cpu));
+ rebind_evtchn_from_irq(cpu, 0, per_cpu(callfunc_irq, cpu));
+
/* It's now safe to remove this processor from the online map */
cpu_clear(cpu, cpu_online_map);
@@ -1514,11 +1528,6 @@
extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *);
extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *);
-static DEFINE_PER_CPU(int, resched_irq);
-static DEFINE_PER_CPU(int, callfunc_irq);
-static char resched_name[NR_CPUS][15];
-static char callfunc_name[NR_CPUS][15];
-
void __init smp_intr_init(void)
{
int cpu = smp_processor_id();
diff -urN b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
--- b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-21
22:10:53.000000000 -0500
+++ c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-22
14:31:48.767503708 -0500
@@ -271,6 +271,38 @@
return irq;
}
+void rebind_evtchn_from_ipi(int cpu, int newcpu, int ipi)
+{
+ evtchn_op_t op;
+ int evtchn = per_cpu(ipi_to_evtchn, cpu)[ipi];
+
+ spin_lock(&irq_mapping_update_lock);
+
+ op.cmd = EVTCHNOP_rebind;
+ op.u.rebind.port = evtchn;
+ op.u.rebind.vcpu = newcpu;
+ if ( HYPERVISOR_event_channel_op(&op) != 0 )
+ printk(KERN_INFO "Failed to rebind IPI%d to CPU%d\n",ipi,newcpu);
+
+ spin_unlock(&irq_mapping_update_lock);
+}
+
+void rebind_evtchn_from_irq(int cpu, int newcpu, int irq)
+{
+ evtchn_op_t op;
+ int evtchn = irq_to_evtchn[irq];
+
+ spin_lock(&irq_mapping_update_lock);
+
+ op.cmd = EVTCHNOP_rebind;
+ op.u.rebind.port = evtchn;
+ op.u.rebind.vcpu = newcpu;
+ if ( HYPERVISOR_event_channel_op(&op) != 0 )
+ printk(KERN_INFO "Failed to rebind IRQ%d to CPU%d\n",irq,newcpu);
+
+ spin_unlock(&irq_mapping_update_lock);
+}
+
void unbind_ipi_on_cpu_from_irq(int cpu, int ipi)
{
evtchn_op_t op;
diff -urN b/tools/python/xen/xend/XendDomainInfo.py
c/tools/python/xen/xend/XendDomainInfo.py
--- b/tools/python/xen/xend/XendDomainInfo.py 2005-06-21 22:10:54.000000000
-0500
+++ c/tools/python/xen/xend/XendDomainInfo.py 2005-06-22 14:31:48.000000000
-0500
@@ -423,8 +423,10 @@
sxpr.append(['cpu_time', self.info['cpu_time']/1e9])
sxpr.append(['vcpus', self.info['vcpus']])
sxpr.append(['cpumap', self.info['cpumap']])
- sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x),
- self.info['vcpu_to_cpu'][0:self.info['vcpus']]))])
+ # build a string, using '|' to seperate items, show only up
+ # to number of vcpus in domain, and trim the trailing '|'
+ sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x)+'|',
+ self.info['vcpu_to_cpu'][0:self.info['vcpus']]))[:-1]])
if self.start_time:
up_time = time.time() - self.start_time
diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py
--- b/tools/python/xen/xm/main.py 2005-06-21 22:10:53.000000000 -0500
+++ c/tools/python/xen/xm/main.py 2005-06-22 14:54:50.810258798 -0500
@@ -410,8 +410,7 @@
print 'Name Id VCPU CPU CPUMAP'
for dom in doms:
info = server.xend_domain(dom)
- # XXX this is quite broken for cpu's > 9
- vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu',
'?').replace('-1','#')
+ vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').split('|')
cpumap = sxp.child_value(info, 'cpumap', [])
mask = ((int(sxp.child_value(info, 'vcpus', '0')))**2) - 1
count = 0
@@ -420,10 +419,7 @@
d['name'] = sxp.child_value(info, 'name', '??')
d['dom'] = int(sxp.child_value(info, 'id', '-1'))
d['vcpu'] = int(count)
- if cpu == "#":
- d['cpu'] = int("-1")
- else:
- d['cpu'] = int(cpu)
+ d['cpu'] = int(cpu)
d['cpumap'] = int(cpumap[count])&mask
count = count + 1
print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d
0x%(cpumap)x" % d)
@@ -593,15 +589,7 @@
state = int(args[3])
dom = server.xend_domain(name)
id = sxp.child_value(dom, 'id')
- vcpu_to_cpu = sxp.child_value(dom, 'vcpu_to_cpu', '-1')
- # only send state change if states differ
- try:
- # (down going up) or (up going down)
- if (vcpu_to_cpu[vcpu] == "-1" and state == 1) or \
- (vcpu_to_cpu[vcpu] != "-1" and state == 0):
- server.xend_domain_vcpu_hotplug(id, vcpu, state)
- except IndexError:
- print "Invalid VCPU(%d)"%(vcpu)
+ server.xend_domain_vcpu_hotplug(id, vcpu, state)
xm.prog(ProgVcpuhotplug)
diff -urN b/xen/common/event_channel.c c/xen/common/event_channel.c
--- b/xen/common/event_channel.c 2005-06-21 22:10:55.000000000 -0500
+++ c/xen/common/event_channel.c 2005-06-22 14:55:20.936237917 -0500
@@ -579,6 +579,29 @@
return rc;
}
+static long evtchn_rebind(evtchn_rebind_t *bind)
+{
+ struct domain *d = current->domain;
+ int port = bind->port;
+ int vcpu = bind->vcpu;
+ struct evtchn *chn;
+ long rc = 0;
+
+ spin_lock(&d->evtchn_lock);
+
+ if ( !port_is_valid(d, port) )
+ {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ chn = evtchn_from_port(d, port);
+ chn->notify_vcpu_id = vcpu;
+
+ out:
+ spin_unlock(&d->evtchn_lock);
+ return rc;
+}
long do_event_channel_op(evtchn_op_t *uop)
{
@@ -637,6 +660,12 @@
rc = -EFAULT;
break;
+ case EVTCHNOP_rebind:
+ rc = evtchn_rebind(&op.u.rebind);
+ if ( (rc == 0) && (copy_to_user(uop, &op, sizeof(op)) != 0) )
+ rc = -EFAULT;
+ break;
+
default:
rc = -ENOSYS;
break;
diff -urN b/xen/include/public/event_channel.h
c/xen/include/public/event_channel.h
--- b/xen/include/public/event_channel.h 2005-06-21 22:10:55.000000000
-0500
+++ c/xen/include/public/event_channel.h 2005-06-22 14:31:48.801498950
-0500
@@ -162,6 +162,13 @@
} PACKED u;
} PACKED evtchn_status_t; /* 20 bytes */
+#define EVTCHNOP_rebind 8
+typedef struct {
+ /* IN parameters. */
+ u32 port; /* 0 */
+ u32 vcpu; /* 4 */
+} PACKED evtchn_rebind_t; /* 8 bytes */
+
typedef struct {
u32 cmd; /* EVTCHNOP_* */ /* 0 */
u32 __reserved; /* 4 */
@@ -174,7 +181,8 @@
evtchn_close_t close;
evtchn_send_t send;
evtchn_status_t status;
- u8 __dummy[24];
+ evtchn_rebind_t rebind;
+ u8 __dummy[16];
} PACKED u;
} PACKED evtchn_op_t; /* 32 bytes */
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |