From 1a6e4a8c276e122dbeb6f9c610f29735e4236bfd Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:19 +0300 Subject: KVM: Move irq sharing information to irqchip level This removes assumptions that max GSIs is smaller than number of pins. Sharing is tracked on pin level not GSI level. [avi: no PIC on ia64] Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 59 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 23 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 001663ff401..9783f5c43da 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -31,20 +31,39 @@ #include "ioapic.h" +static inline int kvm_irq_line_state(unsigned long *irq_state, + int irq_source_id, int level) +{ + /* Logical OR for level trig interrupt */ + if (level) + set_bit(irq_source_id, irq_state); + else + clear_bit(irq_source_id, irq_state); + + return !!(*irq_state); +} + static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, - struct kvm *kvm, int level) + struct kvm *kvm, int irq_source_id, int level) { #ifdef CONFIG_X86 - return kvm_pic_set_irq(pic_irqchip(kvm), e->irqchip.pin, level); + struct kvm_pic *pic = pic_irqchip(kvm); + level = kvm_irq_line_state(&pic->irq_states[e->irqchip.pin], + irq_source_id, level); + return kvm_pic_set_irq(pic, e->irqchip.pin, level); #else return -1; #endif } static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, - struct kvm *kvm, int level) + struct kvm *kvm, int irq_source_id, int level) { - return kvm_ioapic_set_irq(kvm->arch.vioapic, e->irqchip.pin, level); + struct kvm_ioapic *ioapic = kvm->arch.vioapic; + level = kvm_irq_line_state(&ioapic->irq_states[e->irqchip.pin], + irq_source_id, level); + + return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, level); } inline static bool kvm_is_dm_lowest_prio(struct kvm_lapic_irq *irq) @@ -96,10 +115,13 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, } static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, - struct kvm *kvm, int level) + struct kvm *kvm, int irq_source_id, int level) { struct kvm_lapic_irq irq; + if (!level) + return -1; + trace_kvm_msi_set_irq(e->msi.address_lo, e->msi.data); irq.dest_id = (e->msi.address_lo & @@ -125,34 +147,19 @@ static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level) { struct kvm_kernel_irq_routing_entry *e; - unsigned long *irq_state, sig_level; int ret = -1; trace_kvm_set_irq(irq, level, irq_source_id); WARN_ON(!mutex_is_locked(&kvm->irq_lock)); - if (irq < KVM_IOAPIC_NUM_PINS) { - irq_state = (unsigned long *)&kvm->arch.irq_states[irq]; - - /* Logical OR for level trig interrupt */ - if (level) - set_bit(irq_source_id, irq_state); - else - clear_bit(irq_source_id, irq_state); - sig_level = !!(*irq_state); - } else if (!level) - return ret; - else /* Deal with MSI/MSI-X */ - sig_level = 1; - /* Not possible to detect if the guest uses the PIC or the * IOAPIC. So set the bit in both. The guest will ignore * writes to the unused one. */ list_for_each_entry(e, &kvm->irq_routing, link) if (e->gsi == irq) { - int r = e->set(e, kvm, sig_level); + int r = e->set(e, kvm, irq_source_id, level); if (r < 0) continue; @@ -232,8 +239,14 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); return; } - for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++) - clear_bit(irq_source_id, &kvm->arch.irq_states[i]); + for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++) { + clear_bit(irq_source_id, &kvm->arch.vioapic->irq_states[i]); + if (i >= 16) + continue; +#ifdef CONFIG_X86 + clear_bit(irq_source_id, &pic_irqchip(kvm)->irq_states[i]); +#endif + } clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); mutex_unlock(&kvm->irq_lock); } -- cgit v1.2.3 From 46e624b95c36d729bdf24010fff11d16f6fe94fa Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:20 +0300 Subject: KVM: Change irq routing table to use gsi indexed array Use gsi indexed array instead of scanning all entries on each interrupt injection. Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 88 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 35 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 9783f5c43da..81950f6f6fd 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -144,10 +144,12 @@ static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, * = 0 Interrupt was coalesced (previous irq is still pending) * > 0 Number of CPUs interrupt was delivered to */ -int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level) +int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) { struct kvm_kernel_irq_routing_entry *e; int ret = -1; + struct kvm_irq_routing_table *irq_rt; + struct hlist_node *n; trace_kvm_set_irq(irq, level, irq_source_id); @@ -157,8 +159,9 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level) * IOAPIC. So set the bit in both. The guest will ignore * writes to the unused one. */ - list_for_each_entry(e, &kvm->irq_routing, link) - if (e->gsi == irq) { + irq_rt = kvm->irq_routing; + if (irq < irq_rt->nr_rt_entries) + hlist_for_each_entry(e, n, &irq_rt->map[irq], link) { int r = e->set(e, kvm, irq_source_id, level); if (r < 0) continue; @@ -170,20 +173,23 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level) void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) { - struct kvm_kernel_irq_routing_entry *e; struct kvm_irq_ack_notifier *kian; struct hlist_node *n; unsigned gsi = pin; + int i; trace_kvm_ack_irq(irqchip, pin); - list_for_each_entry(e, &kvm->irq_routing, link) + for (i = 0; i < kvm->irq_routing->nr_rt_entries; i++) { + struct kvm_kernel_irq_routing_entry *e; + e = &kvm->irq_routing->rt_entries[i]; if (e->type == KVM_IRQ_ROUTING_IRQCHIP && e->irqchip.irqchip == irqchip && e->irqchip.pin == pin) { gsi = e->gsi; break; } + } hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, link) if (kian->gsi == gsi) @@ -280,26 +286,30 @@ void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask) kimn->func(kimn, mask); } -static void __kvm_free_irq_routing(struct list_head *irq_routing) -{ - struct kvm_kernel_irq_routing_entry *e, *n; - - list_for_each_entry_safe(e, n, irq_routing, link) - kfree(e); -} - void kvm_free_irq_routing(struct kvm *kvm) { mutex_lock(&kvm->irq_lock); - __kvm_free_irq_routing(&kvm->irq_routing); + kfree(kvm->irq_routing); mutex_unlock(&kvm->irq_lock); } -static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e, +static int setup_routing_entry(struct kvm_irq_routing_table *rt, + struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { int r = -EINVAL; int delta; + struct kvm_kernel_irq_routing_entry *ei; + struct hlist_node *n; + + /* + * Do not allow GSI to be mapped to the same irqchip more than once. + * Allow only one to one mapping between GSI and MSI. + */ + hlist_for_each_entry(ei, n, &rt->map[ue->gsi], link) + if (ei->type == KVM_IRQ_ROUTING_MSI || + ue->u.irqchip.irqchip == ei->irqchip.irqchip) + return r; e->gsi = ue->gsi; e->type = ue->type; @@ -332,6 +342,8 @@ static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e, default: goto out; } + + hlist_add_head(&e->link, &rt->map[e->gsi]); r = 0; out: return r; @@ -343,43 +355,49 @@ int kvm_set_irq_routing(struct kvm *kvm, unsigned nr, unsigned flags) { - struct list_head irq_list = LIST_HEAD_INIT(irq_list); - struct list_head tmp = LIST_HEAD_INIT(tmp); - struct kvm_kernel_irq_routing_entry *e = NULL; - unsigned i; + struct kvm_irq_routing_table *new, *old; + u32 i, nr_rt_entries = 0; int r; + for (i = 0; i < nr; ++i) { + if (ue[i].gsi >= KVM_MAX_IRQ_ROUTES) + return -EINVAL; + nr_rt_entries = max(nr_rt_entries, ue[i].gsi); + } + + nr_rt_entries += 1; + + new = kzalloc(sizeof(*new) + (nr_rt_entries * sizeof(struct hlist_head)) + + (nr * sizeof(struct kvm_kernel_irq_routing_entry)), + GFP_KERNEL); + + if (!new) + return -ENOMEM; + + new->rt_entries = (void *)&new->map[nr_rt_entries]; + + new->nr_rt_entries = nr_rt_entries; + for (i = 0; i < nr; ++i) { r = -EINVAL; - if (ue->gsi >= KVM_MAX_IRQ_ROUTES) - goto out; if (ue->flags) goto out; - r = -ENOMEM; - e = kzalloc(sizeof(*e), GFP_KERNEL); - if (!e) - goto out; - r = setup_routing_entry(e, ue); + r = setup_routing_entry(new, &new->rt_entries[i], ue); if (r) goto out; ++ue; - list_add(&e->link, &irq_list); - e = NULL; } mutex_lock(&kvm->irq_lock); - list_splice(&kvm->irq_routing, &tmp); - INIT_LIST_HEAD(&kvm->irq_routing); - list_splice(&irq_list, &kvm->irq_routing); - INIT_LIST_HEAD(&irq_list); - list_splice(&tmp, &irq_list); + old = kvm->irq_routing; + kvm->irq_routing = new; mutex_unlock(&kvm->irq_lock); + new = old; r = 0; out: - kfree(e); - __kvm_free_irq_routing(&irq_list); + kfree(new); return r; } -- cgit v1.2.3 From 3e71f88bc90792a187703860cf22fbed7c12cbd9 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:21 +0300 Subject: KVM: Maintain back mapping from irqchip/pin to gsi Maintain back mapping from irqchip/pin to gsi to speedup interrupt acknowledgment notifications. [avi: build fix on non-x86/ia64] Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 81950f6f6fd..59cf8dae006 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -175,25 +175,16 @@ void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) { struct kvm_irq_ack_notifier *kian; struct hlist_node *n; - unsigned gsi = pin; - int i; + int gsi; trace_kvm_ack_irq(irqchip, pin); - for (i = 0; i < kvm->irq_routing->nr_rt_entries; i++) { - struct kvm_kernel_irq_routing_entry *e; - e = &kvm->irq_routing->rt_entries[i]; - if (e->type == KVM_IRQ_ROUTING_IRQCHIP && - e->irqchip.irqchip == irqchip && - e->irqchip.pin == pin) { - gsi = e->gsi; - break; - } - } - - hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, link) - if (kian->gsi == gsi) - kian->irq_acked(kian); + gsi = kvm->irq_routing->chip[irqchip][pin]; + if (gsi != -1) + hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, + link) + if (kian->gsi == gsi) + kian->irq_acked(kian); } void kvm_register_irq_ack_notifier(struct kvm *kvm, @@ -332,6 +323,9 @@ static int setup_routing_entry(struct kvm_irq_routing_table *rt, } e->irqchip.irqchip = ue->u.irqchip.irqchip; e->irqchip.pin = ue->u.irqchip.pin + delta; + if (e->irqchip.pin >= KVM_IOAPIC_NUM_PINS) + goto out; + rt->chip[ue->u.irqchip.irqchip][e->irqchip.pin] = ue->gsi; break; case KVM_IRQ_ROUTING_MSI: e->set = kvm_set_msi; @@ -356,7 +350,7 @@ int kvm_set_irq_routing(struct kvm *kvm, unsigned flags) { struct kvm_irq_routing_table *new, *old; - u32 i, nr_rt_entries = 0; + u32 i, j, nr_rt_entries = 0; int r; for (i = 0; i < nr; ++i) { @@ -377,6 +371,9 @@ int kvm_set_irq_routing(struct kvm *kvm, new->rt_entries = (void *)&new->map[nr_rt_entries]; new->nr_rt_entries = nr_rt_entries; + for (i = 0; i < 3; i++) + for (j = 0; j < KVM_IOAPIC_NUM_PINS; j++) + new->chip[i][j] = -1; for (i = 0; i < nr; ++i) { r = -EINVAL; -- cgit v1.2.3 From e42bba90bfe724e32fc66ab1b6cb283c51d064b1 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:22 +0300 Subject: KVM: Move irq routing data structure to rcu locking Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 59cf8dae006..fb861dd956f 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -159,7 +159,8 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) * IOAPIC. So set the bit in both. The guest will ignore * writes to the unused one. */ - irq_rt = kvm->irq_routing; + rcu_read_lock(); + irq_rt = rcu_dereference(kvm->irq_routing); if (irq < irq_rt->nr_rt_entries) hlist_for_each_entry(e, n, &irq_rt->map[irq], link) { int r = e->set(e, kvm, irq_source_id, level); @@ -168,6 +169,7 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) ret = r + ((ret < 0) ? 0 : ret); } + rcu_read_unlock(); return ret; } @@ -179,7 +181,10 @@ void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) trace_kvm_ack_irq(irqchip, pin); - gsi = kvm->irq_routing->chip[irqchip][pin]; + rcu_read_lock(); + gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin]; + rcu_read_unlock(); + if (gsi != -1) hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, link) @@ -279,9 +284,9 @@ void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask) void kvm_free_irq_routing(struct kvm *kvm) { - mutex_lock(&kvm->irq_lock); + /* Called only during vm destruction. Nobody can use the pointer + at this stage */ kfree(kvm->irq_routing); - mutex_unlock(&kvm->irq_lock); } static int setup_routing_entry(struct kvm_irq_routing_table *rt, @@ -387,8 +392,9 @@ int kvm_set_irq_routing(struct kvm *kvm, mutex_lock(&kvm->irq_lock); old = kvm->irq_routing; - kvm->irq_routing = new; + rcu_assign_pointer(kvm->irq_routing, new); mutex_unlock(&kvm->irq_lock); + synchronize_rcu(); new = old; r = 0; -- cgit v1.2.3 From 136bdfeee7b5bc986fc94af3a40d7d13ea37bb95 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:23 +0300 Subject: KVM: Move irq ack notifier list to arch independent code Mask irq notifier list is already there. Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index fb861dd956f..f0197259593 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -186,8 +186,7 @@ void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) rcu_read_unlock(); if (gsi != -1) - hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, - link) + hlist_for_each_entry(kian, n, &kvm->irq_ack_notifier_list, link) if (kian->gsi == gsi) kian->irq_acked(kian); } @@ -196,7 +195,7 @@ void kvm_register_irq_ack_notifier(struct kvm *kvm, struct kvm_irq_ack_notifier *kian) { mutex_lock(&kvm->irq_lock); - hlist_add_head(&kian->link, &kvm->arch.irq_ack_notifier_list); + hlist_add_head(&kian->link, &kvm->irq_ack_notifier_list); mutex_unlock(&kvm->irq_lock); } -- cgit v1.2.3 From 280aa177dcd1edc718d8a92f17f235b783ec6307 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:24 +0300 Subject: KVM: Convert irq notifiers lists to RCU locking Use RCU locking for mask/ack notifiers lists. Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index f0197259593..6c946141dbc 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -183,19 +183,19 @@ void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin) rcu_read_lock(); gsi = rcu_dereference(kvm->irq_routing)->chip[irqchip][pin]; - rcu_read_unlock(); - if (gsi != -1) - hlist_for_each_entry(kian, n, &kvm->irq_ack_notifier_list, link) + hlist_for_each_entry_rcu(kian, n, &kvm->irq_ack_notifier_list, + link) if (kian->gsi == gsi) kian->irq_acked(kian); + rcu_read_unlock(); } void kvm_register_irq_ack_notifier(struct kvm *kvm, struct kvm_irq_ack_notifier *kian) { mutex_lock(&kvm->irq_lock); - hlist_add_head(&kian->link, &kvm->irq_ack_notifier_list); + hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list); mutex_unlock(&kvm->irq_lock); } @@ -203,8 +203,9 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm, struct kvm_irq_ack_notifier *kian) { mutex_lock(&kvm->irq_lock); - hlist_del_init(&kian->link); + hlist_del_init_rcu(&kian->link); mutex_unlock(&kvm->irq_lock); + synchronize_rcu(); } int kvm_request_irq_source_id(struct kvm *kvm) @@ -257,7 +258,7 @@ void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, { mutex_lock(&kvm->irq_lock); kimn->irq = irq; - hlist_add_head(&kimn->link, &kvm->mask_notifier_list); + hlist_add_head_rcu(&kimn->link, &kvm->mask_notifier_list); mutex_unlock(&kvm->irq_lock); } @@ -265,8 +266,9 @@ void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, struct kvm_irq_mask_notifier *kimn) { mutex_lock(&kvm->irq_lock); - hlist_del(&kimn->link); + hlist_del_rcu(&kimn->link); mutex_unlock(&kvm->irq_lock); + synchronize_rcu(); } void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask) @@ -274,11 +276,11 @@ void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask) struct kvm_irq_mask_notifier *kimn; struct hlist_node *n; - WARN_ON(!mutex_is_locked(&kvm->irq_lock)); - - hlist_for_each_entry(kimn, n, &kvm->mask_notifier_list, link) + rcu_read_lock(); + hlist_for_each_entry_rcu(kimn, n, &kvm->mask_notifier_list, link) if (kimn->irq == irq) kimn->func(kimn, mask); + rcu_read_unlock(); } void kvm_free_irq_routing(struct kvm *kvm) -- cgit v1.2.3 From eba0226bdfffe262e72b8360e4d0d12070e9a0f0 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:25 +0300 Subject: KVM: Move IO APIC to its own lock The allows removal of irq_lock from the injection path. Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 6c946141dbc..fadf4408a82 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -146,8 +146,8 @@ static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, */ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) { - struct kvm_kernel_irq_routing_entry *e; - int ret = -1; + struct kvm_kernel_irq_routing_entry *e, irq_set[KVM_NR_IRQCHIPS]; + int ret = -1, i = 0; struct kvm_irq_routing_table *irq_rt; struct hlist_node *n; @@ -162,14 +162,19 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) rcu_read_lock(); irq_rt = rcu_dereference(kvm->irq_routing); if (irq < irq_rt->nr_rt_entries) - hlist_for_each_entry(e, n, &irq_rt->map[irq], link) { - int r = e->set(e, kvm, irq_source_id, level); - if (r < 0) - continue; - - ret = r + ((ret < 0) ? 0 : ret); - } + hlist_for_each_entry(e, n, &irq_rt->map[irq], link) + irq_set[i++] = *e; rcu_read_unlock(); + + while(i--) { + int r; + r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level); + if (r < 0) + continue; + + ret = r + ((ret < 0) ? 0 : ret); + } + return ret; } -- cgit v1.2.3 From 680b3648ba89c44ac8d0316f78a0d6e147b88809 Mon Sep 17 00:00:00 2001 From: Gleb Natapov Date: Mon, 24 Aug 2009 11:54:26 +0300 Subject: KVM: Drop kvm->irq_lock lock from irq injection path The only thing it protects now is interrupt injection into lapic and this can work lockless. Even now with kvm->irq_lock in place access to lapic is not entirely serialized since vcpu access doesn't take kvm->irq_lock. Signed-off-by: Gleb Natapov Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index fadf4408a82..15a83b93566 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -82,8 +82,6 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, int i, r = -1; struct kvm_vcpu *vcpu, *lowest = NULL; - WARN_ON(!mutex_is_locked(&kvm->irq_lock)); - if (irq->dest_mode == 0 && irq->dest_id == 0xff && kvm_is_dm_lowest_prio(irq)) printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n"); @@ -138,7 +136,7 @@ static int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, return kvm_irq_delivery_to_apic(kvm, NULL, &irq); } -/* This should be called with the kvm->irq_lock mutex held +/* * Return value: * < 0 Interrupt was ignored (masked or not delivered for other reasons) * = 0 Interrupt was coalesced (previous irq is still pending) @@ -153,8 +151,6 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level) trace_kvm_set_irq(irq, level, irq_source_id); - WARN_ON(!mutex_is_locked(&kvm->irq_lock)); - /* Not possible to detect if the guest uses the PIC or the * IOAPIC. So set the bit in both. The guest will ignore * writes to the unused one. -- cgit v1.2.3 From 0c6ddcebd8303ada6faefa6f72ac18b6230320c4 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 25 Sep 2009 09:33:38 +0200 Subject: KVM: fix lock imbalance in kvm_*_irq_source_id() Stanse found 2 lock imbalances in kvm_request_irq_source_id and kvm_free_irq_source_id. They omit to unlock kvm->irq_lock on fail paths. Fix that by adding unlock labels at the end of the functions and jump there from the fail paths. Signed-off-by: Jiri Slaby Cc: Marcelo Tosatti Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 15a83b93566..00c68d29857 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -220,11 +220,13 @@ int kvm_request_irq_source_id(struct kvm *kvm) if (irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) { printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); - return -EFAULT; + irq_source_id = -EFAULT; + goto unlock; } ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); set_bit(irq_source_id, bitmap); +unlock: mutex_unlock(&kvm->irq_lock); return irq_source_id; @@ -240,7 +242,7 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) if (irq_source_id < 0 || irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) { printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); - return; + goto unlock; } for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++) { clear_bit(irq_source_id, &kvm->arch.vioapic->irq_states[i]); @@ -251,6 +253,7 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) #endif } clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); +unlock: mutex_unlock(&kvm->irq_lock); } -- cgit v1.2.3 From cd5a2685de4a642fd0bd763e8c19711ef08dbe27 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Sat, 17 Oct 2009 22:47:23 -0300 Subject: KVM: fix irq_source_id size verification find_first_zero_bit works with bit numbers, not bytes. Fixes https://sourceforge.net/tracker/?func=detail&aid=2847560&group_id=180599&atid=893831 Reported-by: "Xu, Jiajun" Cc: stable@kernel.org Signed-off-by: Marcelo Tosatti --- virt/kvm/irq_comm.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 00c68d29857..0d454d35271 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -215,10 +215,9 @@ int kvm_request_irq_source_id(struct kvm *kvm) int irq_source_id; mutex_lock(&kvm->irq_lock); - irq_source_id = find_first_zero_bit(bitmap, - sizeof(kvm->arch.irq_sources_bitmap)); + irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); - if (irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) { + if (irq_source_id >= BITS_PER_LONG) { printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); irq_source_id = -EFAULT; goto unlock; @@ -240,7 +239,7 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) mutex_lock(&kvm->irq_lock); if (irq_source_id < 0 || - irq_source_id >= sizeof(kvm->arch.irq_sources_bitmap)) { + irq_source_id >= BITS_PER_LONG) { printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); goto unlock; } -- cgit v1.2.3 From e50212bb51356f0df48d6cce0aae5acf41df336d Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 29 Oct 2009 13:44:17 -0200 Subject: KVM: only clear irq_source_id if irqchip is present Otherwise kvm might attempt to dereference a NULL pointer. Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- virt/kvm/irq_comm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'virt/kvm/irq_comm.c') diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c index 0d454d35271..9b077342ab5 100644 --- a/virt/kvm/irq_comm.c +++ b/virt/kvm/irq_comm.c @@ -243,6 +243,10 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); goto unlock; } + clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); + if (!irqchip_in_kernel(kvm)) + goto unlock; + for (i = 0; i < KVM_IOAPIC_NUM_PINS; i++) { clear_bit(irq_source_id, &kvm->arch.vioapic->irq_states[i]); if (i >= 16) @@ -251,7 +255,6 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) clear_bit(irq_source_id, &pic_irqchip(kvm)->irq_states[i]); #endif } - clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); unlock: mutex_unlock(&kvm->irq_lock); } -- cgit v1.2.3