diff options
403 files changed, 10750 insertions, 2389 deletions
diff --git a/Documentation/irqflags-tracing.txt b/Documentation/irqflags-tracing.txt new file mode 100644 index 00000000000..6a444877ee0 --- /dev/null +++ b/Documentation/irqflags-tracing.txt @@ -0,0 +1,57 @@ +IRQ-flags state tracing + +started by Ingo Molnar <mingo@redhat.com> + +the "irq-flags tracing" feature "traces" hardirq and softirq state, in +that it gives interested subsystems an opportunity to be notified of +every hardirqs-off/hardirqs-on, softirqs-off/softirqs-on event that +happens in the kernel. + +CONFIG_TRACE_IRQFLAGS_SUPPORT is needed for CONFIG_PROVE_SPIN_LOCKING +and CONFIG_PROVE_RW_LOCKING to be offered by the generic lock debugging +code. Otherwise only CONFIG_PROVE_MUTEX_LOCKING and +CONFIG_PROVE_RWSEM_LOCKING will be offered on an architecture - these +are locking APIs that are not used in IRQ context. (the one exception +for rwsems is worked around) + +architecture support for this is certainly not in the "trivial" +category, because lots of lowlevel assembly code deal with irq-flags +state changes. But an architecture can be irq-flags-tracing enabled in a +rather straightforward and risk-free manner. + +Architectures that want to support this need to do a couple of +code-organizational changes first: + +- move their irq-flags manipulation code from their asm/system.h header + to asm/irqflags.h + +- rename local_irq_disable()/etc to raw_local_irq_disable()/etc. so that + the linux/irqflags.h code can inject callbacks and can construct the + real local_irq_disable()/etc APIs. + +- add and enable TRACE_IRQFLAGS_SUPPORT in their arch level Kconfig file + +and then a couple of functional changes are needed as well to implement +irq-flags-tracing support: + +- in lowlevel entry code add (build-conditional) calls to the + trace_hardirqs_off()/trace_hardirqs_on() functions. The lock validator + closely guards whether the 'real' irq-flags matches the 'virtual' + irq-flags state, and complains loudly (and turns itself off) if the + two do not match. Usually most of the time for arch support for + irq-flags-tracing is spent in this state: look at the lockdep + complaint, try to figure out the assembly code we did not cover yet, + fix and repeat. Once the system has booted up and works without a + lockdep complaint in the irq-flags-tracing functions arch support is + complete. +- if the architecture has non-maskable interrupts then those need to be + excluded from the irq-tracing [and lock validation] mechanism via + lockdep_off()/lockdep_on(). + +in general there is no risk from having an incomplete irq-flags-tracing +implementation in an architecture: lockdep will detect that and will +turn itself off. I.e. the lock validator will still be reliable. There +should be no crashes due to irq-tracing bugs. (except if the assembly +changes break other code by modifying conditions or registers that +shouldnt be) + diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 86e9282d1c2..149f62ba14a 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -435,6 +435,15 @@ running once the system is up. debug [KNL] Enable kernel debugging (events log level). + debug_locks_verbose= + [KNL] verbose self-tests + Format=<0|1> + Print debugging info while doing the locking API + self-tests. + We default to 0 (no extra messages), setting it to + 1 will print _a lot_ more information - normally + only useful to kernel developers. + decnet= [HW,NET] Format: <area>[,<node>] See also Documentation/networking/decnet.txt. diff --git a/Documentation/lockdep-design.txt b/Documentation/lockdep-design.txt new file mode 100644 index 00000000000..00d93605bfd --- /dev/null +++ b/Documentation/lockdep-design.txt @@ -0,0 +1,197 @@ +Runtime locking correctness validator +===================================== + +started by Ingo Molnar <mingo@redhat.com> +additions by Arjan van de Ven <arjan@linux.intel.com> + +Lock-class +---------- + +The basic object the validator operates upon is a 'class' of locks. + +A class of locks is a group of locks that are logically the same with +respect to locking rules, even if the locks may have multiple (possibly +tens of thousands of) instantiations. For example a lock in the inode +struct is one class, while each inode has its own instantiation of that +lock class. + +The validator tracks the 'state' of lock-classes, and it tracks +dependencies between different lock-classes. The validator maintains a +rolling proof that the state and the dependencies are correct. + +Unlike an lock instantiation, the lock-class itself never goes away: when +a lock-class is used for the first time after bootup it gets registered, +and all subsequent uses of that lock-class will be attached to this +lock-class. + +State +----- + +The validator tracks lock-class usage history into 5 separate state bits: + +- 'ever held in hardirq context' [ == hardirq-safe ] +- 'ever held in softirq context' [ == softirq-safe ] +- 'ever held with hardirqs enabled' [ == hardirq-unsafe ] +- 'ever held with softirqs and hardirqs enabled' [ == softirq-unsafe ] + +- 'ever used' [ == !unused ] + +Single-lock state rules: +------------------------ + +A softirq-unsafe lock-class is automatically hardirq-unsafe as well. The +following states are exclusive, and only one of them is allowed to be +set for any lock-class: + + <hardirq-safe> and <hardirq-unsafe> + <softirq-safe> and <softirq-unsafe> + +The validator detects and reports lock usage that violate these +single-lock state rules. + +Multi-lock dependency rules: +---------------------------- + +The same lock-class must not be acquired twice, because this could lead +to lock recursion deadlocks. + +Furthermore, two locks may not be taken in different order: + + <L1> -> <L2> + <L2> -> <L1> + +because this could lead to lock inversion deadlocks. (The validator +finds such dependencies in arbitrary complexity, i.e. there can be any +other locking sequence between the acquire-lock operations, the +validator will still track all dependencies between locks.) + +Furthermore, the following usage based lock dependencies are not allowed +between any two lock-classes: + + <hardirq-safe> -> <hardirq-unsafe> + <softirq-safe> -> <softirq-unsafe> + +The first rule comes from the fact the a hardirq-safe lock could be +taken by a hardirq context, interrupting a hardirq-unsafe lock - and +thus could result in a lock inversion deadlock. Likewise, a softirq-safe +lock could be taken by an softirq context, interrupting a softirq-unsafe +lock. + +The above rules are enforced for any locking sequence that occurs in the +kernel: when acquiring a new lock, the validator checks whether there is +any rule violation between the new lock and any of the held locks. + +When a lock-class changes its state, the following aspects of the above +dependency rules are enforced: + +- if a new hardirq-safe lock is discovered, we check whether it + took any hardirq-unsafe lock in the past. + +- if a new softirq-safe lock is discovered, we check whether it took + any softirq-unsafe lock in the past. + +- if a new hardirq-unsafe lock is discovered, we check whether any + hardirq-safe lock took it in the past. + +- if a new softirq-unsafe lock is discovered, we check whether any + softirq-safe lock took it in the past. + +(Again, we do these checks too on the basis that an interrupt context +could interrupt _any_ of the irq-unsafe or hardirq-unsafe locks, which +could lead to a lock inversion deadlock - even if that lock scenario did +not trigger in practice yet.) + +Exception: Nested data dependencies leading to nested locking +------------------------------------------------------------- + +There are a few cases where the Linux kernel acquires more than one +instance of the same lock-class. Such cases typically happen when there +is some sort of hierarchy within objects of the same type. In these +cases there is an inherent "natural" ordering between the two objects +(defined by the properties of the hierarchy), and the kernel grabs the +locks in this fixed order on each of the objects. + +An example of such an object hieararchy that results in "nested locking" +is that of a "whole disk" block-dev object and a "partition" block-dev +object; the partition is "part of" the whole device and as long as one +always takes the whole disk lock as a higher lock than the partition +lock, the lock ordering is fully correct. The validator does not +automatically detect this natural ordering, as the locking rule behind +the ordering is not static. + +In order to teach the validator about this correct usage model, new +versions of the various locking primitives were added that allow you to +specify a "nesting level". An example call, for the block device mutex, +looks like this: + +enum bdev_bd_mutex_lock_class +{ + BD_MUTEX_NORMAL, + BD_MUTEX_WHOLE, + BD_MUTEX_PARTITION +}; + + mutex_lock_nested(&bdev->bd_contains->bd_mutex, BD_MUTEX_PARTITION); + +In this case the locking is done on a bdev object that is known to be a +partition. + +The validator treats a lock that is taken in such a nested fasion as a +separate (sub)class for the purposes of validation. + +Note: When changing code to use the _nested() primitives, be careful and +check really thoroughly that the hiearchy is correctly mapped; otherwise +you can get false positives or false negatives. + +Proof of 100% correctness: +-------------------------- + +The validator achieves perfect, mathematical 'closure' (proof of locking +correctness) in the sense that for every simple, standalone single-task +locking sequence that occured at least once during the lifetime of the +kernel, the validator proves it with a 100% certainty that no +combination and timing of these locking sequences can cause any class of +lock related deadlock. [*] + +I.e. complex multi-CPU and multi-task locking scenarios do not have to +occur in practice to prove a deadlock: only the simple 'component' +locking chains have to occur at least once (anytime, in any +task/context) for the validator to be able to prove correctness. (For +example, complex deadlocks that would normally need more than 3 CPUs and +a very unlikely constellation of tasks, irq-contexts and timings to +occur, can be detected on a plain, lightly loaded single-CPU system as +well!) + +This radically decreases the complexity of locking related QA of the +kernel: what has to be done during QA is to trigger as many "simple" +single-task locking dependencies in the kernel as possible, at least +once, to prove locking correctness - instead of having to trigger every +possible combination of locking interaction between CPUs, combined with +every possible hardirq and softirq nesting scenario (which is impossible +to do in practice). + +[*] assuming that the validator itself is 100% correct, and no other + part of the system corrupts the state of the validator in any way. + We also assume that all NMI/SMM paths [which could interrupt + even hardirq-disabled codepaths] are correct and do not interfere + with the validator. We also assume that the 64-bit 'chain hash' + value is unique for every lock-chain in the system. Also, lock + recursion must not be higher than 20. + +Performance: +------------ + +The above rules require _massive_ amounts of runtime checking. If we did +that for every lock taken and for every irqs-enable event, it would +render the system practically unusably slow. The complexity of checking +is O(N^2), so even with just a few hundred lock-classes we'd have to do +tens of thousands of checks for every event. + +This problem is solved by checking any given 'locking scenario' (unique +sequence of locks taken after each other) only once. A simple stack of +held locks is maintained, and a lightweight 64-bit hash value is +calculated, which hash is unique for every lock chain. The hash value, +when the chain is validated for the first time, is then put into a hash +table, which hash-table can be checked in a lockfree manner. If the +locking chain occurs again later on, the hash table tells us that we +dont have to validate the chain again. diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt index 86754eb390d..7cee90223d3 100644 --- a/Documentation/sysctl/vm.txt +++ b/Documentation/sysctl/vm.txt @@ -28,6 +28,7 @@ Currently, these files are in /proc/sys/vm: - block_dump - drop-caches - zone_reclaim_mode +- min_unmapped_ratio - panic_on_oom ============================================================== @@ -168,6 +169,19 @@ in all nodes of the system. ============================================================= +min_unmapped_ratio: + +This is available only on NUMA kernels. + +A percentage of the file backed pages in each zone. Zone reclaim will only +occur if more than this percentage of pages are file backed and unmapped. +This is to insure that a minimal amount of local pages is still available for +file I/O even if the node is overallocated. + +The default is 1 percent. + +============================================================= + panic_on_oom This enables or disables panic on out-of-memory feature. If this is set to 1, diff --git a/MAINTAINERS b/MAINTAINERS index 42be131139c..5f76a4f5cd4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -861,6 +861,8 @@ S: Maintained DOCBOOK FOR DOCUMENTATION P: Martin Waitz M: tali@admingilde.org +P: Randy Dunlap +M: rdunlap@xenotime.net T: git http://tali.admingilde.org/git/linux-docbook.git S: Maintained @@ -2298,6 +2300,14 @@ M: promise@pnd-pc.demon.co.uk W: http://www.pnd-pc.demon.co.uk/promise/ S: Maintained +PVRUSB2 VIDEO4LINUX DRIVER +P: Mike Isely +M: isely@pobox.com +L: pvrusb2@isely.net +L: video4linux-list@redhat.com +W: http://www.isely.net/pvrusb2/ +S: Maintained + PXA2xx SUPPORT P: Nicolas Pitre M: nico@cam.org diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index 01c8c8b2333..41ebf51a107 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -474,7 +474,7 @@ out: */ unsigned long -thread_saved_pc(task_t *t) +thread_saved_pc(struct task_struct *t) { unsigned long base = (unsigned long)task_stack_page(t); unsigned long fp, sp = task_thread_info(t)->pcb.ksp; diff --git a/arch/arm/common/time-acorn.c b/arch/arm/common/time-acorn.c index 31b65e2231d..3f60dd9aca8 100644 --- a/arch/arm/common/time-acorn.c +++ b/arch/arm/common/time-acorn.c @@ -77,7 +77,7 @@ ioc_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction ioc_timer_irq = { .name = "timer", - .flags = SA_INTERRUPT, + .flags = IRQF_DISABLED, .handler = ioc_timer_interrupt }; diff --git a/arch/arm/mach-aaec2000/core.c b/arch/arm/mach-aaec2000/core.c index aa01d6753d6..baa997c857d 100644 --- a/arch/arm/mach-aaec2000/core.c +++ b/arch/arm/mach-aaec2000/core.c @@ -142,7 +142,7 @@ aaec2000_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction aaec2000_timer_irq = { .name = "AAEC-2000 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = aaec2000_timer_interrupt, }; diff --git a/arch/arm/mach-at91rm9200/at91rm9200_time.c b/arch/arm/mach-at91rm9200/at91rm9200_time.c index 0aa22650a00..a92a8622c78 100644 --- a/arch/arm/mach-at91rm9200/at91rm9200_time.c +++ b/arch/arm/mach-at91rm9200/at91rm9200_time.c @@ -85,7 +85,7 @@ static irqreturn_t at91rm9200_timer_interrupt(int irq, void *dev_id, struct pt_r static struct irqaction at91rm9200_timer_irq = { .name = "at91_tick", - .flags = SA_SHIRQ | SA_INTERRUPT | SA_TIMER, + .flags = IRQF_SHARED | IRQF_DISABLED | IRQF_TIMER, .handler = at91rm9200_timer_interrupt }; diff --git a/arch/arm/mach-clps711x/time.c b/arch/arm/mach-clps711x/time.c index b0f1db258e8..a071eac4a30 100644 --- a/arch/arm/mach-clps711x/time.c +++ b/arch/arm/mach-clps711x/time.c @@ -58,7 +58,7 @@ p720t_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction clps711x_timer_irq = { .name = "CLPS711x Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = p720t_timer_interrupt, }; diff --git a/arch/arm/mach-clps7500/core.c b/arch/arm/mach-clps7500/core.c index cd66df89636..92eaebdd560 100644 --- a/arch/arm/mach-clps7500/core.c +++ b/arch/arm/mach-clps7500/core.c @@ -316,7 +316,7 @@ clps7500_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction clps7500_timer_irq = { .name = "CLPS7500 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = clps7500_timer_interrupt, }; diff --git a/arch/arm/mach-ebsa110/core.c b/arch/arm/mach-ebsa110/core.c index 6d620d8268c..70dd12ef3c4 100644 --- a/arch/arm/mach-ebsa110/core.c +++ b/arch/arm/mach-ebsa110/core.c @@ -199,7 +199,7 @@ ebsa110_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction ebsa110_timer_irq = { .name = "EBSA110 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = ebsa110_timer_interrupt, }; diff --git a/arch/arm/mach-ep93xx/core.c b/arch/arm/mach-ep93xx/core.c index 26df3b666b5..a87a784b920 100644 --- a/arch/arm/mach-ep93xx/core.c +++ b/arch/arm/mach-ep93xx/core.c @@ -116,7 +116,7 @@ static int ep93xx_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction ep93xx_timer_irq = { .name = "ep93xx timer", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = ep93xx_timer_interrupt, }; diff --git a/arch/arm/mach-footbridge/dc21285-timer.c b/arch/arm/mach-footbridge/dc21285-timer.c index e668d4acd80..2af610811ca 100644 --- a/arch/arm/mach-footbridge/dc21285-timer.c +++ b/arch/arm/mach-footbridge/dc21285-timer.c @@ -44,7 +44,7 @@ timer1_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction footbridge_timer_irq = { .name = "Timer1 timer tick", .handler = timer1_interrupt, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; /* diff --git a/arch/arm/mach-footbridge/dc21285.c b/arch/arm/mach-footbridge/dc21285.c index 5dace259783..607ed1f5b3f 100644 --- a/arch/arm/mach-footbridge/dc21285.c +++ b/arch/arm/mach-footbridge/dc21285.c @@ -332,15 +332,15 @@ void __init dc21285_preinit(void) /* * We don't care if these fail. */ - request_irq(IRQ_PCI_SERR, dc21285_serr_irq, SA_INTERRUPT, + request_irq(IRQ_PCI_SERR, dc21285_serr_irq, IRQF_DISABLED, "PCI system error", &serr_timer); - request_irq(IRQ_PCI_PERR, dc21285_parity_irq, SA_INTERRUPT, + request_irq(IRQ_PCI_PERR, dc21285_parity_irq, IRQF_DISABLED, "PCI parity error", &perr_timer); - request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, SA_INTERRUPT, + request_irq(IRQ_PCI_ABORT, dc21285_abort_irq, IRQF_DISABLED, "PCI abort", NULL); - request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, SA_INTERRUPT, + request_irq(IRQ_DISCARD_TIMER, dc21285_discard_irq, IRQF_DISABLED, "Discard timer", NULL); - request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, SA_INTERRUPT, + request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, IRQF_DISABLED, "PCI data parity", NULL); if (cfn_mode) { diff --git a/arch/arm/mach-footbridge/isa-timer.c b/arch/arm/mach-footbridge/isa-timer.c index 282b473c21f..c4810a40c8e 100644 --- a/arch/arm/mach-footbridge/isa-timer.c +++ b/arch/arm/mach-footbridge/isa-timer.c @@ -73,7 +73,7 @@ isa_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction isa_timer_irq = { .name = "ISA timer tick", .handler = isa_timer_interrupt, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; static void __init isa_timer_init(void) diff --git a/arch/arm/mach-h720x/cpu-h7201.c b/arch/arm/mach-h720x/cpu-h7201.c index af9e4a5d5ea..a9a8255a3a0 100644 --- a/arch/arm/mach-h720x/cpu-h7201.c +++ b/arch/arm/mach-h720x/cpu-h7201.c @@ -41,7 +41,7 @@ h7201_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction h7201_timer_irq = { .name = "h7201 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = h7201_timer_interrupt, }; diff --git a/arch/arm/mach-h720x/cpu-h7202.c b/arch/arm/mach-h720x/cpu-h7202.c index a4a7c0125d0..da678d163fd 100644 --- a/arch/arm/mach-h720x/cpu-h7202.c +++ b/arch/arm/mach-h720x/cpu-h7202.c @@ -171,7 +171,7 @@ static struct irqchip h7202_timerx_chip = { static struct irqaction h7202_timer_irq = { .name = "h7202 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = h7202_timer_interrupt, }; diff --git a/arch/arm/mach-imx/time.c b/arch/arm/mach-imx/time.c index 5f9a04775a4..6ed7523c65b 100644 --- a/arch/arm/mach-imx/time.c +++ b/arch/arm/mach-imx/time.c @@ -72,7 +72,7 @@ imx_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction imx_timer_irq = { .name = "i.MX Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = imx_timer_interrupt, }; diff --git a/arch/arm/mach-integrator/core.c b/arch/arm/mach-integrator/core.c index 2d7e505e748..42021fdfa0c 100644 --- a/arch/arm/mach-integrator/core.c +++ b/arch/arm/mach-integrator/core.c @@ -282,7 +282,7 @@ integrator_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction integrator_timer_irq = { .name = "Integrator Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = integrator_timer_interrupt, }; diff --git a/arch/arm/mach-integrator/time.c b/arch/arm/mach-integrator/time.c index bc07f52a6fd..ee49cf790da 100644 --- a/arch/arm/mach-integrator/time.c +++ b/arch/arm/mach-integrator/time.c @@ -125,7 +125,7 @@ static int rtc_probe(struct amba_device *dev, void *id) xtime.tv_sec = __raw_readl(rtc_base + RTC_DR); - ret = request_irq(dev->irq[0], arm_rtc_interrupt, SA_INTERRUPT, + ret = request_irq(dev->irq[0], arm_rtc_interrupt, IRQF_DISABLED, "rtc-pl030", dev); if (ret) goto map_out; diff --git a/arch/arm/mach-iop3xx/iop321-time.c b/arch/arm/mach-iop3xx/iop321-time.c index d67ac0e5d43..04b1a6f7eba 100644 --- a/arch/arm/mach-iop3xx/iop321-time.c +++ b/arch/arm/mach-iop3xx/iop321-time.c @@ -85,7 +85,7 @@ iop321_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction iop321_timer_irq = { .name = "IOP321 Timer Tick", .handler = iop321_timer_interrupt, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; static void __init iop321_timer_init(void) diff --git a/arch/arm/mach-iop3xx/iop331-time.c b/arch/arm/mach-iop3xx/iop331-time.c index 3c1f0ebbd63..0c09e74c574 100644 --- a/arch/arm/mach-iop3xx/iop331-time.c +++ b/arch/arm/mach-iop3xx/iop331-time.c @@ -82,7 +82,7 @@ iop331_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction iop331_timer_irq = { .name = "IOP331 Timer Tick", .handler = iop331_timer_interrupt, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; static void __init iop331_timer_init(void) diff --git a/arch/arm/mach-ixp2000/core.c b/arch/arm/mach-ixp2000/core.c index b31f31ac937..7f91f689a04 100644 --- a/arch/arm/mach-ixp2000/core.c +++ b/arch/arm/mach-ixp2000/core.c @@ -224,7 +224,7 @@ static int ixp2000_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction ixp2000_timer_irq = { .name = "IXP2000 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = ixp2000_timer_interrupt, }; diff --git a/arch/arm/mach-ixp23xx/core.c b/arch/arm/mach-ixp23xx/core.c index 7c218aecf44..566a07821c7 100644 --- a/arch/arm/mach-ixp23xx/core.c +++ b/arch/arm/mach-ixp23xx/core.c @@ -363,7 +363,7 @@ ixp23xx_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction ixp23xx_timer_irq = { .name = "IXP23xx Timer Tick", .handler = ixp23xx_timer_interrupt, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; void __init ixp23xx_init_timer(void) diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 13c7c629d03..7c25dbd5a18 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -287,7 +287,7 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id, struct pt_regs static struct irqaction ixp4xx_timer_irq = { .name = "IXP4xx Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = ixp4xx_timer_interrupt, }; diff --git a/arch/arm/mach-ixp4xx/nas100d-power.c b/arch/arm/mach-ixp4xx/nas100d-power.c index f58a1d05a02..81ffcae1f56 100644 --- a/arch/arm/mach-ixp4xx/nas100d-power.c +++ b/arch/arm/mach-ixp4xx/nas100d-power.c @@ -42,7 +42,7 @@ static int __init nas100d_power_init(void) set_irq_type(NAS100D_RB_IRQ, IRQT_LOW); if (request_irq(NAS100D_RB_IRQ, &nas100d_reset_handler, - SA_INTERRUPT, "NAS100D reset button", NULL) < 0) { + IRQF_DISABLED, "NAS100D reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", NAS100D_RB_IRQ); diff --git a/arch/arm/mach-ixp4xx/nslu2-power.c b/arch/arm/mach-ixp4xx/nslu2-power.c index 6d38e97142c..e2a2230b69f 100644 --- a/arch/arm/mach-ixp4xx/nslu2-power.c +++ b/arch/arm/mach-ixp4xx/nslu2-power.c @@ -54,7 +54,7 @@ static int __init nslu2_power_init(void) set_irq_type(NSLU2_PB_IRQ, IRQT_HIGH); if (request_irq(NSLU2_RB_IRQ, &nslu2_reset_handler, - SA_INTERRUPT, "NSLU2 reset button", NULL) < 0) { + IRQF_DISABLED, "NSLU2 reset button", NULL) < 0) { printk(KERN_DEBUG "Reset Button IRQ %d not available\n", NSLU2_RB_IRQ); @@ -63,7 +63,7 @@ static int __init nslu2_power_init(void) } if (request_irq(NSLU2_PB_IRQ, &nslu2_power_handler, - SA_INTERRUPT, "NSLU2 power button", NULL) < 0) { + IRQF_DISABLED, "NSLU2 power button", NULL) < 0) { printk(KERN_DEBUG "Power Button IRQ %d not available\n", NSLU2_PB_IRQ); diff --git a/arch/arm/mach-lh7a40x/time.c b/arch/arm/mach-lh7a40x/time.c index 4d26c9f62c7..ad5652e0150 100644 --- a/arch/arm/mach-lh7a40x/time.c +++ b/arch/arm/mach-lh7a40x/time.c @@ -53,7 +53,7 @@ lh7a40x_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction lh7a40x_timer_irq = { .name = "LHA740x Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = lh7a40x_timer_interrupt, }; diff --git a/arch/arm/mach-netx/time.c b/arch/arm/mach-netx/time.c index edfbdf40c60..6d72c81b7d9 100644 --- a/arch/arm/mach-netx/time.c +++ b/arch/arm/mach-netx/time.c @@ -54,7 +54,7 @@ netx_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction netx_timer_irq = { .name = "NetX Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = netx_timer_interrupt, }; diff --git a/arch/arm/mach-omap1/board-osk.c b/arch/arm/mach-omap1/board-osk.c index 91933301bb7..b742261c97a 100644 --- a/arch/arm/mach-omap1/board-osk.c +++ b/arch/arm/mach-omap1/board-osk.c @@ -357,7 +357,7 @@ static void __init osk_mistral_init(void) */ ret = request_irq(OMAP_GPIO_IRQ(OMAP_MPUIO(2)), &osk_mistral_wake_interrupt, - SA_SHIRQ, "mistral_wakeup", + IRQF_SHARED, "mistral_wakeup", &osk_mistral_wake_interrupt); if (ret != 0) { omap_free_gpio(OMAP_MPUIO(2)); diff --git a/arch/arm/mach-omap1/fpga.c b/arch/arm/mach-omap1/fpga.c index 880cd2d8f4a..34eb79ee6e6 100644 --- a/arch/arm/mach-omap1/fpga.c +++ b/arch/arm/mach-omap1/fpga.c @@ -133,7 +133,7 @@ static struct irqchip omap_fpga_irq = { * mask_ack routine for all of the FPGA interrupts has been changed from * fpga_mask_ack_irq() to fpga_ack_irq() so that the specific FPGA interrupt * being serviced is left unmasked. We can do this because the FPGA cascade - * interrupt is installed with the SA_INTERRUPT flag, which leaves all + * interrupt is installed with the IRQF_DISABLED flag, which leaves all * interrupts masked at the CPU while an FPGA interrupt handler executes. * * Limited testing indicates that this workaround appears to be effective diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c index 1b4e1d57afb..cd76185bab7 100644 --- a/arch/arm/mach-omap1/pm.c +++ b/arch/arm/mach-omap1/pm.c @@ -690,7 +690,7 @@ static irqreturn_t omap_wakeup_interrupt(int irq, void * dev, static struct irqaction omap_wakeup_irq = { .name = "peripheral wakeup", - .flags = SA_INTERRUPT, + .flags = IRQF_DISABLED, .handler = omap_wakeup_interrupt }; diff --git a/arch/arm/mach-omap1/serial.c b/arch/arm/mach-omap1/serial.c index 5615fb8a3d5..976edfb882e 100644 --- a/arch/arm/mach-omap1/serial.c +++ b/arch/arm/mach-omap1/serial.c @@ -253,7 +253,7 @@ static void __init omap_serial_set_port_wakeup(int gpio_nr) } omap_set_gpio_direction(gpio_nr, 1); ret = request_irq(OMAP_GPIO_IRQ(gpio_nr), &omap_serial_wake_interrupt, - SA_TRIGGER_RISING, "serial wakeup", NULL); + IRQF_TRIGGER_RISING, "serial wakeup", NULL); if (ret) { omap_free_gpio(gpio_nr); printk(KERN_ERR "No interrupt for UART wake GPIO: %i\n", diff --git a/arch/arm/mach-omap1/time.c b/arch/arm/mach-omap1/time.c index a01f0efdae1..4d91b9f5108 100644 --- a/arch/arm/mach-omap1/time.c +++ b/arch/arm/mach-omap1/time.c @@ -177,7 +177,7 @@ static irqreturn_t omap_mpu_timer_interrupt(int irq, void *dev_id, static struct irqaction omap_mpu_timer_irq = { .name = "mpu timer", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = omap_mpu_timer_interrupt, }; @@ -191,7 +191,7 @@ static irqreturn_t omap_mpu_timer1_interrupt(int irq, void *dev_id, static struct irqaction omap_mpu_timer1_irq = { .name = "mpu timer1 overflow", - .flags = SA_INTERRUPT, + .flags = IRQF_DISABLED, .handler = omap_mpu_timer1_interrupt, }; diff --git a/arch/arm/mach-omap2/board-apollon.c b/arch/arm/mach-omap2/board-apollon.c index 6c6ba172cdf..7993b7bae2b 100644 --- a/arch/arm/mach-omap2/board-apollon.c +++ b/arch/arm/mach-omap2/board-apollon.c @@ -234,17 +234,17 @@ static void __init apollon_sw_init(void) set_irq_type(OMAP_GPIO_IRQ(SW_ENTER_GPIO16), IRQT_RISING); if (request_irq(OMAP_GPIO_IRQ(SW_ENTER_GPIO16), &apollon_sw_interrupt, - SA_SHIRQ, "enter sw", + IRQF_SHARED, "enter sw", &apollon_sw_interrupt)) return; set_irq_type(OMAP_GPIO_IRQ(SW_UP_GPIO17), IRQT_RISING); if (request_irq(OMAP_GPIO_IRQ(SW_UP_GPIO17), &apollon_sw_interrupt, - SA_SHIRQ, "up sw", + IRQF_SHARED, "up sw", &apollon_sw_interrupt)) return; set_irq_type(OMAP_GPIO_IRQ(SW_DOWN_GPIO58), IRQT_RISING); if (request_irq(OMAP_GPIO_IRQ(SW_DOWN_GPIO58), &apollon_sw_interrupt, - SA_SHIRQ, "down sw", + IRQF_SHARED, "down sw", &apollon_sw_interrupt)) return; } diff --git a/arch/arm/mach-omap2/timer-gp.c b/arch/arm/mach-omap2/timer-gp.c index cf78e6c5a27..fe5fd6d42de 100644 --- a/arch/arm/mach-omap2/timer-gp.c +++ b/arch/arm/mach-omap2/timer-gp.c @@ -52,7 +52,7 @@ static irqreturn_t omap2_gp_timer_interrupt(int irq, void *dev_id, static struct irqaction omap2_gp_timer_irq = { .name = "gp timer", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = omap2_gp_timer_interrupt, }; diff --git a/arch/arm/mach-pnx4008/time.c b/arch/arm/mach-pnx4008/time.c index 4ce68069852..888bf6cfba8 100644 --- a/arch/arm/mach-pnx4008/time.c +++ b/arch/arm/mach-pnx4008/time.c @@ -86,7 +86,7 @@ static irqreturn_t pnx4008_timer_interrupt(int irq, void *dev_id, static struct irqaction pnx4008_timer_irq = { .name = "PNX4008 Tick Timer", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = pnx4008_timer_interrupt }; diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c index bf6648a8390..cce26576999 100644 --- a/arch/arm/mach-pxa/corgi.c +++ b/arch/arm/mach-pxa/corgi.c @@ -225,7 +225,7 @@ static int corgi_mci_init(struct device *dev, irqreturn_t (*corgi_detect_int)(in corgi_mci_platform_data.detect_delay = msecs_to_jiffies(250); err = request_irq(CORGI_IRQ_GPIO_nSD_DETECT, corgi_detect_int, - SA_INTERRUPT | SA_TRIGGER_RISING | SA_TRIGGER_FALLING, + IRQF_DISABLED | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "MMC card detect", data); if (err) { printk(KERN_ERR "corgi_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); diff --git a/arch/arm/mach-pxa/lubbock.c b/arch/arm/mach-pxa/lubbock.c index 1ab26c6914f..6a9a669d60d 100644 --- a/arch/arm/mach-pxa/lubbock.c +++ b/arch/arm/mach-pxa/lubbock.c @@ -419,7 +419,7 @@ static int lubbock_mci_init(struct device *dev, init_timer(&mmc_timer); mmc_timer.data = (unsigned long) data; return request_irq(LUBBOCK_SD_IRQ, lubbock_detect_int, - SA_SAMPLE_RANDOM, "lubbock-sd-detect", data); + IRQF_SAMPLE_RANDOM, "lubbock-sd-detect", data); } static int lubbock_mci_get_ro(struct device *dev) diff --git a/arch/arm/mach-pxa/mainstone.c b/arch/arm/mach-pxa/mainstone.c index b307f11951d..21ddf3de2f6 100644 --- a/arch/arm/mach-pxa/mainstone.c +++ b/arch/arm/mach-pxa/mainstone.c @@ -331,7 +331,7 @@ static int mainstone_mci_init(struct device *dev, irqreturn_t (*mstone_detect_in */ MST_MSCWR1 &= ~MST_MSCWR1_MS_SEL; - err = request_irq(MAINSTONE_MMC_IRQ, mstone_detect_int, SA_INTERRUPT, + err = request_irq(MAINSTONE_MMC_IRQ, mstone_detect_int, IRQF_DISABLED, "MMC card detect", data); if (err) { printk(KERN_ERR "mainstone_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); diff --git a/arch/arm/mach-pxa/poodle.c b/arch/arm/mach-pxa/poodle.c index 9a9fa87cea9..6dbff6d9480 100644 --- a/arch/arm/mach-pxa/poodle.c +++ b/arch/arm/mach-pxa/poodle.c @@ -212,7 +212,7 @@ static int poodle_mci_init(struct device *dev, irqreturn_t (*poodle_detect_int)( poodle_mci_platform_data.detect_delay = msecs_to_jiffies(250); err = request_irq(POODLE_IRQ_GPIO_nSD_DETECT, poodle_detect_int, - SA_INTERRUPT | SA_TRIGGER_RISING | SA_TRIGGER_FALLING, + IRQF_DISABLED | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "MMC card detect", data); if (err) { printk(KERN_ERR "poodle_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c index 1e5e6ca693e..db6e8f56a75 100644 --- a/arch/arm/mach-pxa/sharpsl_pm.c +++ b/arch/arm/mach-pxa/sharpsl_pm.c @@ -142,18 +142,18 @@ void sharpsl_pm_pxa_init(void) pxa_gpio_mode(sharpsl_pm.machinfo->gpio_batlock | GPIO_IN); /* Register interrupt handlers */ - if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_acin), sharpsl_ac_isr, SA_INTERRUPT, "AC Input Detect", sharpsl_ac_isr)) { + if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_acin), sharpsl_ac_isr, IRQF_DISABLED, "AC Input Detect", sharpsl_ac_isr)) { dev_err(sharpsl_pm.dev, "Could not get irq %d.\n", IRQ_GPIO(sharpsl_pm.machinfo->gpio_acin)); } else set_irq_type(IRQ_GPIO(sharpsl_pm.machinfo->gpio_acin),IRQT_BOTHEDGE); - if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batlock), sharpsl_fatal_isr, SA_INTERRUPT, "Battery Cover", sharpsl_fatal_isr)) { + if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batlock), sharpsl_fatal_isr, IRQF_DISABLED, "Battery Cover", sharpsl_fatal_isr)) { dev_err(sharpsl_pm.dev, "Could not get irq %d.\n", IRQ_GPIO(sharpsl_pm.machinfo->gpio_batlock)); } else set_irq_type(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batlock),IRQT_FALLING); if (sharpsl_pm.machinfo->gpio_fatal) { - if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_fatal), sharpsl_fatal_isr, SA_INTERRUPT, "Fatal Battery", sharpsl_fatal_isr)) { + if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_fatal), sharpsl_fatal_isr, IRQF_DISABLED, "Fatal Battery", sharpsl_fatal_isr)) { dev_err(sharpsl_pm.dev, "Could not get irq %d.\n", IRQ_GPIO(sharpsl_pm.machinfo->gpio_fatal)); } else set_irq_type(IRQ_GPIO(sharpsl_pm.machinfo->gpio_fatal),IRQT_FALLING); @@ -162,7 +162,7 @@ void sharpsl_pm_pxa_init(void) if (sharpsl_pm.machinfo->batfull_irq) { /* Register interrupt handler. */ - if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batfull), sharpsl_chrg_full_isr, SA_INTERRUPT, "CO", sharpsl_chrg_full_isr)) { + if (request_irq(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batfull), sharpsl_chrg_full_isr, IRQF_DISABLED, "CO", sharpsl_chrg_full_isr)) { dev_err(sharpsl_pm.dev, "Could not get irq %d.\n", IRQ_GPIO(sharpsl_pm.machinfo->gpio_batfull)); } else set_irq_type(IRQ_GPIO(sharpsl_pm.machinfo->gpio_batfull),IRQT_RISING); diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c index eb9937f6f5c..1c32a9310dc 100644 --- a/arch/arm/mach-pxa/spitz.c +++ b/arch/arm/mach-pxa/spitz.c @@ -308,7 +308,7 @@ static int spitz_mci_init(struct device *dev, irqreturn_t (*spitz_detect_int)(in spitz_mci_platform_data.detect_delay = msecs_to_jiffies(250); err = request_irq(SPITZ_IRQ_GPIO_nSD_DETECT, spitz_detect_int, - SA_INTERRUPT | SA_TRIGGER_RISING | SA_TRIGGER_FALLING, + IRQF_DISABLED | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "MMC card detect", data); if (err) { printk(KERN_ERR "spitz_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c index 1ef85fcc6c0..5dbd191c57c 100644 --- a/arch/arm/mach-pxa/time.c +++ b/arch/arm/mach-pxa/time.c @@ -117,7 +117,7 @@ pxa_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction pxa_timer_irq = { .name = "PXA Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = pxa_timer_interrupt, }; diff --git a/arch/arm/mach-pxa/tosa.c b/arch/arm/mach-pxa/tosa.c index 7152bc13680..249353616ab 100644 --- a/arch/arm/mach-pxa/tosa.c +++ b/arch/arm/mach-pxa/tosa.c @@ -185,7 +185,7 @@ static int tosa_mci_init(struct device *dev, irqreturn_t (*tosa_detect_int)(int, tosa_mci_platform_data.detect_delay = msecs_to_jiffies(250); - err = request_irq(TOSA_IRQ_GPIO_nSD_DETECT, tosa_detect_int, SA_INTERRUPT, + err = request_irq(TOSA_IRQ_GPIO_nSD_DETECT, tosa_detect_int, IRQF_DISABLED, "MMC/SD card detect", data); if (err) { printk(KERN_ERR "tosa_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); diff --git a/arch/arm/mach-pxa/trizeps4.c b/arch/arm/mach-pxa/trizeps4.c index 4ffff9e95ec..7c3007df1bd 100644 --- a/arch/arm/mach-pxa/trizeps4.c +++ b/arch/arm/mach-pxa/trizeps4.c @@ -283,7 +283,9 @@ static int trizeps4_mci_init(struct device *dev, irqreturn_t (*mci_detect_int)(i pxa_gpio_mode(GPIO_MMC_DET | GPIO_IN); - err = request_irq(TRIZEPS4_MMC_IRQ, mci_detect_int, SA_INTERRUPT | SA_TRIGGER_RISING, "MMC card detect", data); + err = request_irq(TRIZEPS4_MMC_IRQ, mci_detect_int, + IRQF_DISABLED | IRQF_TRIGGER_RISING, + "MMC card detect", data); if (err) { printk(KERN_ERR "trizeps4_mci_init: MMC/SD: can't request MMC card detect IRQ\n"); return -1; diff --git a/arch/arm/mach-realview/core.c b/arch/arm/mach-realview/core.c index eb09d3859d6..da028697382 100644 --- a/arch/arm/mach-realview/core.c +++ b/arch/arm/mach-realview/core.c @@ -536,7 +536,7 @@ static irqreturn_t realview_timer_interrupt(int irq, void *dev_id, struct pt_reg static struct irqaction realview_timer_irq = { .name = "RealView Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = realview_timer_interrupt, }; diff --git a/arch/arm/mach-rpc/dma.c b/arch/arm/mach-rpc/dma.c index bd86ffba881..ac511d41d4d 100644 --- a/arch/arm/mach-rpc/dma.c +++ b/arch/arm/mach-rpc/dma.c @@ -128,7 +128,7 @@ static irqreturn_t iomd_dma_handle(int irq, void *dev_id, struct pt_regs *regs) static int iomd_request_dma(dmach_t channel, dma_t *dma) { return request_irq(dma->dma_irq, iomd_dma_handle, - SA_INTERRUPT, dma->device_id, dma); + IRQF_DISABLED, dma->device_id, dma); } static void iomd_free_dma(dmach_t channel, dma_t *dma) diff --git a/arch/arm/mach-s3c2410/dma.c b/arch/arm/mach-s3c2410/dma.c index b0aaf432873..094cc52745c 100644 --- a/arch/arm/mach-s3c2410/dma.c +++ b/arch/arm/mach-s3c2410/dma.c @@ -718,7 +718,7 @@ int s3c2410_dma_request(unsigned int channel, s3c2410_dma_client_t *client, pr_debug("dma%d: %s : requesting irq %d\n", channel, __FUNCTION__, chan->irq); - err = request_irq(chan->irq, s3c2410_dma_irq, SA_INTERRUPT, + err = request_irq(chan->irq, s3c2410_dma_irq, IRQF_DISABLED, client->name, (void *)chan); if (err) { diff --git a/arch/arm/mach-s3c2410/time.c b/arch/arm/mach-s3c2410/time.c index c153c49c75d..00d1cfca971 100644 --- a/arch/arm/mach-s3c2410/time.c +++ b/arch/arm/mach-s3c2410/time.c @@ -138,7 +138,7 @@ s3c2410_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction s3c2410_timer_irq = { .name = "S3C2410 Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = s3c2410_timer_interrupt, }; diff --git a/arch/arm/mach-s3c2410/usb-simtec.c b/arch/arm/mach-s3c2410/usb-simtec.c index 495f8c6ffcb..6b22d8f0a00 100644 --- a/arch/arm/mach-s3c2410/usb-simtec.c +++ b/arch/arm/mach-s3c2410/usb-simtec.c @@ -85,8 +85,8 @@ static void usb_simtec_enableoc(struct s3c2410_hcd_info *info, int on) if (on) { ret = request_irq(IRQ_USBOC, usb_simtec_ocirq, - SA_INTERRUPT | SA_TRIGGER_RISING | - SA_TRIGGER_FALLING, + IRQF_DISABLED | IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING, "USB Over-current", info); if (ret != 0) { printk(KERN_ERR "failed to request usb oc irq\n"); diff --git a/arch/arm/mach-sa1100/collie_pm.c b/arch/arm/mach-sa1100/collie_pm.c index 696d7d29c8a..45b1e71f111 100644 --- a/arch/arm/mach-sa1100/collie_pm.c +++ b/arch/arm/mach-sa1100/collie_pm.c @@ -45,12 +45,12 @@ static void collie_charger_init(void) } /* Register interrupt handler. */ - if ((err = request_irq(COLLIE_IRQ_GPIO_AC_IN, sharpsl_ac_isr, SA_INTERRUPT, + if ((err = request_irq(COLLIE_IRQ_GPIO_AC_IN, sharpsl_ac_isr, IRQF_DISABLED, "ACIN", sharpsl_ac_isr))) { printk("Could not get irq %d.\n", COLLIE_IRQ_GPIO_AC_IN); return; } - if ((err = request_irq(COLLIE_IRQ_GPIO_CO, sharpsl_chrg_full_isr, SA_INTERRUPT, + if ((err = request_irq(COLLIE_IRQ_GPIO_CO, sharpsl_chrg_full_isr, IRQF_DISABLED, "CO", sharpsl_chrg_full_isr))) { free_irq(COLLIE_IRQ_GPIO_AC_IN, sharpsl_ac_isr); printk("Could not get irq %d.\n", COLLIE_IRQ_GPIO_CO); diff --git a/arch/arm/mach-sa1100/dma.c b/arch/arm/mach-sa1100/dma.c index be0e4427bec..3c6441d4bc5 100644 --- a/arch/arm/mach-sa1100/dma.c +++ b/arch/arm/mach-sa1100/dma.c @@ -124,7 +124,7 @@ int sa1100_request_dma (dma_device_t device, const char *device_id, i = dma - dma_chan; regs = (dma_regs_t *)&DDAR(i); - err = request_irq(IRQ_DMA0 + i, dma_irq_handler, SA_INTERRUPT, + err = request_irq(IRQ_DMA0 + i, dma_irq_handler, IRQF_DISABLED, device_id, regs); if (err) { printk(KERN_ERR diff --git a/arch/arm/mach-sa1100/h3600.c b/arch/arm/mach-sa1100/h3600.c index e10d661c015..7364478cec1 100644 --- a/arch/arm/mach-sa1100/h3600.c +++ b/arch/arm/mach-sa1100/h3600.c @@ -740,7 +740,7 @@ static void h3800_IRQ_demux(unsigned int irq, struct irqdesc *desc, struct pt_re static struct irqaction h3800_irq = { .name = "h3800_asic", .handler = h3800_IRQ_demux, - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, }; u32 kpio_int_shadow = 0; diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 688b1e109a4..49ae716e16c 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -111,7 +111,7 @@ sa1100_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction sa1100_timer_irq = { .name = "SA11xx Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = sa1100_timer_interrupt, }; diff --git a/arch/arm/mach-shark/core.c b/arch/arm/mach-shark/core.c index 2f2c6e97b7a..1095df34fec 100644 --- a/arch/arm/mach-shark/core.c +++ b/arch/arm/mach-shark/core.c @@ -90,7 +90,7 @@ shark_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs) static struct irqaction shark_timer_irq = { .name = "Shark Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = shark_timer_interrupt, }; diff --git a/arch/arm/mach-versatile/core.c b/arch/arm/mach-versatile/core.c index 08de8490fb6..a432539cc1b 100644 --- a/arch/arm/mach-versatile/core.c +++ b/arch/arm/mach-versatile/core.c @@ -869,7 +869,7 @@ static irqreturn_t versatile_timer_interrupt(int irq, void *dev_id, struct pt_re static struct irqaction versatile_timer_irq = { .name = "Versatile Timer Tick", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = versatile_timer_interrupt, }; diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S index 0e6946ab6e5..c2f0705bfd4 100644 --- a/arch/arm/mm/proc-arm720.S +++ b/arch/arm/mm/proc-arm720.S @@ -41,6 +41,8 @@ #include <asm/procinfo.h> #include <asm/ptrace.h> +#include "proc-macros.S" + /* * Function: arm720_proc_init (void) * : arm720_proc_fin (void) diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S index e812246277c..c878064e9b8 100644 --- a/arch/arm/mm/proc-sa110.S +++ b/arch/arm/mm/proc-sa110.S @@ -23,6 +23,8 @@ #include <asm/pgtable.h> #include <asm/ptrace.h> +#include "proc-macros.S" + /* * the cache line size of the I and D cache */ diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S index ba32cc6296a..b23b66a6155 100644 --- a/arch/arm/mm/proc-sa1100.S +++ b/arch/arm/mm/proc-sa1100.S @@ -27,6 +27,8 @@ #include <asm/pgtable-hwdef.h> #include <asm/pgtable.h> +#include "proc-macros.S" + /* * the cache line size of the I and D cache */ diff --git a/arch/arm/oprofile/op_model_xscale.c b/arch/arm/oprofile/op_model_xscale.c index e0f0b320d76..34fdc733743 100644 --- a/arch/arm/oprofile/op_model_xscale.c +++ b/arch/arm/oprofile/op_model_xscale.c @@ -384,7 +384,7 @@ static int xscale_pmu_start(void) int ret; u32 pmnc = read_pmnc(); - ret = request_irq(XSCALE_PMU_IRQ, xscale_pmu_interrupt, SA_INTERRUPT, + ret = request_irq(XSCALE_PMU_IRQ, xscale_pmu_interrupt, IRQF_DISABLED, "XScale PMU", (void *)results); if (ret < 0) { diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c index c2c05ef8634..9eddc950714 100644 --- a/arch/arm/plat-omap/dma.c +++ b/arch/arm/plat-omap/dma.c @@ -939,7 +939,7 @@ static irqreturn_t omap2_dma_irq_handler(int irq, void *dev_id, static struct irqaction omap24xx_dma_irq = { .name = "DMA", .handler = omap2_dma_irq_handler, - .flags = SA_INTERRUPT + .flags = IRQF_DISABLED }; #else diff --git a/arch/arm/plat-omap/gpio.c b/arch/arm/plat-omap/gpio.c index cb0c21d384c..fec7970e564 100644 --- a/arch/arm/plat-omap/gpio.c +++ b/arch/arm/plat-omap/gpio.c @@ -850,7 +850,8 @@ static void gpio_irq_handler(unsigned int irq, struct irqdesc *desc, /* Don't run the handler if it's already running * or was disabled lazely. */ - if (unlikely((d->disable_depth || d->running))) { + if (unlikely((d->depth || + (d->status & IRQ_INPROGRESS)))) { irq_mask = 1 << (gpio_irq - bank->virtual_irq_start); /* The unmasking will be done by @@ -859,22 +860,22 @@ static void gpio_irq_handler(unsigned int irq, struct irqdesc *desc, * it's already running. */ _enable_gpio_irqbank(bank, irq_mask, 0); - if (!d->disable_depth) { + if (!d->depth) { /* Level triggered interrupts * won't ever be reentered */ BUG_ON(level_mask & irq_mask); - d->pending = 1; + d->status |= IRQ_PENDING; } continue; } - d->running = 1; + desc_handle_irq(gpio_irq, d, regs); - d->running = 0; - if (unlikely(d->pending && !d->disable_depth)) { + + if (unlikely((d->status & IRQ_PENDING) && !d->depth)) { irq_mask = 1 << (gpio_irq - bank->virtual_irq_start); - d->pending = 0; + d->status &= ~IRQ_PENDING; _enable_gpio_irqbank(bank, irq_mask, 1); retrigger |= irq_mask; } diff --git a/arch/arm/plat-omap/pm.c b/arch/arm/plat-omap/pm.c index 1a24e2c1071..04b4102727a 100644 --- a/arch/arm/plat-omap/pm.c +++ b/arch/arm/plat-omap/pm.c @@ -580,7 +580,7 @@ static irqreturn_t omap_wakeup_interrupt(int irq, void * dev, static struct irqaction omap_wakeup_irq = { .name = "peripheral wakeup", - .flags = SA_INTERRUPT, + .flags = IRQF_DISABLED, .handler = omap_wakeup_interrupt }; diff --git a/arch/arm/plat-omap/timer32k.c b/arch/arm/plat-omap/timer32k.c index ddf4360dea7..281ecc7fcdf 100644 --- a/arch/arm/plat-omap/timer32k.c +++ b/arch/arm/plat-omap/timer32k.c @@ -258,7 +258,7 @@ static struct dyn_tick_timer omap_dyn_tick_timer = { static struct irqaction omap_32k_timer_irq = { .name = "32KHz timer", - .flags = SA_INTERRUPT | SA_TIMER, + .flags = IRQF_DISABLED | IRQF_TIMER, .handler = omap_32k_timer_interrupt, }; diff --git a/arch/i386/Kconfig b/arch/i386/Kconfig index 27d8dddbaa4..daa75ce4b77 100644 --- a/arch/i386/Kconfig +++ b/arch/i386/Kconfig @@ -18,6 +18,14 @@ config GENERIC_TIME bool default y +config LOCKDEP_SUPPORT + bool + default y + +config STACKTRACE_SUPPORT + bool + default y + config SEMAPHORE_SLEEPERS bool default y diff --git a/arch/i386/Kconfig.debug b/arch/i386/Kconfig.debug index c92191b1fb6..b31c0802e1c 100644 --- a/arch/i386/Kconfig.debug +++ b/arch/i386/Kconfig.debug @@ -1,5 +1,9 @@ menu "Kernel hacking" +config TRACE_IRQFLAGS_SUPPORT + bool + default y + source "lib/Kconfig.debug" config EARLY_PRINTK @@ -31,15 +35,6 @@ config DEBUG_STACK_USAGE This option will slow down process creation somewhat. -config STACK_BACKTRACE_COLS - int "Stack backtraces per line" if DEBUG_KERNEL - range 1 3 - default 2 - help - Selects how many stack backtrace entries per line to display. - - This can save screen space when displaying traces. - comment "Page alloc debug is incompatible with Software Suspend on i386" depends on DEBUG_KERNEL && SOFTWARE_SUSPEND diff --git a/arch/i386/kernel/Makefile b/arch/i386/kernel/Makefile index cbc1184e947..1b452a1665c 100644 --- a/arch/i386/kernel/Makefile +++ b/arch/i386/kernel/Makefile @@ -9,6 +9,7 @@ obj-y := process.o semaphore.o signal.o entry.o traps.o irq.o \ pci-dma.o i386_ksyms.o i387.o bootflag.o \ quirks.o i8237.o topology.o alternative.o i8253.o tsc.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-y += cpu/ obj-y += acpi/ obj-$(CONFIG_X86_BIOS_REBOOT) += reboot.o diff --git a/arch/i386/kernel/alternative.c b/arch/i386/kernel/alternative.c index 7b421b3a053..28ab8064976 100644 --- a/arch/i386/kernel/alternative.c +++ b/arch/i386/kernel/alternative.c @@ -303,6 +303,16 @@ void alternatives_smp_switch(int smp) struct smp_alt_module *mod; unsigned long flags; +#ifdef CONFIG_LOCKDEP + /* + * A not yet fixed binutils section handling bug prevents + * alternatives-replacement from working reliably, so turn + * it off: + */ + printk("lockdep: not fixing up alternatives.\n"); + return; +#endif + if (no_replacement || smp_alt_once) return; BUG_ON(!smp && (num_online_cpus() > 1)); diff --git a/arch/i386/kernel/entry.S b/arch/i386/kernel/entry.S index 787190c45fd..d9a260f2efb 100644 --- a/arch/i386/kernel/entry.S +++ b/arch/i386/kernel/entry.S @@ -42,6 +42,7 @@ #include <linux/linkage.h> #include <asm/thread_info.h> +#include <asm/irqflags.h> #include <asm/errno.h> #include <asm/segment.h> #include <asm/smp.h> @@ -76,12 +77,21 @@ NT_MASK = 0x00004000 VM_MASK = 0x00020000 #ifdef CONFIG_PREEMPT -#define preempt_stop cli +#define preempt_stop cli; TRACE_IRQS_OFF #else #define preempt_stop #define resume_kernel restore_nocheck #endif +.macro TRACE_IRQS_IRET +#ifdef CONFIG_TRACE_IRQFLAGS + testl $IF_MASK,EFLAGS(%esp) # interrupts off? + jz 1f + TRACE_IRQS_ON +1: +#endif +.endm + #ifdef CONFIG_VM86 #define resume_userspace_sig check_userspace #else @@ -257,6 +267,10 @@ ENTRY(sysenter_entry) CFI_REGISTER esp, ebp movl TSS_sysenter_esp0(%esp),%esp sysenter_past_esp: + /* + * No need to follow this irqs on/off section: the syscall + * disabled irqs and here we enable it straight after entry: + */ sti pushl $(__USER_DS) CFI_ADJUST_CFA_OFFSET 4 @@ -303,6 +317,7 @@ sysenter_past_esp: call *sys_call_table(,%eax,4) movl %eax,EAX(%esp) cli + TRACE_IRQS_OFF movl TI_flags(%ebp), %ecx testw $_TIF_ALLWORK_MASK, %cx jne syscall_exit_work @@ -310,6 +325,7 @@ sysenter_past_esp: movl EIP(%esp), %edx movl OLDESP(%esp), %ecx xorl %ebp,%ebp + TRACE_IRQS_ON sti sysexit CFI_ENDPROC @@ -339,6 +355,7 @@ syscall_exit: cli # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret + TRACE_IRQS_OFF movl TI_flags(%ebp), %ecx testw $_TIF_ALLWORK_MASK, %cx # current->work jne syscall_exit_work @@ -355,12 +372,15 @@ restore_all: CFI_REMEMBER_STATE je ldt_ss # returning to user-space with LDT SS restore_nocheck: + TRACE_IRQS_IRET +restore_nocheck_notrace: RESTORE_REGS addl $4, %esp CFI_ADJUST_CFA_OFFSET -4 1: iret .section .fixup,"ax" iret_exc: + TRACE_IRQS_ON sti pushl $0 # no error code pushl $do_iret_error @@ -386,11 +406,13 @@ ldt_ss: subl $8, %esp # reserve space for switch16 pointer CFI_ADJUST_CFA_OFFSET 8 cli + TRACE_IRQS_OFF movl %esp, %eax /* Set up the 16bit stack frame with switch32 pointer on top, * and a switch16 pointer on top of the current frame. */ call setup_x86_bogus_stack CFI_ADJUST_CFA_OFFSET -8 # frame has moved + TRACE_IRQS_IRET RESTORE_REGS lss 20+4(%esp), %esp # switch to 16bit stack 1: iret @@ -411,6 +433,7 @@ work_resched: cli # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret + TRACE_IRQS_OFF movl TI_flags(%ebp), %ecx andl $_TIF_WORK_MASK, %ecx # is there any work to be done other # than syscall tracing? @@ -462,6 +485,7 @@ syscall_trace_entry: syscall_exit_work: testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl jz work_pending + TRACE_IRQS_ON sti # could let do_syscall_trace() call # schedule() instead movl %esp, %eax @@ -535,9 +559,14 @@ ENTRY(irq_entries_start) vector=vector+1 .endr +/* + * the CPU automatically disables interrupts when executing an IRQ vector, + * so IRQ-flags tracing has to follow that: + */ ALIGN common_interrupt: SAVE_ALL + TRACE_IRQS_OFF movl %esp,%eax call do_IRQ jmp ret_from_intr @@ -549,9 +578,10 @@ ENTRY(name) \ pushl $~(nr); \ CFI_ADJUST_CFA_OFFSET 4; \ SAVE_ALL; \ + TRACE_IRQS_OFF \ movl %esp,%eax; \ call smp_/**/name; \ - jmp ret_from_intr; \ + jmp ret_from_intr; \ CFI_ENDPROC /* The include is where all of the SMP etc. interrupts come from */ @@ -726,7 +756,7 @@ nmi_stack_correct: xorl %edx,%edx # zero error code movl %esp,%eax # pt_regs pointer call do_nmi - jmp restore_all + jmp restore_nocheck_notrace CFI_ENDPROC nmi_stack_fixup: diff --git a/arch/i386/kernel/irq.c b/arch/i386/kernel/irq.c index 16b49170396..6cb529f60dc 100644 --- a/arch/i386/kernel/irq.c +++ b/arch/i386/kernel/irq.c @@ -166,7 +166,7 @@ void irq_ctx_init(int cpu) irqctx->tinfo.task = NULL; irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; - irqctx->tinfo.preempt_count = SOFTIRQ_OFFSET; + irqctx->tinfo.preempt_count = 0; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); softirq_ctx[cpu] = irqctx; @@ -211,6 +211,10 @@ asmlinkage void do_softirq(void) : "0"(isp) : "memory", "cc", "edx", "ecx", "eax" ); + /* + * Shouldnt happen, we returned above if in_interrupt(): + */ + WARN_ON_ONCE(softirq_count()); } local_irq_restore(flags); diff --git a/arch/i386/kernel/nmi.c b/arch/i386/kernel/nmi.c index a76e9314658..2dd928a8464 100644 --- a/arch/i386/kernel/nmi.c +++ b/arch/i386/kernel/nmi.c @@ -107,7 +107,7 @@ int nmi_active; static __init void nmi_cpu_busy(void *data) { volatile int *endflag = data; - local_irq_enable(); + local_irq_enable_in_hardirq(); /* Intentionally don't use cpu_relax here. This is to make sure that the performance counter really ticks, even if there is a simulator or similar that catches the diff --git a/arch/i386/kernel/stacktrace.c b/arch/i386/kernel/stacktrace.c new file mode 100644 index 00000000000..e62a037ab39 --- /dev/null +++ b/arch/i386/kernel/stacktrace.c @@ -0,0 +1,98 @@ +/* + * arch/i386/kernel/stacktrace.c + * + * Stack trace management functions + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + */ +#include <linux/sched.h> +#include <linux/stacktrace.h> + +static inline int valid_stack_ptr(struct thread_info *tinfo, void *p) +{ + return p > (void *)tinfo && + p < (void *)tinfo + THREAD_SIZE - 3; +} + +/* + * Save stack-backtrace addresses into a stack_trace buffer: + */ +static inline unsigned long +save_context_stack(struct stack_trace *trace, unsigned int skip, + struct thread_info *tinfo, unsigned long *stack, + unsigned long ebp) +{ + unsigned long addr; + +#ifdef CONFIG_FRAME_POINTER + while (valid_stack_ptr(tinfo, (void *)ebp)) { + addr = *(unsigned long *)(ebp + 4); + if (!skip) + trace->entries[trace->nr_entries++] = addr; + else + skip--; + if (trace->nr_entries >= trace->max_entries) + break; + /* + * break out of recursive entries (such as + * end_of_stack_stop_unwind_function): + */ + if (ebp == *(unsigned long *)ebp) + break; + + ebp = *(unsigned long *)ebp; + } +#else + while (valid_stack_ptr(tinfo, stack)) { + addr = *stack++; + if (__kernel_text_address(addr)) { + if (!skip) + trace->entries[trace->nr_entries++] = addr; + else + skip--; + if (trace->nr_entries >= trace->max_entries) + break; + } + } +#endif + + return ebp; +} + +/* + * Save stack-backtrace addresses into a stack_trace buffer. + * If all_contexts is set, all contexts (hardirq, softirq and process) + * are saved. If not set then only the current context is saved. + */ +void save_stack_trace(struct stack_trace *trace, + struct task_struct *task, int all_contexts, + unsigned int skip) +{ + unsigned long ebp; + unsigned long *stack = &ebp; + + WARN_ON(trace->nr_entries || !trace->max_entries); + + if (!task || task == current) { + /* Grab ebp right from our regs: */ + asm ("movl %%ebp, %0" : "=r" (ebp)); + } else { + /* ebp is the last reg pushed by switch_to(): */ + ebp = *(unsigned long *) task->thread.esp; + } + + while (1) { + struct thread_info *context = (struct thread_info *) + ((unsigned long)stack & (~(THREAD_SIZE - 1))); + + ebp = save_context_stack(trace, skip, context, stack, ebp); + stack = (unsigned long *)context->previous_esp; + if (!all_contexts || !stack || + trace->nr_entries >= trace->max_entries) + break; + trace->entries[trace->nr_entries++] = ULONG_MAX; + if (trace->nr_entries >= trace->max_entries) + break; + } +} + diff --git a/arch/i386/kernel/traps.c b/arch/i386/kernel/traps.c index e8c6086b2aa..2bf8b55b91f 100644 --- a/arch/i386/kernel/traps.c +++ b/arch/i386/kernel/traps.c @@ -115,28 +115,13 @@ static inline int valid_stack_ptr(struct thread_info *tinfo, void *p) } /* - * Print CONFIG_STACK_BACKTRACE_COLS address/symbol entries per line. + * Print one address/symbol entries per line. */ -static inline int print_addr_and_symbol(unsigned long addr, char *log_lvl, - int printed) +static inline void print_addr_and_symbol(unsigned long addr, char *log_lvl) { - if (!printed) - printk(log_lvl); - -#if CONFIG_STACK_BACKTRACE_COLS == 1 printk(" [<%08lx>] ", addr); -#else - printk(" <%08lx> ", addr); -#endif - print_symbol("%s", addr); - printed = (printed + 1) % CONFIG_STACK_BACKTRACE_COLS; - if (printed) - printk(" "); - else - printk("\n"); - - return printed; + print_symbol("%s\n", addr); } static inline unsigned long print_context_stack(struct thread_info *tinfo, @@ -144,12 +129,11 @@ static inline unsigned long print_context_stack(struct thread_info *tinfo, char *log_lvl) { unsigned long addr; - int printed = 0; /* nr of entries already printed on current line */ #ifdef CONFIG_FRAME_POINTER while (valid_stack_ptr(tinfo, (void *)ebp)) { addr = *(unsigned long *)(ebp + 4); - printed = print_addr_and_symbol(addr, log_lvl, printed); + print_addr_and_symbol(addr, log_lvl); /* * break out of recursive entries (such as * end_of_stack_stop_unwind_function): @@ -162,28 +146,23 @@ static inline unsigned long print_context_stack(struct thread_info *tinfo, while (valid_stack_ptr(tinfo, stack)) { addr = *stack++; if (__kernel_text_address(addr)) - printed = print_addr_and_symbol(addr, log_lvl, printed); + print_addr_and_symbol(addr, log_lvl); } #endif - if (printed) - printk("\n"); - return ebp; } -static asmlinkage int show_trace_unwind(struct unwind_frame_info *info, void *log_lvl) +static asmlinkage int +show_trace_unwind(struct unwind_frame_info *info, void *log_lvl) { int n = 0; - int printed = 0; /* nr of entries already printed on current line */ while (unwind(info) == 0 && UNW_PC(info)) { - ++n; - printed = print_addr_and_symbol(UNW_PC(info), log_lvl, printed); + n++; + print_addr_and_symbol(UNW_PC(info), log_lvl); if (arch_unw_user_mode(info)) break; } - if (printed) - printk("\n"); return n; } diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index eb8e8dc5ac8..2fbe4536fe1 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -678,7 +678,7 @@ copy_reg(const u64 *fr, u64 fnat, u64 *tr, u64 *tnat) */ static void -ia64_mca_modify_comm(const task_t *previous_current) +ia64_mca_modify_comm(const struct task_struct *previous_current) { char *p, comm[sizeof(current->comm)]; if (previous_current->pid) @@ -709,7 +709,7 @@ ia64_mca_modify_comm(const task_t *previous_current) * that we can do backtrace on the MCA/INIT handler code itself. */ -static task_t * +static struct task_struct * ia64_mca_modify_original_stack(struct pt_regs *regs, const struct switch_stack *sw, struct ia64_sal_os_state *sos, @@ -719,7 +719,7 @@ ia64_mca_modify_original_stack(struct pt_regs *regs, ia64_va va; extern char ia64_leave_kernel[]; /* Need asm address, not function descriptor */ const pal_min_state_area_t *ms = sos->pal_min_state; - task_t *previous_current; + struct task_struct *previous_current; struct pt_regs *old_regs; struct switch_stack *old_sw; unsigned size = sizeof(struct pt_regs) + @@ -1023,7 +1023,7 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw, pal_processor_state_info_t *psp = (pal_processor_state_info_t *) &sos->proc_state_param; int recover, cpu = smp_processor_id(); - task_t *previous_current; + struct task_struct *previous_current; struct ia64_mca_notify_die nd = { .sos = sos, .monarch_cpu = &monarch_cpu }; @@ -1352,7 +1352,7 @@ ia64_init_handler(struct pt_regs *regs, struct switch_stack *sw, { static atomic_t slaves; static atomic_t monarchs; - task_t *previous_current; + struct task_struct *previous_current; int cpu = smp_processor_id(); struct ia64_mca_notify_die nd = { .sos = sos, .monarch_cpu = &monarch_cpu }; diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index e1960979be2..6203ed4ec8c 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -124,7 +124,7 @@ extern void __devinit calibrate_delay (void); extern void start_ap (void); extern unsigned long ia64_iobase; -task_t *task_for_booting_cpu; +struct task_struct *task_for_booting_cpu; /* * State for each CPU diff --git a/arch/mips/kernel/entry.S b/arch/mips/kernel/entry.S index ecfd637d702..01e7fa86aa4 100644 --- a/arch/mips/kernel/entry.S +++ b/arch/mips/kernel/entry.S @@ -65,7 +65,7 @@ need_resched: #endif FEXPORT(ret_from_fork) - jal schedule_tail # a0 = task_t *prev + jal schedule_tail # a0 = struct task_struct *prev FEXPORT(syscall_exit) local_irq_disable # make sure need_resched and diff --git a/arch/mips/kernel/mips-mt.c b/arch/mips/kernel/mips-mt.c index 02237a685ec..4dcc39f4295 100644 --- a/arch/mips/kernel/mips-mt.c +++ b/arch/mips/kernel/mips-mt.c @@ -47,7 +47,7 @@ unsigned long mt_fpemul_threshold = 0; * used in sys_sched_set/getaffinity() in kernel/sched.c, so * cloned here. */ -static inline task_t *find_process_by_pid(pid_t pid) +static inline struct task_struct *find_process_by_pid(pid_t pid) { return pid ? find_task_by_pid(pid) : current; } @@ -62,7 +62,7 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, cpumask_t new_mask; cpumask_t effective_mask; int retval; - task_t *p; + struct task_struct *p; if (len < sizeof(new_mask)) return -EINVAL; @@ -127,7 +127,7 @@ asmlinkage long mipsmt_sys_sched_getaffinity(pid_t pid, unsigned int len, unsigned int real_len; cpumask_t mask; int retval; - task_t *p; + struct task_struct *p; real_len = sizeof(mask); if (len < real_len) diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 05a700940f6..8cf987809c6 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -306,13 +306,8 @@ void do_softirq(void) local_irq_save(flags); - if (local_softirq_pending()) { - account_system_vtime(current); - local_bh_disable(); + if (local_softirq_pending()) do_softirq_onstack(); - account_system_vtime(current); - __local_bh_enable(); - } local_irq_restore(flags); } diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 821a141889d..224fbff7996 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -7,6 +7,14 @@ config MMU bool default y +config LOCKDEP_SUPPORT + bool + default y + +config STACKTRACE_SUPPORT + bool + default y + config RWSEM_GENERIC_SPINLOCK bool diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug index f53b6d5300e..2283933a9a9 100644 --- a/arch/s390/Kconfig.debug +++ b/arch/s390/Kconfig.debug @@ -1,5 +1,9 @@ menu "Kernel hacking" +config TRACE_IRQFLAGS_SUPPORT + bool + default y + source "lib/Kconfig.debug" endmenu diff --git a/arch/s390/Makefile b/arch/s390/Makefile index b3791fb094a..74ef57dcfa6 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -34,6 +34,11 @@ cflags-$(CONFIG_MARCH_G5) += $(call cc-option,-march=g5) cflags-$(CONFIG_MARCH_Z900) += $(call cc-option,-march=z900) cflags-$(CONFIG_MARCH_Z990) += $(call cc-option,-march=z990) +# +# Prevent tail-call optimizations, to get clearer backtraces: +# +cflags-$(CONFIG_FRAME_POINTER) += -fno-optimize-sibling-calls + # old style option for packed stacks ifeq ($(call cc-option-yn,-mkernel-backchain),y) cflags-$(CONFIG_PACK_STACK) += -mkernel-backchain -D__PACK_STACK diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 9269b5788fa..eabf00a6f77 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -21,6 +21,7 @@ obj-$(CONFIG_COMPAT) += compat_linux.o compat_signal.o \ obj-$(CONFIG_BINFMT_ELF32) += binfmt_elf32.o obj-$(CONFIG_VIRT_TIMER) += vtime.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o # Kexec part S390_KEXEC_OBJS := machine_kexec.o crash.o diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index d8948c342ca..5b5799ac8f8 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -58,6 +58,21 @@ STACK_SIZE = 1 << STACK_SHIFT #define BASED(name) name-system_call(%r13) +#ifdef CONFIG_TRACE_IRQFLAGS + .macro TRACE_IRQS_ON + l %r1,BASED(.Ltrace_irq_on) + basr %r14,%r1 + .endm + + .macro TRACE_IRQS_OFF + l %r1,BASED(.Ltrace_irq_off) + basr %r14,%r1 + .endm +#else +#define TRACE_IRQS_ON +#define TRACE_IRQS_OFF +#endif + /* * Register usage in interrupt handlers: * R9 - pointer to current task structure @@ -361,6 +376,7 @@ ret_from_fork: st %r15,SP_R15(%r15) # store stack pointer for new kthread 0: l %r1,BASED(.Lschedtail) basr %r14,%r1 + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts b BASED(sysc_return) @@ -516,6 +532,7 @@ pgm_no_vtime3: mvc __THREAD_per+__PER_address(4,%r1),__LC_PER_ADDRESS mvc __THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID oi __TI_flags+3(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts b BASED(sysc_do_svc) @@ -539,9 +556,11 @@ io_int_handler: io_no_vtime: #endif l %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF l %r1,BASED(.Ldo_IRQ) # load address of do_IRQ la %r2,SP_PTREGS(%r15) # address of register-save area basr %r14,%r1 # branch to standard irq handler + TRACE_IRQS_ON io_return: tm SP_PSW+1(%r15),0x01 # returning to user ? @@ -651,10 +670,12 @@ ext_int_handler: ext_no_vtime: #endif l %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF la %r2,SP_PTREGS(%r15) # address of register-save area lh %r3,__LC_EXT_INT_CODE # get interruption code l %r1,BASED(.Ldo_extint) basr %r14,%r1 + TRACE_IRQS_ON b BASED(io_return) __critical_end: @@ -731,8 +752,10 @@ mcck_no_vtime: stosm __SF_EMPTY(%r15),0x04 # turn dat on tm __TI_flags+3(%r9),_TIF_MCCK_PENDING bno BASED(mcck_return) + TRACE_IRQS_OFF l %r1,BASED(.Ls390_handle_mcck) basr %r14,%r1 # call machine check handler + TRACE_IRQS_ON mcck_return: mvc __LC_RETURN_MCCK_PSW(8),SP_PSW(%r15) # move return PSW ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit @@ -1012,7 +1035,11 @@ cleanup_io_leave_insn: .Lvfork: .long sys_vfork .Lschedtail: .long schedule_tail .Lsysc_table: .long sys_call_table - +#ifdef CONFIG_TRACE_IRQFLAGS +.Ltrace_irq_on:.long trace_hardirqs_on +.Ltrace_irq_off: + .long trace_hardirqs_off +#endif .Lcritical_start: .long __critical_start + 0x80000000 .Lcritical_end: diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 1ca499fa54b..56f5f613b86 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -58,6 +58,19 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \ #define BASED(name) name-system_call(%r13) +#ifdef CONFIG_TRACE_IRQFLAGS + .macro TRACE_IRQS_ON + brasl %r14,trace_hardirqs_on + .endm + + .macro TRACE_IRQS_OFF + brasl %r14,trace_hardirqs_off + .endm +#else +#define TRACE_IRQS_ON +#define TRACE_IRQS_OFF +#endif + .macro STORE_TIMER lc_offset #ifdef CONFIG_VIRT_CPU_ACCOUNTING stpt \lc_offset @@ -354,6 +367,7 @@ ret_from_fork: jo 0f stg %r15,SP_R15(%r15) # store stack pointer for new kthread 0: brasl %r14,schedule_tail + TRACE_IRQS_ON stosm 24(%r15),0x03 # reenable interrupts j sysc_return @@ -535,6 +549,7 @@ pgm_no_vtime3: mvc __THREAD_per+__PER_address(8,%r1),__LC_PER_ADDRESS mvc __THREAD_per+__PER_access_id(1,%r1),__LC_PER_ACCESS_ID oi __TI_flags+7(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts j sysc_do_svc @@ -557,8 +572,10 @@ io_int_handler: io_no_vtime: #endif lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF la %r2,SP_PTREGS(%r15) # address of register-save area brasl %r14,do_IRQ # call standard irq handler + TRACE_IRQS_ON io_return: tm SP_PSW+1(%r15),0x01 # returning to user ? @@ -665,9 +682,11 @@ ext_int_handler: ext_no_vtime: #endif lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF la %r2,SP_PTREGS(%r15) # address of register-save area llgh %r3,__LC_EXT_INT_CODE # get interruption code brasl %r14,do_extint + TRACE_IRQS_ON j io_return __critical_end: @@ -743,7 +762,9 @@ mcck_no_vtime: stosm __SF_EMPTY(%r15),0x04 # turn dat on tm __TI_flags+7(%r9),_TIF_MCCK_PENDING jno mcck_return + TRACE_IRQS_OFF brasl %r14,s390_handle_mcck + TRACE_IRQS_ON mcck_return: mvc __LC_RETURN_MCCK_PSW(16),SP_PSW(%r15) # move return PSW ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 480b6a5fef3..1eef5091861 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -69,10 +69,6 @@ asmlinkage void do_softirq(void) local_irq_save(flags); - account_system_vtime(current); - - local_bh_disable(); - if (local_softirq_pending()) { /* Get current stack pointer. */ asm volatile("la %0,0(15)" : "=a" (old)); @@ -95,10 +91,6 @@ asmlinkage void do_softirq(void) __do_softirq(); } - account_system_vtime(current); - - __local_bh_enable(); - local_irq_restore(flags); } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 1f939919179..78c8e5548ca 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -142,6 +142,7 @@ static void default_idle(void) return; } + trace_hardirqs_on(); /* Wait for external, I/O or machine check interrupt. */ __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT); diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c new file mode 100644 index 00000000000..de83f38288d --- /dev/null +++ b/arch/s390/kernel/stacktrace.c @@ -0,0 +1,90 @@ +/* + * arch/s390/kernel/stacktrace.c + * + * Stack trace management functions + * + * Copyright (C) IBM Corp. 2006 + * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> + */ + +#include <linux/sched.h> +#include <linux/stacktrace.h> +#include <linux/kallsyms.h> + +static inline unsigned long save_context_stack(struct stack_trace *trace, + unsigned int *skip, + unsigned long sp, + unsigned long low, + unsigned long high) +{ + struct stack_frame *sf; + struct pt_regs *regs; + unsigned long addr; + + while(1) { + sp &= PSW_ADDR_INSN; + if (sp < low || sp > high) + return sp; + sf = (struct stack_frame *)sp; + while(1) { + addr = sf->gprs[8] & PSW_ADDR_INSN; + if (!(*skip)) + trace->entries[trace->nr_entries++] = addr; + else + (*skip)--; + if (trace->nr_entries >= trace->max_entries) + return sp; + low = sp; + sp = sf->back_chain & PSW_ADDR_INSN; + if (!sp) + break; + if (sp <= low || sp > high - sizeof(*sf)) + return sp; + sf = (struct stack_frame *)sp; + } + /* Zero backchain detected, check for interrupt frame. */ + sp = (unsigned long)(sf + 1); + if (sp <= low || sp > high - sizeof(*regs)) + return sp; + regs = (struct pt_regs *)sp; + addr = regs->psw.addr & PSW_ADDR_INSN; + if (!(*skip)) + trace->entries[trace->nr_entries++] = addr; + else + (*skip)--; + if (trace->nr_entries >= trace->max_entries) + return sp; + low = sp; + sp = regs->gprs[15]; + } +} + +void save_stack_trace(struct stack_trace *trace, + struct task_struct *task, int all_contexts, + unsigned int skip) +{ + register unsigned long sp asm ("15"); + unsigned long orig_sp; + + sp &= PSW_ADDR_INSN; + orig_sp = sp; + + sp = save_context_stack(trace, &skip, sp, + S390_lowcore.panic_stack - PAGE_SIZE, + S390_lowcore.panic_stack); + if ((sp != orig_sp) && !all_contexts) + return; + sp = save_context_stack(trace, &skip, sp, + S390_lowcore.async_stack - ASYNC_SIZE, + S390_lowcore.async_stack); + if ((sp != orig_sp) && !all_contexts) + return; + if (task) + save_context_stack(trace, &skip, sp, + (unsigned long) task_stack_page(task), + (unsigned long) task_stack_page(task) + THREAD_SIZE); + else + save_context_stack(trace, &skip, sp, S390_lowcore.thread_info, + S390_lowcore.thread_info + THREAD_SIZE); + return; +} diff --git a/arch/um/kernel/tt/process_kern.c b/arch/um/kernel/tt/process_kern.c index a9c1443fc54..8368c2dbe63 100644 --- a/arch/um/kernel/tt/process_kern.c +++ b/arch/um/kernel/tt/process_kern.c @@ -119,7 +119,7 @@ void suspend_new_thread(int fd) panic("read failed in suspend_new_thread, err = %d", -err); } -void schedule_tail(task_t *prev); +void schedule_tail(struct task_struct *prev); static void new_thread_handler(int sig) { diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 7d51dd7201c..37cfe7701f0 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -495,6 +495,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end) { } +#ifdef CONFIG_SMP void alternatives_smp_module_add(struct module *mod, char *name, void *locks, void *locks_end, void *text, void *text_end) @@ -504,3 +505,4 @@ void alternatives_smp_module_add(struct module *mod, char *name, void alternatives_smp_module_del(struct module *mod) { } +#endif diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index e856804c447..28df7d88ce2 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -24,6 +24,14 @@ config X86 bool default y +config LOCKDEP_SUPPORT + bool + default y + +config STACKTRACE_SUPPORT + bool + default y + config SEMAPHORE_SLEEPERS bool default y diff --git a/arch/x86_64/Kconfig.debug b/arch/x86_64/Kconfig.debug index 1d92ab56c0f..775d211a5cf 100644 --- a/arch/x86_64/Kconfig.debug +++ b/arch/x86_64/Kconfig.debug @@ -1,5 +1,9 @@ menu "Kernel hacking" +config TRACE_IRQFLAGS_SUPPORT + bool + default y + source "lib/Kconfig.debug" config DEBUG_RODATA diff --git a/arch/x86_64/ia32/ia32entry.S b/arch/x86_64/ia32/ia32entry.S index c536fa98ea3..9b5bb413a6e 100644 --- a/arch/x86_64/ia32/ia32entry.S +++ b/arch/x86_64/ia32/ia32entry.S @@ -13,6 +13,7 @@ #include <asm/thread_info.h> #include <asm/segment.h> #include <asm/vsyscall32.h> +#include <asm/irqflags.h> #include <linux/linkage.h> #define IA32_NR_syscalls ((ia32_syscall_end - ia32_sys_call_table)/8) @@ -75,6 +76,10 @@ ENTRY(ia32_sysenter_target) swapgs movq %gs:pda_kernelstack, %rsp addq $(PDA_STACKOFFSET),%rsp + /* + * No need to follow this irqs on/off section: the syscall + * disabled irqs, here we enable it straight after entry: + */ sti movl %ebp,%ebp /* zero extension */ pushq $__USER32_DS @@ -118,6 +123,7 @@ sysenter_do_call: movq %rax,RAX-ARGOFFSET(%rsp) GET_THREAD_INFO(%r10) cli + TRACE_IRQS_OFF testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) jnz int_ret_from_sys_call andl $~TS_COMPAT,threadinfo_status(%r10) @@ -132,6 +138,7 @@ sysenter_do_call: CFI_REGISTER rsp,rcx movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */ CFI_REGISTER rip,rdx + TRACE_IRQS_ON swapgs sti /* sti only takes effect after the next instruction */ /* sysexit */ @@ -186,6 +193,10 @@ ENTRY(ia32_cstar_target) movl %esp,%r8d CFI_REGISTER rsp,r8 movq %gs:pda_kernelstack,%rsp + /* + * No need to follow this irqs on/off section: the syscall + * disabled irqs and here we enable it straight after entry: + */ sti SAVE_ARGS 8,1,1 movl %eax,%eax /* zero extension */ @@ -220,6 +231,7 @@ cstar_do_call: movq %rax,RAX-ARGOFFSET(%rsp) GET_THREAD_INFO(%r10) cli + TRACE_IRQS_OFF testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) jnz int_ret_from_sys_call andl $~TS_COMPAT,threadinfo_status(%r10) @@ -228,6 +240,7 @@ cstar_do_call: CFI_REGISTER rip,rcx movl EFLAGS-ARGOFFSET(%rsp),%r11d /*CFI_REGISTER rflags,r11*/ + TRACE_IRQS_ON movl RSP-ARGOFFSET(%rsp),%esp CFI_RESTORE rsp swapgs @@ -286,7 +299,11 @@ ENTRY(ia32_syscall) /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/ /*CFI_REL_OFFSET cs,CS-RIP*/ CFI_REL_OFFSET rip,RIP-RIP - swapgs + swapgs + /* + * No need to follow this irqs on/off section: the syscall + * disabled irqs and here we enable it straight after entry: + */ sti movl %eax,%eax pushq %rax diff --git a/arch/x86_64/kernel/Makefile b/arch/x86_64/kernel/Makefile index 819e84ec5b6..b5aaeafc1cd 100644 --- a/arch/x86_64/kernel/Makefile +++ b/arch/x86_64/kernel/Makefile @@ -10,6 +10,7 @@ obj-y := process.o signal.o entry.o traps.o irq.o \ setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \ pci-dma.o pci-nommu.o alternative.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_X86_MCE) += mce.o obj-$(CONFIG_X86_MCE_INTEL) += mce_intel.o obj-$(CONFIG_X86_MCE_AMD) += mce_amd.o diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S index ed92c298360..d464dded68c 100644 --- a/arch/x86_64/kernel/entry.S +++ b/arch/x86_64/kernel/entry.S @@ -41,13 +41,24 @@ #include <asm/thread_info.h> #include <asm/hw_irq.h> #include <asm/page.h> +#include <asm/irqflags.h> .code64 #ifndef CONFIG_PREEMPT #define retint_kernel retint_restore_args #endif - + + +.macro TRACE_IRQS_IRETQ offset=ARGOFFSET +#ifdef CONFIG_TRACE_IRQFLAGS + bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ + jnc 1f + TRACE_IRQS_ON +1: +#endif +.endm + /* * C code is not supposed to know about undefined top of stack. Every time * a C function with an pt_regs argument is called from the SYSCALL based @@ -194,6 +205,10 @@ ENTRY(system_call) swapgs movq %rsp,%gs:pda_oldrsp movq %gs:pda_kernelstack,%rsp + /* + * No need to follow this irqs off/on section - it's straight + * and short: + */ sti SAVE_ARGS 8,1 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) @@ -219,10 +234,15 @@ ret_from_sys_call: sysret_check: GET_THREAD_INFO(%rcx) cli + TRACE_IRQS_OFF movl threadinfo_flags(%rcx),%edx andl %edi,%edx CFI_REMEMBER_STATE jnz sysret_careful + /* + * sysretq will re-enable interrupts: + */ + TRACE_IRQS_ON movq RIP-ARGOFFSET(%rsp),%rcx CFI_REGISTER rip,rcx RESTORE_ARGS 0,-ARG_SKIP,1 @@ -237,6 +257,7 @@ sysret_careful: CFI_RESTORE_STATE bt $TIF_NEED_RESCHED,%edx jnc sysret_signal + TRACE_IRQS_ON sti pushq %rdi CFI_ADJUST_CFA_OFFSET 8 @@ -247,6 +268,7 @@ sysret_careful: /* Handle a signal */ sysret_signal: + TRACE_IRQS_ON sti testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx jz 1f @@ -261,6 +283,7 @@ sysret_signal: /* Use IRET because user could have changed frame. This works because ptregscall_common has called FIXUP_TOP_OF_STACK. */ cli + TRACE_IRQS_OFF jmp int_with_check badsys: @@ -309,6 +332,7 @@ ENTRY(int_ret_from_sys_call) CFI_REL_OFFSET r10,R10-ARGOFFSET CFI_REL_OFFSET r11,R11-ARGOFFSET cli + TRACE_IRQS_OFF testl $3,CS-ARGOFFSET(%rsp) je retint_restore_args movl $_TIF_ALLWORK_MASK,%edi @@ -327,6 +351,7 @@ int_with_check: int_careful: bt $TIF_NEED_RESCHED,%edx jnc int_very_careful + TRACE_IRQS_ON sti pushq %rdi CFI_ADJUST_CFA_OFFSET 8 @@ -334,10 +359,12 @@ int_careful: popq %rdi CFI_ADJUST_CFA_OFFSET -8 cli + TRACE_IRQS_OFF jmp int_with_check /* handle signals and tracing -- both require a full stack frame */ int_very_careful: + TRACE_IRQS_ON sti SAVE_REST /* Check for syscall exit trace */ @@ -351,6 +378,7 @@ int_very_careful: CFI_ADJUST_CFA_OFFSET -8 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi cli + TRACE_IRQS_OFF jmp int_restore_rest int_signal: @@ -363,6 +391,7 @@ int_signal: int_restore_rest: RESTORE_REST cli + TRACE_IRQS_OFF jmp int_with_check CFI_ENDPROC END(int_ret_from_sys_call) @@ -484,6 +513,10 @@ END(stub_rt_sigreturn) swapgs 1: incl %gs:pda_irqcount # RED-PEN should check preempt count cmoveq %gs:pda_irqstackptr,%rsp + /* + * We entered an interrupt context - irqs are off: + */ + TRACE_IRQS_OFF call \func .endm @@ -493,6 +526,7 @@ ENTRY(common_interrupt) /* 0(%rsp): oldrsp-ARGOFFSET */ ret_from_intr: cli + TRACE_IRQS_OFF decl %gs:pda_irqcount leaveq CFI_DEF_CFA_REGISTER rsp @@ -515,9 +549,21 @@ retint_check: CFI_REMEMBER_STATE jnz retint_careful retint_swapgs: + /* + * The iretq could re-enable interrupts: + */ + cli + TRACE_IRQS_IRETQ swapgs + jmp restore_args + retint_restore_args: cli + /* + * The iretq could re-enable interrupts: + */ + TRACE_IRQS_IRETQ +restore_args: RESTORE_ARGS 0,8,0 iret_label: iretq @@ -530,6 +576,7 @@ iret_label: /* running with kernel gs */ bad_iret: movq $11,%rdi /* SIGSEGV */ + TRACE_IRQS_ON sti jmp do_exit .previous @@ -539,6 +586,7 @@ retint_careful: CFI_RESTORE_STATE bt $TIF_NEED_RESCHED,%edx jnc retint_signal + TRACE_IRQS_ON sti pushq %rdi CFI_ADJUST_CFA_OFFSET 8 @@ -547,11 +595,13 @@ retint_careful: CFI_ADJUST_CFA_OFFSET -8 GET_THREAD_INFO(%rcx) cli + TRACE_IRQS_OFF jmp retint_check retint_signal: testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx jz retint_swapgs + TRACE_IRQS_ON sti SAVE_REST movq $-1,ORIG_RAX(%rsp) @@ -560,6 +610,7 @@ retint_signal: call do_notify_resume RESTORE_REST cli + TRACE_IRQS_OFF movl $_TIF_NEED_RESCHED,%edi GET_THREAD_INFO(%rcx) jmp retint_check @@ -666,7 +717,7 @@ END(spurious_interrupt) /* error code is on the stack already */ /* handle NMI like exceptions that can happen everywhere */ - .macro paranoidentry sym, ist=0 + .macro paranoidentry sym, ist=0, irqtrace=1 SAVE_ALL cld movl $1,%ebx @@ -691,8 +742,73 @@ END(spurious_interrupt) addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) .endif cli + .if \irqtrace + TRACE_IRQS_OFF + .endif .endm - + + /* + * "Paranoid" exit path from exception stack. + * Paranoid because this is used by NMIs and cannot take + * any kernel state for granted. + * We don't do kernel preemption checks here, because only + * NMI should be common and it does not enable IRQs and + * cannot get reschedule ticks. + * + * "trace" is 0 for the NMI handler only, because irq-tracing + * is fundamentally NMI-unsafe. (we cannot change the soft and + * hard flags at once, atomically) + */ + .macro paranoidexit trace=1 + /* ebx: no swapgs flag */ +paranoid_exit\trace: + testl %ebx,%ebx /* swapgs needed? */ + jnz paranoid_restore\trace + testl $3,CS(%rsp) + jnz paranoid_userspace\trace +paranoid_swapgs\trace: + TRACE_IRQS_IRETQ 0 + swapgs +paranoid_restore\trace: + RESTORE_ALL 8 + iretq +paranoid_userspace\trace: + GET_THREAD_INFO(%rcx) + movl threadinfo_flags(%rcx),%ebx + andl $_TIF_WORK_MASK,%ebx + jz paranoid_swapgs\trace + movq %rsp,%rdi /* &pt_regs */ + call sync_regs + movq %rax,%rsp /* switch stack for scheduling */ + testl $_TIF_NEED_RESCHED,%ebx + jnz paranoid_schedule\trace + movl %ebx,%edx /* arg3: thread flags */ + .if \trace + TRACE_IRQS_ON + .endif + sti + xorl %esi,%esi /* arg2: oldset */ + movq %rsp,%rdi /* arg1: &pt_regs */ + call do_notify_resume + cli + .if \trace + TRACE_IRQS_OFF + .endif + jmp paranoid_userspace\trace +paranoid_schedule\trace: + .if \trace + TRACE_IRQS_ON + .endif + sti + call schedule + cli + .if \trace + TRACE_IRQS_OFF + .endif + jmp paranoid_userspace\trace + CFI_ENDPROC + .endm + /* * Exception entry point. This expects an error code/orig_rax on the stack * and the exception handler in %rax. @@ -748,6 +864,7 @@ error_exit: movl %ebx,%eax RESTORE_REST cli + TRACE_IRQS_OFF GET_THREAD_INFO(%rcx) testl %eax,%eax jne retint_kernel @@ -755,6 +872,10 @@ error_exit: movl $_TIF_WORK_MASK,%edi andl %edi,%edx jnz retint_careful + /* + * The iret might restore flags: + */ + TRACE_IRQS_IRETQ swapgs RESTORE_ARGS 0,8,0 jmp iret_label @@ -916,8 +1037,7 @@ KPROBE_ENTRY(debug) pushq $0 CFI_ADJUST_CFA_OFFSET 8 paranoidentry do_debug, DEBUG_STACK - jmp paranoid_exit - CFI_ENDPROC + paranoidexit END(debug) .previous .text @@ -926,49 +1046,13 @@ KPROBE_ENTRY(nmi) INTR_FRAME pushq $-1 CFI_ADJUST_CFA_OFFSET 8 - paranoidentry do_nmi - /* - * "Paranoid" exit path from exception stack. - * Paranoid because this is used by NMIs and cannot take - * any kernel state for granted. - * We don't do kernel preemption checks here, because only - * NMI should be common and it does not enable IRQs and - * cannot get reschedule ticks. - */ - /* ebx: no swapgs flag */ -paranoid_exit: - testl %ebx,%ebx /* swapgs needed? */ - jnz paranoid_restore - testl $3,CS(%rsp) - jnz paranoid_userspace -paranoid_swapgs: - swapgs -paranoid_restore: - RESTORE_ALL 8 - iretq -paranoid_userspace: - GET_THREAD_INFO(%rcx) - movl threadinfo_flags(%rcx),%ebx - andl $_TIF_WORK_MASK,%ebx - jz paranoid_swapgs - movq %rsp,%rdi /* &pt_regs */ - call sync_regs - movq %rax,%rsp /* switch stack for scheduling */ - testl $_TIF_NEED_RESCHED,%ebx - jnz paranoid_schedule - movl %ebx,%edx /* arg3: thread flags */ - sti - xorl %esi,%esi /* arg2: oldset */ - movq %rsp,%rdi /* arg1: &pt_regs */ - call do_notify_resume - cli - jmp paranoid_userspace -paranoid_schedule: - sti - call schedule - cli - jmp paranoid_userspace - CFI_ENDPROC + paranoidentry do_nmi, 0, 0 +#ifdef CONFIG_TRACE_IRQFLAGS + paranoidexit 0 +#else + jmp paranoid_exit1 + CFI_ENDPROC +#endif END(nmi) .previous .text @@ -977,7 +1061,7 @@ KPROBE_ENTRY(int3) pushq $0 CFI_ADJUST_CFA_OFFSET 8 paranoidentry do_int3, DEBUG_STACK - jmp paranoid_exit + jmp paranoid_exit1 CFI_ENDPROC END(int3) .previous .text @@ -1006,7 +1090,7 @@ END(reserved) ENTRY(double_fault) XCPT_FRAME paranoidentry do_double_fault - jmp paranoid_exit + jmp paranoid_exit1 CFI_ENDPROC END(double_fault) @@ -1022,7 +1106,7 @@ END(segment_not_present) ENTRY(stack_segment) XCPT_FRAME paranoidentry do_stack_segment - jmp paranoid_exit + jmp paranoid_exit1 CFI_ENDPROC END(stack_segment) @@ -1050,7 +1134,7 @@ ENTRY(machine_check) pushq $0 CFI_ADJUST_CFA_OFFSET 8 paranoidentry do_machine_check - jmp paranoid_exit + jmp paranoid_exit1 CFI_ENDPROC END(machine_check) #endif diff --git a/arch/x86_64/kernel/head64.c b/arch/x86_64/kernel/head64.c index e6a71c9556d..36647ce6aec 100644 --- a/arch/x86_64/kernel/head64.c +++ b/arch/x86_64/kernel/head64.c @@ -85,6 +85,11 @@ void __init x86_64_start_kernel(char * real_mode_data) clear_bss(); /* + * This must be called really, really early: + */ + lockdep_init(); + + /* * switch to init_level4_pgt from boot_level4_pgt */ memcpy(init_level4_pgt, boot_level4_pgt, PTRS_PER_PGD*sizeof(pgd_t)); diff --git a/arch/x86_64/kernel/irq.c b/arch/x86_64/kernel/irq.c index a1f1df5f7bf..5221a53e90c 100644 --- a/arch/x86_64/kernel/irq.c +++ b/arch/x86_64/kernel/irq.c @@ -177,8 +177,10 @@ asmlinkage void do_softirq(void) local_irq_save(flags); pending = local_softirq_pending(); /* Switch to interrupt stack */ - if (pending) + if (pending) { call_softirq(); + WARN_ON_ONCE(softirq_count()); + } local_irq_restore(flags); } EXPORT_SYMBOL(do_softirq); diff --git a/arch/x86_64/kernel/nmi.c b/arch/x86_64/kernel/nmi.c index 476c1472fc0..5baa0c726e9 100644 --- a/arch/x86_64/kernel/nmi.c +++ b/arch/x86_64/kernel/nmi.c @@ -127,7 +127,7 @@ void __cpuinit nmi_watchdog_default(void) static __init void nmi_cpu_busy(void *data) { volatile int *endflag = data; - local_irq_enable(); + local_irq_enable_in_hardirq(); /* Intentionally don't use cpu_relax here. This is to make sure that the performance counter really ticks, even if there is a simulator or similar that catches the diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c index ca56e19b8b6..bb6745d13b8 100644 --- a/arch/x86_64/kernel/process.c +++ b/arch/x86_64/kernel/process.c @@ -296,7 +296,7 @@ void __show_regs(struct pt_regs * regs) system_utsname.version); printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip); printk_address(regs->rip); - printk("\nRSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp, + printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->rsp, regs->eflags); printk("RAX: %016lx RBX: %016lx RCX: %016lx\n", regs->rax, regs->rbx, regs->rcx); diff --git a/arch/x86_64/kernel/smpboot.c b/arch/x86_64/kernel/smpboot.c index 9705a6a384f..b7c70596979 100644 --- a/arch/x86_64/kernel/smpboot.c +++ b/arch/x86_64/kernel/smpboot.c @@ -775,6 +775,8 @@ static int __cpuinit do_boot_cpu(int cpu, int apicid) }; DECLARE_WORK(work, do_fork_idle, &c_idle); + lockdep_set_class(&c_idle.done.wait.lock, &waitqueue_lock_key); + /* allocate memory for gdts of secondary cpus. Hotplug is considered */ if (!cpu_gdt_descr[cpu].address && !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) { diff --git a/arch/x86_64/kernel/stacktrace.c b/arch/x86_64/kernel/stacktrace.c new file mode 100644 index 00000000000..32cf55eb9af --- /dev/null +++ b/arch/x86_64/kernel/stacktrace.c @@ -0,0 +1,221 @@ +/* + * arch/x86_64/kernel/stacktrace.c + * + * Stack trace management functions + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + */ +#include <linux/sched.h> +#include <linux/stacktrace.h> + +#include <asm/smp.h> + +static inline int +in_range(unsigned long start, unsigned long addr, unsigned long end) +{ + return addr >= start && addr <= end; +} + +static unsigned long +get_stack_end(struct task_struct *task, unsigned long stack) +{ + unsigned long stack_start, stack_end, flags; + int i, cpu; + + /* + * The most common case is that we are in the task stack: + */ + stack_start = (unsigned long)task->thread_info; + stack_end = stack_start + THREAD_SIZE; + + if (in_range(stack_start, stack, stack_end)) + return stack_end; + + /* + * We are in an interrupt if irqstackptr is set: + */ + raw_local_irq_save(flags); + cpu = safe_smp_processor_id(); + stack_end = (unsigned long)cpu_pda(cpu)->irqstackptr; + + if (stack_end) { + stack_start = stack_end & ~(IRQSTACKSIZE-1); + if (in_range(stack_start, stack, stack_end)) + goto out_restore; + /* + * We get here if we are in an IRQ context but we + * are also in an exception stack. + */ + } + + /* + * Iterate over all exception stacks, and figure out whether + * 'stack' is in one of them: + */ + for (i = 0; i < N_EXCEPTION_STACKS; i++) { + /* + * set 'end' to the end of the exception stack. + */ + stack_end = per_cpu(init_tss, cpu).ist[i]; + stack_start = stack_end - EXCEPTION_STKSZ; + + /* + * Is 'stack' above this exception frame's end? + * If yes then skip to the next frame. + */ + if (stack >= stack_end) + continue; + /* + * Is 'stack' above this exception frame's start address? + * If yes then we found the right frame. + */ + if (stack >= stack_start) + goto out_restore; + + /* + * If this is a debug stack, and if it has a larger size than + * the usual exception stacks, then 'stack' might still + * be within the lower portion of the debug stack: + */ +#if DEBUG_STKSZ > EXCEPTION_STKSZ + if (i == DEBUG_STACK - 1 && stack >= stack_end - DEBUG_STKSZ) { + /* + * Black magic. A large debug stack is composed of + * multiple exception stack entries, which we + * iterate through now. Dont look: + */ + do { + stack_end -= EXCEPTION_STKSZ; + stack_start -= EXCEPTION_STKSZ; + } while (stack < stack_start); + + goto out_restore; + } +#endif + } + /* + * Ok, 'stack' is not pointing to any of the system stacks. + */ + stack_end = 0; + +out_restore: + raw_local_irq_restore(flags); + + return stack_end; +} + + +/* + * Save stack-backtrace addresses into a stack_trace buffer: + */ +static inline unsigned long +save_context_stack(struct stack_trace *trace, unsigned int skip, + unsigned long stack, unsigned long stack_end) +{ + unsigned long addr; + +#ifdef CONFIG_FRAME_POINTER + unsigned long prev_stack = 0; + + while (in_range(prev_stack, stack, stack_end)) { + pr_debug("stack: %p\n", (void *)stack); + addr = (unsigned long)(((unsigned long *)stack)[1]); + pr_debug("addr: %p\n", (void *)addr); + if (!skip) + trace->entries[trace->nr_entries++] = addr-1; + else + skip--; + if (trace->nr_entries >= trace->max_entries) + break; + if (!addr) + return 0; + /* + * Stack frames must go forwards (otherwise a loop could + * happen if the stackframe is corrupted), so we move + * prev_stack forwards: + */ + prev_stack = stack; + stack = (unsigned long)(((unsigned long *)stack)[0]); + } + pr_debug("invalid: %p\n", (void *)stack); +#else + while (stack < stack_end) { + addr = ((unsigned long *)stack)[0]; + stack += sizeof(long); + if (__kernel_text_address(addr)) { + if (!skip) + trace->entries[trace->nr_entries++] = addr-1; + else + skip--; + if (trace->nr_entries >= trace->max_entries) + break; + } + } +#endif + return stack; +} + +#define MAX_STACKS 10 + +/* + * Save stack-backtrace addresses into a stack_trace buffer. + * If all_contexts is set, all contexts (hardirq, softirq and process) + * are saved. If not set then only the current context is saved. + */ +void save_stack_trace(struct stack_trace *trace, + struct task_struct *task, int all_contexts, + unsigned int skip) +{ + unsigned long stack = (unsigned long)&stack; + int i, nr_stacks = 0, stacks_done[MAX_STACKS]; + + WARN_ON(trace->nr_entries || !trace->max_entries); + + if (!task) + task = current; + + pr_debug("task: %p, ti: %p\n", task, task->thread_info); + + if (!task || task == current) { + /* Grab rbp right from our regs: */ + asm ("mov %%rbp, %0" : "=r" (stack)); + pr_debug("rbp: %p\n", (void *)stack); + } else { + /* rbp is the last reg pushed by switch_to(): */ + stack = task->thread.rsp; + pr_debug("other task rsp: %p\n", (void *)stack); + stack = (unsigned long)(((unsigned long *)stack)[0]); + pr_debug("other task rbp: %p\n", (void *)stack); + } + + while (1) { + unsigned long stack_end = get_stack_end(task, stack); + + pr_debug("stack: %p\n", (void *)stack); + pr_debug("stack end: %p\n", (void *)stack_end); + + /* + * Invalid stack addres? + */ + if (!stack_end) + return; + /* + * Were we in this stack already? (recursion) + */ + for (i = 0; i < nr_stacks; i++) + if (stacks_done[i] == stack_end) + return; + stacks_done[nr_stacks] = stack_end; + + stack = save_context_stack(trace, skip, stack, stack_end); + if (!all_contexts || !stack || + trace->nr_entries >= trace->max_entries) + return; + trace->entries[trace->nr_entries++] = ULONG_MAX; + if (trace->nr_entries >= trace->max_entries) + return; + if (++nr_stacks >= MAX_STACKS) + return; + } +} + diff --git a/arch/x86_64/kernel/traps.c b/arch/x86_64/kernel/traps.c index 5a5311d3de0..79d05c48207 100644 --- a/arch/x86_64/kernel/traps.c +++ b/arch/x86_64/kernel/traps.c @@ -110,28 +110,31 @@ static int kstack_depth_to_print = 12; static int call_trace = 1; #ifdef CONFIG_KALLSYMS -#include <linux/kallsyms.h> -int printk_address(unsigned long address) -{ +# include <linux/kallsyms.h> +void printk_address(unsigned long address) +{ unsigned long offset = 0, symsize; const char *symname; char *modname; - char *delim = ":"; + char *delim = ":"; char namebuf[128]; - symname = kallsyms_lookup(address, &symsize, &offset, &modname, namebuf); - if (!symname) - return printk("[<%016lx>]", address); - if (!modname) + symname = kallsyms_lookup(address, &symsize, &offset, + &modname, namebuf); + if (!symname) { + printk(" [<%016lx>]\n", address); + return; + } + if (!modname) modname = delim = ""; - return printk("<%016lx>{%s%s%s%s%+ld}", - address, delim, modname, delim, symname, offset); -} + printk(" [<%016lx>] %s%s%s%s+0x%lx/0x%lx\n", + address, delim, modname, delim, symname, offset, symsize); +} #else -int printk_address(unsigned long address) -{ - return printk("[<%016lx>]", address); -} +void printk_address(unsigned long address) +{ + printk(" [<%016lx>]\n", address); +} #endif static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack, @@ -149,10 +152,22 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack, }; unsigned k; + /* + * Iterate over all exception stacks, and figure out whether + * 'stack' is in one of them: + */ for (k = 0; k < N_EXCEPTION_STACKS; k++) { unsigned long end; + /* + * set 'end' to the end of the exception stack. + */ switch (k + 1) { + /* + * TODO: this block is not needed i think, because + * setup64.c:cpu_init() sets up t->ist[DEBUG_STACK] + * properly too. + */ #if DEBUG_STKSZ > EXCEPTION_STKSZ case DEBUG_STACK: end = cpu_pda(cpu)->debugstack + DEBUG_STKSZ; @@ -162,19 +177,43 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack, end = per_cpu(init_tss, cpu).ist[k]; break; } + /* + * Is 'stack' above this exception frame's end? + * If yes then skip to the next frame. + */ if (stack >= end) continue; + /* + * Is 'stack' above this exception frame's start address? + * If yes then we found the right frame. + */ if (stack >= end - EXCEPTION_STKSZ) { + /* + * Make sure we only iterate through an exception + * stack once. If it comes up for the second time + * then there's something wrong going on - just + * break out and return NULL: + */ if (*usedp & (1U << k)) break; *usedp |= 1U << k; *idp = ids[k]; return (unsigned long *)end; } + /* + * If this is a debug stack, and if it has a larger size than + * the usual exception stacks, then 'stack' might still + * be within the lower portion of the debug stack: + */ #if DEBUG_STKSZ > EXCEPTION_STKSZ if (k == DEBUG_STACK - 1 && stack >= end - DEBUG_STKSZ) { unsigned j = N_EXCEPTION_STACKS - 1; + /* + * Black magic. A large debug stack is composed of + * multiple exception stack entries, which we + * iterate through now. Dont look: + */ do { ++j; end -= EXCEPTION_STKSZ; @@ -193,20 +232,14 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack, static int show_trace_unwind(struct unwind_frame_info *info, void *context) { - int i = 11, n = 0; + int n = 0; while (unwind(info) == 0 && UNW_PC(info)) { - ++n; - if (i > 50) { - printk("\n "); - i = 7; - } else - i += printk(" "); - i += printk_address(UNW_PC(info)); + n++; + printk_address(UNW_PC(info)); if (arch_unw_user_mode(info)) break; } - printk("\n"); return n; } @@ -224,7 +257,7 @@ void show_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s int i = 11; unsigned used = 0; - printk("\nCall Trace:"); + printk("\nCall Trace:\n"); if (!tsk) tsk = current; @@ -250,16 +283,15 @@ void show_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s } } + /* + * Print function call entries within a stack. 'cond' is the + * "end of stackframe" condition, that the 'stack++' + * iteration will eventually trigger. + */ #define HANDLE_STACK(cond) \ do while (cond) { \ unsigned long addr = *stack++; \ if (kernel_text_address(addr)) { \ - if (i > 50) { \ - printk("\n "); \ - i = 0; \ - } \ - else \ - i += printk(" "); \ /* \ * If the address is either in the text segment of the \ * kernel, or in the region which contains vmalloc'ed \ @@ -268,20 +300,30 @@ void show_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s * down the cause of the crash will be able to figure \ * out the call path that was taken. \ */ \ - i += printk_address(addr); \ + printk_address(addr); \ } \ } while (0) - for(; ; ) { + /* + * Print function call entries in all stacks, starting at the + * current stack address. If the stacks consist of nested + * exceptions + */ + for ( ; ; ) { const char *id; unsigned long *estack_end; estack_end = in_exception_stack(cpu, (unsigned long)stack, &used, &id); if (estack_end) { - i += printk(" <%s>", id); + printk(" <%s>", id); HANDLE_STACK (stack < estack_end); - i += printk(" <EOE>"); + printk(" <EOE>"); + /* + * We link to the next stack via the + * second-to-last pointer (index -2 to end) in the + * exception stack: + */ stack = (unsigned long *) estack_end[-2]; continue; } @@ -291,19 +333,28 @@ void show_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s (IRQSTACKSIZE - 64) / sizeof(*irqstack); if (stack >= irqstack && stack < irqstack_end) { - i += printk(" <IRQ>"); + printk(" <IRQ>"); HANDLE_STACK (stack < irqstack_end); + /* + * We link to the next stack (which would be + * the process stack normally) the last + * pointer (index -1 to end) in the IRQ stack: + */ stack = (unsigned long *) (irqstack_end[-1]); irqstack_end = NULL; - i += printk(" <EOI>"); + printk(" <EOI>"); continue; } } break; } + /* + * This prints the process stack: + */ HANDLE_STACK (((long) stack & (THREAD_SIZE-1)) != 0); #undef HANDLE_STACK + printk("\n"); } @@ -337,8 +388,8 @@ static void _show_stack(struct task_struct *tsk, struct pt_regs *regs, unsigned break; } if (i && ((i % 4) == 0)) - printk("\n "); - printk("%016lx ", *stack++); + printk("\n"); + printk(" %016lx", *stack++); touch_nmi_watchdog(); } show_trace(tsk, regs, rsp); diff --git a/arch/x86_64/lib/thunk.S b/arch/x86_64/lib/thunk.S index e49af0032e9..332ea5dff91 100644 --- a/arch/x86_64/lib/thunk.S +++ b/arch/x86_64/lib/thunk.S @@ -47,6 +47,11 @@ thunk_retrax __down_failed_interruptible,__down_interruptible thunk_retrax __down_failed_trylock,__down_trylock thunk __up_wakeup,__up + +#ifdef CONFIG_TRACE_IRQFLAGS + thunk trace_hardirqs_on_thunk,trace_hardirqs_on + thunk trace_hardirqs_off_thunk,trace_hardirqs_off +#endif /* SAVE_ARGS below is used only for the .cfi directives it contains. */ CFI_STARTPROC diff --git a/arch/x86_64/mm/fault.c b/arch/x86_64/mm/fault.c index 5afcf6eb00f..ac8ea66ccb9 100644 --- a/arch/x86_64/mm/fault.c +++ b/arch/x86_64/mm/fault.c @@ -570,7 +570,6 @@ no_context: printk(KERN_ALERT "Unable to handle kernel paging request"); printk(" at %016lx RIP: \n" KERN_ALERT,address); printk_address(regs->rip); - printk("\n"); dump_pagetable(address); tsk->thread.cr2 = address; tsk->thread.trap_no = 14; diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 5813d63c20a..ab17c7224bb 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -2516,7 +2516,7 @@ EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); int blk_execute_rq(request_queue_t *q, struct gendisk *bd_disk, struct request *rq, int at_head) { - DECLARE_COMPLETION(wait); + DECLARE_COMPLETION_ONSTACK(wait); char sense[SCSI_SENSE_BUFFERSIZE]; int err = 0; diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 0242cbb86a8..5109fa37c66 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -249,18 +249,6 @@ static int irqdma_allocated; #include <linux/cdrom.h> /* for the compatibility eject ioctl */ #include <linux/completion.h> -/* - * Interrupt freeing also means /proc VFS work - dont do it - * from interrupt context. We push this work into keventd: - */ -static void fd_free_irq_fn(void *data) -{ - fd_free_irq(); -} - -static DECLARE_WORK(fd_free_irq_work, fd_free_irq_fn, NULL); - - static struct request *current_req; static struct request_queue *floppy_queue; static void do_fd_request(request_queue_t * q); @@ -826,15 +814,6 @@ static int set_dor(int fdc, char mask, char data) UDRS->select_date = jiffies; } } - /* - * We should propagate failures to grab the resources back - * nicely from here. Actually we ought to rewrite the fd - * driver some day too. - */ - if (newdor & FLOPPY_MOTOR_MASK) - floppy_grab_irq_and_dma(); - if (olddor & FLOPPY_MOTOR_MASK) - floppy_release_irq_and_dma(); return olddor; } @@ -892,8 +871,6 @@ static int _lock_fdc(int drive, int interruptible, int line) line); return -1; } - if (floppy_grab_irq_and_dma() == -1) - return -EBUSY; if (test_and_set_bit(0, &fdc_busy)) { DECLARE_WAITQUEUE(wait, current); @@ -915,6 +892,8 @@ static int _lock_fdc(int drive, int interruptible, int line) set_current_state(TASK_RUNNING); remove_wait_queue(&fdc_wait, &wait); + + flush_scheduled_work(); } command_status = FD_COMMAND_NONE; @@ -948,7 +927,6 @@ static inline void unlock_fdc(void) if (elv_next_request(floppy_queue)) do_fd_request(floppy_queue); spin_unlock_irqrestore(&floppy_lock, flags); - floppy_release_irq_and_dma(); wake_up(&fdc_wait); } @@ -3694,8 +3672,8 @@ static int floppy_release(struct inode *inode, struct file *filp) } if (!UDRS->fd_ref) opened_bdev[drive] = NULL; - floppy_release_irq_and_dma(); mutex_unlock(&open_lock); + return 0; } @@ -3726,9 +3704,6 @@ static int floppy_open(struct inode *inode, struct file *filp) if (UDRS->fd_ref == -1 || (UDRS->fd_ref && (filp->f_flags & O_EXCL))) goto out2; - if (floppy_grab_irq_and_dma()) - goto out2; - if (filp->f_flags & O_EXCL) UDRS->fd_ref = -1; else @@ -3805,7 +3780,6 @@ out: UDRS->fd_ref--; if (!UDRS->fd_ref) opened_bdev[drive] = NULL; - floppy_release_irq_and_dma(); out2: mutex_unlock(&open_lock); return res; @@ -3822,14 +3796,9 @@ static int check_floppy_change(struct gendisk *disk) return 1; if (time_after(jiffies, UDRS->last_checked + UDP->checkfreq)) { - if (floppy_grab_irq_and_dma()) { - return 1; - } - lock_fdc(drive, 0); poll_drive(0, 0); process_fd_request(); - floppy_release_irq_and_dma(); } if (UTESTF(FD_DISK_CHANGED) || @@ -4346,7 +4315,6 @@ static int __init floppy_init(void) fdc = 0; del_timer(&fd_timeout); current_drive = 0; - floppy_release_irq_and_dma(); initialising = 0; if (have_no_fdc) { DPRINT("no floppy controllers found\n"); @@ -4504,7 +4472,7 @@ static void floppy_release_irq_and_dma(void) if (irqdma_allocated) { fd_disable_dma(); fd_free_dma(); - schedule_work(&fd_free_irq_work); + fd_free_irq(); irqdma_allocated = 0; } set_dor(0, ~0, 8); @@ -4600,8 +4568,6 @@ void cleanup_module(void) /* eject disk, if any */ fd_eject(0); - flush_scheduled_work(); /* fd_free_irq() might be pending */ - wait_for_completion(&device_release); } diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c index ffcf15c30e9..d9c5a9142ad 100644 --- a/drivers/char/agp/frontend.c +++ b/drivers/char/agp/frontend.c @@ -1059,7 +1059,7 @@ ioctl_out: return ret_val; } -static struct file_operations agp_fops = +static const struct file_operations agp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/applicom.c b/drivers/char/applicom.c index bcc4668835b..10a389dafd6 100644 --- a/drivers/char/applicom.c +++ b/drivers/char/applicom.c @@ -112,7 +112,7 @@ static int ac_ioctl(struct inode *, struct file *, unsigned int, unsigned long); static irqreturn_t ac_interrupt(int, void *, struct pt_regs *); -static struct file_operations ac_fops = { +static const struct file_operations ac_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = ac_read, diff --git a/drivers/char/cs5535_gpio.c b/drivers/char/cs5535_gpio.c index 46d66037b91..8ce3f34cfc2 100644 --- a/drivers/char/cs5535_gpio.c +++ b/drivers/char/cs5535_gpio.c @@ -158,7 +158,7 @@ static int cs5535_gpio_open(struct inode *inode, struct file *file) return nonseekable_open(inode, file); } -static struct file_operations cs5535_gpio_fops = { +static const struct file_operations cs5535_gpio_fops = { .owner = THIS_MODULE, .write = cs5535_gpio_write, .read = cs5535_gpio_read, diff --git a/drivers/char/ds1286.c b/drivers/char/ds1286.c index d755cac14bc..21c8229f544 100644 --- a/drivers/char/ds1286.c +++ b/drivers/char/ds1286.c @@ -281,7 +281,7 @@ static unsigned int ds1286_poll(struct file *file, poll_table *wait) * The various file operations we support. */ -static struct file_operations ds1286_fops = { +static const struct file_operations ds1286_fops = { .llseek = no_llseek, .read = ds1286_read, .poll = ds1286_poll, diff --git a/drivers/char/ds1302.c b/drivers/char/ds1302.c index 625e8b51700..bcdb107aa96 100644 --- a/drivers/char/ds1302.c +++ b/drivers/char/ds1302.c @@ -282,7 +282,7 @@ get_rtc_status(char *buf) /* The various file operations we support. */ -static struct file_operations rtc_fops = { +static const struct file_operations rtc_fops = { .owner = THIS_MODULE, .ioctl = rtc_ioctl, }; diff --git a/drivers/char/ds1620.c b/drivers/char/ds1620.c index 953e670dcd0..48cb8f0e8eb 100644 --- a/drivers/char/ds1620.c +++ b/drivers/char/ds1620.c @@ -336,7 +336,7 @@ proc_therm_ds1620_read(char *buf, char **start, off_t offset, static struct proc_dir_entry *proc_therm_ds1620; #endif -static struct file_operations ds1620_fops = { +static const struct file_operations ds1620_fops = { .owner = THIS_MODULE, .open = nonseekable_open, .read = ds1620_read, diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c index 09b413618b5..9b1bf60ffbe 100644 --- a/drivers/char/dsp56k.c +++ b/drivers/char/dsp56k.c @@ -483,7 +483,7 @@ static int dsp56k_release(struct inode *inode, struct file *file) return 0; } -static struct file_operations dsp56k_fops = { +static const struct file_operations dsp56k_fops = { .owner = THIS_MODULE, .read = dsp56k_read, .write = dsp56k_write, diff --git a/drivers/char/dtlk.c b/drivers/char/dtlk.c index da2c89f1b8b..5e82c3bad2e 100644 --- a/drivers/char/dtlk.c +++ b/drivers/char/dtlk.c @@ -94,7 +94,7 @@ static int dtlk_release(struct inode *, struct file *); static int dtlk_ioctl(struct inode *inode, struct file *file, unsigned int cmd, unsigned long arg); -static struct file_operations dtlk_fops = +static const struct file_operations dtlk_fops = { .owner = THIS_MODULE, .read = dtlk_read, diff --git a/drivers/char/efirtc.c b/drivers/char/efirtc.c index 0090e7a4fcd..004141d535a 100644 --- a/drivers/char/efirtc.c +++ b/drivers/char/efirtc.c @@ -285,7 +285,7 @@ efi_rtc_close(struct inode *inode, struct file *file) * The various file operations we support. */ -static struct file_operations efi_rtc_fops = { +static const struct file_operations efi_rtc_fops = { .owner = THIS_MODULE, .ioctl = efi_rtc_ioctl, .open = efi_rtc_open, diff --git a/drivers/char/ftape/zftape/zftape-init.c b/drivers/char/ftape/zftape/zftape-init.c index 55272566b74..164a1aa77a2 100644 --- a/drivers/char/ftape/zftape/zftape-init.c +++ b/drivers/char/ftape/zftape/zftape-init.c @@ -86,7 +86,7 @@ static ssize_t zft_read (struct file *fp, char __user *buff, static ssize_t zft_write(struct file *fp, const char __user *buff, size_t req_len, loff_t *ppos); -static struct file_operations zft_cdev = +static const struct file_operations zft_cdev = { .owner = THIS_MODULE, .read = zft_read, diff --git a/drivers/char/genrtc.c b/drivers/char/genrtc.c index bebd7e34f79..817dc409ac2 100644 --- a/drivers/char/genrtc.c +++ b/drivers/char/genrtc.c @@ -482,7 +482,7 @@ static inline int gen_rtc_proc_init(void) { return 0; } * The various file operations we support. */ -static struct file_operations gen_rtc_fops = { +static const struct file_operations gen_rtc_fops = { .owner = THIS_MODULE, #ifdef CONFIG_GEN_RTC_X .read = gen_rtc_read, diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c index e5643f3aa73..8afba339f05 100644 --- a/drivers/char/hpet.c +++ b/drivers/char/hpet.c @@ -553,7 +553,7 @@ hpet_ioctl_common(struct hpet_dev *devp, int cmd, unsigned long arg, int kernel) return err; } -static struct file_operations hpet_fops = { +static const struct file_operations hpet_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = hpet_read, diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 88b026639f1..154a81d328c 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -149,7 +149,7 @@ out: } -static struct file_operations rng_chrdev_ops = { +static const struct file_operations rng_chrdev_ops = { .owner = THIS_MODULE, .open = rng_dev_open, .read = rng_dev_read, diff --git a/drivers/char/i8k.c b/drivers/char/i8k.c index f3c3aaf4560..353d9f3cf8d 100644 --- a/drivers/char/i8k.c +++ b/drivers/char/i8k.c @@ -80,7 +80,7 @@ static int i8k_open_fs(struct inode *inode, struct file *file); static int i8k_ioctl(struct inode *, struct file *, unsigned int, unsigned long); -static struct file_operations i8k_fops = { +static const struct file_operations i8k_fops = { .open = i8k_open_fs, .read = seq_read, .llseek = seq_lseek, diff --git a/drivers/char/ip2/ip2main.c b/drivers/char/ip2/ip2main.c index a4200a2b081..518ece7ac65 100644 --- a/drivers/char/ip2/ip2main.c +++ b/drivers/char/ip2/ip2main.c @@ -233,7 +233,7 @@ static void *DevTableMem[IP2_MAX_BOARDS]; /* This is the driver descriptor for the ip2ipl device, which is used to * download the loadware to the boards. */ -static struct file_operations ip2_ipl = { +static const struct file_operations ip2_ipl = { .owner = THIS_MODULE, .read = ip2_ipl_read, .write = ip2_ipl_write, diff --git a/drivers/char/ip27-rtc.c b/drivers/char/ip27-rtc.c index 3acdac3c967..a48da02aad2 100644 --- a/drivers/char/ip27-rtc.c +++ b/drivers/char/ip27-rtc.c @@ -196,7 +196,7 @@ static int rtc_release(struct inode *inode, struct file *file) * The various file operations we support. */ -static struct file_operations rtc_fops = { +static const struct file_operations rtc_fops = { .owner = THIS_MODULE, .ioctl = rtc_ioctl, .open = rtc_open, diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c index 2fc894fef1c..68d7c61a864 100644 --- a/drivers/char/ipmi/ipmi_devintf.c +++ b/drivers/char/ipmi/ipmi_devintf.c @@ -765,7 +765,7 @@ static long compat_ipmi_ioctl(struct file *filep, unsigned int cmd, } #endif -static struct file_operations ipmi_fops = { +static const struct file_operations ipmi_fops = { .owner = THIS_MODULE, .ioctl = ipmi_ioctl, #ifdef CONFIG_COMPAT diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c index 74a889c5833..accaaf1a6b6 100644 --- a/drivers/char/ipmi/ipmi_watchdog.c +++ b/drivers/char/ipmi/ipmi_watchdog.c @@ -807,7 +807,7 @@ static int ipmi_close(struct inode *ino, struct file *filep) return 0; } -static struct file_operations ipmi_wdog_fops = { +static const struct file_operations ipmi_wdog_fops = { .owner = THIS_MODULE, .read = ipmi_read, .poll = ipmi_poll, diff --git a/drivers/char/istallion.c b/drivers/char/istallion.c index fbce2f0669d..84dfc427813 100644 --- a/drivers/char/istallion.c +++ b/drivers/char/istallion.c @@ -748,7 +748,7 @@ static int stli_initpcibrd(int brdtype, struct pci_dev *devp); * will give access to the shared memory on the Stallion intelligent * board. This is also a very useful debugging tool. */ -static struct file_operations stli_fsiomem = { +static const struct file_operations stli_fsiomem = { .owner = THIS_MODULE, .read = stli_memread, .write = stli_memwrite, diff --git a/drivers/char/ite_gpio.c b/drivers/char/ite_gpio.c index 747ba45e50e..cde562d70c4 100644 --- a/drivers/char/ite_gpio.c +++ b/drivers/char/ite_gpio.c @@ -357,7 +357,7 @@ DEB(printk("interrupt 0x%x %d\n",ITE_GPAISR, i)); } } -static struct file_operations ite_gpio_fops = { +static const struct file_operations ite_gpio_fops = { .owner = THIS_MODULE, .ioctl = ite_gpio_ioctl, .open = ite_gpio_open, diff --git a/drivers/char/lcd.c b/drivers/char/lcd.c index 7d49b241de5..da601fd6c07 100644 --- a/drivers/char/lcd.c +++ b/drivers/char/lcd.c @@ -598,7 +598,7 @@ static ssize_t lcd_read(struct file *file, char *buf, * The various file operations we support. */ -static struct file_operations lcd_fops = { +static const struct file_operations lcd_fops = { .read = lcd_read, .ioctl = lcd_ioctl, .open = lcd_open, diff --git a/drivers/char/lp.c b/drivers/char/lp.c index 582cdbdb0c4..f875fda3b08 100644 --- a/drivers/char/lp.c +++ b/drivers/char/lp.c @@ -666,7 +666,7 @@ static int lp_ioctl(struct inode *inode, struct file *file, return retval; } -static struct file_operations lp_fops = { +static const struct file_operations lp_fops = { .owner = THIS_MODULE, .write = lp_write, .ioctl = lp_ioctl, diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 70f3954d6df..e97c32ceb79 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -776,7 +776,7 @@ static int open_port(struct inode * inode, struct file * filp) #define open_kmem open_mem #define open_oldmem open_mem -static struct file_operations mem_fops = { +static const struct file_operations mem_fops = { .llseek = memory_lseek, .read = read_mem, .write = write_mem, @@ -784,7 +784,7 @@ static struct file_operations mem_fops = { .open = open_mem, }; -static struct file_operations kmem_fops = { +static const struct file_operations kmem_fops = { .llseek = memory_lseek, .read = read_kmem, .write = write_kmem, @@ -792,7 +792,7 @@ static struct file_operations kmem_fops = { .open = open_kmem, }; -static struct file_operations null_fops = { +static const struct file_operations null_fops = { .llseek = null_lseek, .read = read_null, .write = write_null, @@ -800,7 +800,7 @@ static struct file_operations null_fops = { }; #if defined(CONFIG_ISA) || !defined(__mc68000__) -static struct file_operations port_fops = { +static const struct file_operations port_fops = { .llseek = memory_lseek, .read = read_port, .write = write_port, @@ -808,7 +808,7 @@ static struct file_operations port_fops = { }; #endif -static struct file_operations zero_fops = { +static const struct file_operations zero_fops = { .llseek = zero_lseek, .read = read_zero, .write = write_zero, @@ -819,14 +819,14 @@ static struct backing_dev_info zero_bdi = { .capabilities = BDI_CAP_MAP_COPY, }; -static struct file_operations full_fops = { +static const struct file_operations full_fops = { .llseek = full_lseek, .read = read_full, .write = write_full, }; #ifdef CONFIG_CRASH_DUMP -static struct file_operations oldmem_fops = { +static const struct file_operations oldmem_fops = { .read = read_oldmem, .open = open_oldmem, }; @@ -853,7 +853,7 @@ static ssize_t kmsg_write(struct file * file, const char __user * buf, return ret; } -static struct file_operations kmsg_fops = { +static const struct file_operations kmsg_fops = { .write = kmsg_write, }; @@ -903,7 +903,7 @@ static int memory_open(struct inode * inode, struct file * filp) return 0; } -static struct file_operations memory_fops = { +static const struct file_operations memory_fops = { .open = memory_open, /* just a selector for the real open */ }; diff --git a/drivers/char/misc.c b/drivers/char/misc.c index d5fa19da330..62ebe09656e 100644 --- a/drivers/char/misc.c +++ b/drivers/char/misc.c @@ -113,7 +113,7 @@ static int misc_seq_open(struct inode *inode, struct file *file) return seq_open(file, &misc_seq_ops); } -static struct file_operations misc_proc_fops = { +static const struct file_operations misc_proc_fops = { .owner = THIS_MODULE, .open = misc_seq_open, .read = seq_read, @@ -176,7 +176,7 @@ fail: */ static struct class *misc_class; -static struct file_operations misc_fops = { +static const struct file_operations misc_fops = { .owner = THIS_MODULE, .open = misc_open, }; diff --git a/drivers/char/mmtimer.c b/drivers/char/mmtimer.c index 70b774ff5aa..1f0f2b6dae2 100644 --- a/drivers/char/mmtimer.c +++ b/drivers/char/mmtimer.c @@ -63,7 +63,7 @@ static int mmtimer_mmap(struct file *file, struct vm_area_struct *vma); */ static unsigned long mmtimer_femtoperiod = 0; -static struct file_operations mmtimer_fops = { +static const struct file_operations mmtimer_fops = { .owner = THIS_MODULE, .mmap = mmtimer_mmap, .ioctl = mmtimer_ioctl, diff --git a/drivers/char/mwave/mwavedd.c b/drivers/char/mwave/mwavedd.c index d3ba2f860ef..39a2e661ff5 100644 --- a/drivers/char/mwave/mwavedd.c +++ b/drivers/char/mwave/mwavedd.c @@ -454,7 +454,7 @@ static int register_serial_portandirq(unsigned int port, int irq) } -static struct file_operations mwave_fops = { +static const struct file_operations mwave_fops = { .owner = THIS_MODULE, .read = mwave_read, .write = mwave_write, diff --git a/drivers/char/nvram.c b/drivers/char/nvram.c index 8c5f102622b..a39f19c35a6 100644 --- a/drivers/char/nvram.c +++ b/drivers/char/nvram.c @@ -437,7 +437,7 @@ nvram_read_proc(char *buffer, char **start, off_t offset, #endif /* CONFIG_PROC_FS */ -static struct file_operations nvram_fops = { +static const struct file_operations nvram_fops = { .owner = THIS_MODULE, .llseek = nvram_llseek, .read = nvram_read, diff --git a/drivers/char/nwbutton.c b/drivers/char/nwbutton.c index f240a104d25..7c57ebfa864 100644 --- a/drivers/char/nwbutton.c +++ b/drivers/char/nwbutton.c @@ -183,7 +183,7 @@ static int button_read (struct file *filp, char __user *buffer, * attempts to perform these operations on the device. */ -static struct file_operations button_fops = { +static const struct file_operations button_fops = { .owner = THIS_MODULE, .read = button_read, }; diff --git a/drivers/char/nwflash.c b/drivers/char/nwflash.c index 8865387d344..206cf6f5069 100644 --- a/drivers/char/nwflash.c +++ b/drivers/char/nwflash.c @@ -642,7 +642,7 @@ static void kick_open(void) udelay(25); } -static struct file_operations flash_fops = +static const struct file_operations flash_fops = { .owner = THIS_MODULE, .llseek = flash_llseek, diff --git a/drivers/char/pc8736x_gpio.c b/drivers/char/pc8736x_gpio.c index c860de6a6fd..4005ee0aa11 100644 --- a/drivers/char/pc8736x_gpio.c +++ b/drivers/char/pc8736x_gpio.c @@ -236,7 +236,7 @@ static int pc8736x_gpio_open(struct inode *inode, struct file *file) return nonseekable_open(inode, file); } -static struct file_operations pc8736x_gpio_fops = { +static const struct file_operations pc8736x_gpio_fops = { .owner = THIS_MODULE, .open = pc8736x_gpio_open, .write = nsc_gpio_write, diff --git a/drivers/char/pcmcia/cm4000_cs.c b/drivers/char/pcmcia/cm4000_cs.c index 31c8a21f9d8..50d20aafeb1 100644 --- a/drivers/char/pcmcia/cm4000_cs.c +++ b/drivers/char/pcmcia/cm4000_cs.c @@ -1938,7 +1938,7 @@ static void cm4000_detach(struct pcmcia_device *link) return; } -static struct file_operations cm4000_fops = { +static const struct file_operations cm4000_fops = { .owner = THIS_MODULE, .read = cmm_read, .write = cmm_write, diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c index 47a8465bf95..55cf4be4297 100644 --- a/drivers/char/pcmcia/cm4040_cs.c +++ b/drivers/char/pcmcia/cm4040_cs.c @@ -688,7 +688,7 @@ static void reader_detach(struct pcmcia_device *link) return; } -static struct file_operations reader_fops = { +static const struct file_operations reader_fops = { .owner = THIS_MODULE, .read = cm4040_read, .write = cm4040_write, diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c index 24231d9743d..520d2cf82bc 100644 --- a/drivers/char/ppdev.c +++ b/drivers/char/ppdev.c @@ -739,7 +739,7 @@ static unsigned int pp_poll (struct file * file, poll_table * wait) static struct class *ppdev_class; -static struct file_operations pp_fops = { +static const struct file_operations pp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = pp_read, diff --git a/drivers/char/random.c b/drivers/char/random.c index 164bddae047..4c3a5ca9d8f 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -416,7 +416,7 @@ static struct entropy_store input_pool = { .poolinfo = &poolinfo_table[0], .name = "input", .limit = 1, - .lock = SPIN_LOCK_UNLOCKED, + .lock = __SPIN_LOCK_UNLOCKED(&input_pool.lock), .pool = input_pool_data }; @@ -425,7 +425,7 @@ static struct entropy_store blocking_pool = { .name = "blocking", .limit = 1, .pull = &input_pool, - .lock = SPIN_LOCK_UNLOCKED, + .lock = __SPIN_LOCK_UNLOCKED(&blocking_pool.lock), .pool = blocking_pool_data }; @@ -433,7 +433,7 @@ static struct entropy_store nonblocking_pool = { .poolinfo = &poolinfo_table[1], .name = "nonblocking", .pull = &input_pool, - .lock = SPIN_LOCK_UNLOCKED, + .lock = __SPIN_LOCK_UNLOCKED(&nonblocking_pool.lock), .pool = nonblocking_pool_data }; diff --git a/drivers/char/raw.c b/drivers/char/raw.c index 9bf97c5e38c..579868af4a5 100644 --- a/drivers/char/raw.c +++ b/drivers/char/raw.c @@ -30,7 +30,7 @@ struct raw_device_data { static struct class *raw_class; static struct raw_device_data raw_devices[MAX_RAW_MINORS]; static DEFINE_MUTEX(raw_mutex); -static struct file_operations raw_ctl_fops; /* forward declaration */ +static const struct file_operations raw_ctl_fops; /* forward declaration */ /* * Open/close code for raw IO. @@ -261,7 +261,7 @@ static ssize_t raw_file_aio_write(struct kiocb *iocb, const char __user *buf, } -static struct file_operations raw_fops = { +static const struct file_operations raw_fops = { .read = generic_file_read, .aio_read = generic_file_aio_read, .write = raw_file_write, @@ -274,7 +274,7 @@ static struct file_operations raw_fops = { .owner = THIS_MODULE, }; -static struct file_operations raw_ctl_fops = { +static const struct file_operations raw_ctl_fops = { .ioctl = raw_ctl_ioctl, .open = raw_open, .owner = THIS_MODULE, diff --git a/drivers/char/rio/rio_linux.c b/drivers/char/rio/rio_linux.c index 3afc6a47ebb..3fa80aaf452 100644 --- a/drivers/char/rio/rio_linux.c +++ b/drivers/char/rio/rio_linux.c @@ -243,7 +243,7 @@ static struct real_driver rio_real_driver = { * */ -static struct file_operations rio_fw_fops = { +static const struct file_operations rio_fw_fops = { .owner = THIS_MODULE, .ioctl = rio_fw_ioctl, }; diff --git a/drivers/char/rtc.c b/drivers/char/rtc.c index aefac4ac0bf..cc7bd1a3095 100644 --- a/drivers/char/rtc.c +++ b/drivers/char/rtc.c @@ -877,7 +877,7 @@ int rtc_control(rtc_task_t *task, unsigned int cmd, unsigned long arg) * The various file operations we support. */ -static struct file_operations rtc_fops = { +static const struct file_operations rtc_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = rtc_read, @@ -896,7 +896,7 @@ static struct miscdevice rtc_dev = { .fops = &rtc_fops, }; -static struct file_operations rtc_proc_fops = { +static const struct file_operations rtc_proc_fops = { .owner = THIS_MODULE, .open = rtc_proc_open, .read = seq_read, diff --git a/drivers/char/scx200_gpio.c b/drivers/char/scx200_gpio.c index 45083e5dd23..425c58719db 100644 --- a/drivers/char/scx200_gpio.c +++ b/drivers/char/scx200_gpio.c @@ -63,7 +63,7 @@ static int scx200_gpio_release(struct inode *inode, struct file *file) } -static struct file_operations scx200_gpio_fops = { +static const struct file_operations scx200_gpio_fops = { .owner = THIS_MODULE, .write = nsc_gpio_write, .read = nsc_gpio_read, diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c index 203240b6c08..afc6eda602f 100644 --- a/drivers/char/snsc.c +++ b/drivers/char/snsc.c @@ -347,7 +347,7 @@ scdrv_poll(struct file *file, struct poll_table_struct *wait) return mask; } -static struct file_operations scdrv_fops = { +static const struct file_operations scdrv_fops = { .owner = THIS_MODULE, .read = scdrv_read, .write = scdrv_write, diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c index 45508a03950..d4e434d694b 100644 --- a/drivers/char/sonypi.c +++ b/drivers/char/sonypi.c @@ -1106,7 +1106,7 @@ static int sonypi_misc_ioctl(struct inode *ip, struct file *fp, return ret; } -static struct file_operations sonypi_misc_fops = { +static const struct file_operations sonypi_misc_fops = { .owner = THIS_MODULE, .read = sonypi_misc_read, .poll = sonypi_misc_poll, diff --git a/drivers/char/stallion.c b/drivers/char/stallion.c index ed7b8eaf036..3beb2203d24 100644 --- a/drivers/char/stallion.c +++ b/drivers/char/stallion.c @@ -707,7 +707,7 @@ static unsigned int sc26198_baudtable[] = { * Define the driver info for a user level control device. Used mainly * to get at port stats - only not using the port device itself. */ -static struct file_operations stl_fsiomem = { +static const struct file_operations stl_fsiomem = { .owner = THIS_MODULE, .ioctl = stl_memioctl, }; diff --git a/drivers/char/sx.c b/drivers/char/sx.c index 45c193aa11d..e1cd2bc4b1e 100644 --- a/drivers/char/sx.c +++ b/drivers/char/sx.c @@ -410,7 +410,7 @@ static struct real_driver sx_real_driver = { * */ -static struct file_operations sx_fw_fops = { +static const struct file_operations sx_fw_fops = { .owner = THIS_MODULE, .ioctl = sx_fw_ioctl, }; diff --git a/drivers/char/sysrq.c b/drivers/char/sysrq.c index a064ee9181c..ee3ca8f1768 100644 --- a/drivers/char/sysrq.c +++ b/drivers/char/sysrq.c @@ -147,12 +147,13 @@ static struct sysrq_key_op sysrq_mountro_op = { .enable_mask = SYSRQ_ENABLE_REMOUNT, }; -#ifdef CONFIG_DEBUG_MUTEXES +#ifdef CONFIG_LOCKDEP static void sysrq_handle_showlocks(int key, struct pt_regs *pt_regs, struct tty_struct *tty) { - mutex_debug_show_all_locks(); + debug_show_all_locks(); } + static struct sysrq_key_op sysrq_showlocks_op = { .handler = sysrq_handle_showlocks, .help_msg = "show-all-locks(D)", diff --git a/drivers/char/tb0219.c b/drivers/char/tb0219.c index a80c8321087..bb1bad4c18f 100644 --- a/drivers/char/tb0219.c +++ b/drivers/char/tb0219.c @@ -255,7 +255,7 @@ static int tanbac_tb0219_release(struct inode *inode, struct file *file) return 0; } -static struct file_operations tb0219_fops = { +static const struct file_operations tb0219_fops = { .owner = THIS_MODULE, .read = tanbac_tb0219_read, .write = tanbac_tb0219_write, diff --git a/drivers/char/tipar.c b/drivers/char/tipar.c index e0633a119d2..d30dc09dbbc 100644 --- a/drivers/char/tipar.c +++ b/drivers/char/tipar.c @@ -381,7 +381,7 @@ tipar_ioctl(struct inode *inode, struct file *file, /* ----- kernel module registering ------------------------------------ */ -static struct file_operations tipar_fops = { +static const struct file_operations tipar_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = tipar_read, diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c index 952b829e2cb..d2c5ba4e83b 100644 --- a/drivers/char/tlclk.c +++ b/drivers/char/tlclk.c @@ -247,7 +247,7 @@ static ssize_t tlclk_write(struct file *filp, const char __user *buf, size_t cou return 0; } -static struct file_operations tlclk_fops = { +static const struct file_operations tlclk_fops = { .read = tlclk_read, .write = tlclk_write, .open = tlclk_open, diff --git a/drivers/char/toshiba.c b/drivers/char/toshiba.c index e2fb234dee4..dd36fd04a84 100644 --- a/drivers/char/toshiba.c +++ b/drivers/char/toshiba.c @@ -92,7 +92,7 @@ static int tosh_ioctl(struct inode *, struct file *, unsigned int, unsigned long); -static struct file_operations tosh_fops = { +static const struct file_operations tosh_fops = { .owner = THIS_MODULE, .ioctl = tosh_ioctl, }; diff --git a/drivers/char/tpm/tpm_atmel.c b/drivers/char/tpm/tpm_atmel.c index 58a258cec15..ad8ffe49256 100644 --- a/drivers/char/tpm/tpm_atmel.c +++ b/drivers/char/tpm/tpm_atmel.c @@ -116,7 +116,7 @@ static u8 tpm_atml_status(struct tpm_chip *chip) return ioread8(chip->vendor.iobase + 1); } -static struct file_operations atmel_ops = { +static const struct file_operations atmel_ops = { .owner = THIS_MODULE, .llseek = no_llseek, .open = tpm_open, diff --git a/drivers/char/tpm/tpm_infineon.c b/drivers/char/tpm/tpm_infineon.c index adfff21beb2..1353b5a6bae 100644 --- a/drivers/char/tpm/tpm_infineon.c +++ b/drivers/char/tpm/tpm_infineon.c @@ -338,7 +338,7 @@ static struct attribute *inf_attrs[] = { static struct attribute_group inf_attr_grp = {.attrs = inf_attrs }; -static struct file_operations inf_ops = { +static const struct file_operations inf_ops = { .owner = THIS_MODULE, .llseek = no_llseek, .open = tpm_open, diff --git a/drivers/char/tpm/tpm_nsc.c b/drivers/char/tpm/tpm_nsc.c index 4c8bc06c7d9..26287aace87 100644 --- a/drivers/char/tpm/tpm_nsc.c +++ b/drivers/char/tpm/tpm_nsc.c @@ -226,7 +226,7 @@ static u8 tpm_nsc_status(struct tpm_chip *chip) return inb(chip->vendor.base + NSC_STATUS); } -static struct file_operations nsc_ops = { +static const struct file_operations nsc_ops = { .owner = THIS_MODULE, .llseek = no_llseek, .open = tpm_open, diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c index abb0f2aeae6..3232b193259 100644 --- a/drivers/char/tpm/tpm_tis.c +++ b/drivers/char/tpm/tpm_tis.c @@ -330,7 +330,7 @@ out_err: return rc; } -static struct file_operations tis_ops = { +static const struct file_operations tis_ops = { .owner = THIS_MODULE, .llseek = no_llseek, .open = tpm_open, diff --git a/drivers/char/tty_io.c b/drivers/char/tty_io.c index 615e934da05..bfdb90242a9 100644 --- a/drivers/char/tty_io.c +++ b/drivers/char/tty_io.c @@ -912,7 +912,7 @@ static int hung_up_tty_ioctl(struct inode * inode, struct file * file, return cmd == TIOCSPGRP ? -ENOTTY : -EIO; } -static struct file_operations tty_fops = { +static const struct file_operations tty_fops = { .llseek = no_llseek, .read = tty_read, .write = tty_write, @@ -924,7 +924,7 @@ static struct file_operations tty_fops = { }; #ifdef CONFIG_UNIX98_PTYS -static struct file_operations ptmx_fops = { +static const struct file_operations ptmx_fops = { .llseek = no_llseek, .read = tty_read, .write = tty_write, @@ -936,7 +936,7 @@ static struct file_operations ptmx_fops = { }; #endif -static struct file_operations console_fops = { +static const struct file_operations console_fops = { .llseek = no_llseek, .read = tty_read, .write = redirected_tty_write, @@ -947,7 +947,7 @@ static struct file_operations console_fops = { .fasync = tty_fasync, }; -static struct file_operations hung_up_tty_fops = { +static const struct file_operations hung_up_tty_fops = { .llseek = no_llseek, .read = hung_up_tty_read, .write = hung_up_tty_write, @@ -2336,7 +2336,7 @@ static int fionbio(struct file *file, int __user *p) static int tiocsctty(struct tty_struct *tty, int arg) { - task_t *p; + struct task_struct *p; if (current->signal->leader && (current->signal->session == tty->session)) diff --git a/drivers/char/vc_screen.c b/drivers/char/vc_screen.c index 45e9bd81bc0..a9247b5213d 100644 --- a/drivers/char/vc_screen.c +++ b/drivers/char/vc_screen.c @@ -465,7 +465,7 @@ vcs_open(struct inode *inode, struct file *filp) return 0; } -static struct file_operations vcs_fops = { +static const struct file_operations vcs_fops = { .llseek = vcs_lseek, .read = vcs_read, .write = vcs_write, diff --git a/drivers/char/viotape.c b/drivers/char/viotape.c index 7d42c8ec8db..b72b2049aaa 100644 --- a/drivers/char/viotape.c +++ b/drivers/char/viotape.c @@ -292,7 +292,7 @@ static int proc_viotape_open(struct inode *inode, struct file *file) return single_open(file, proc_viotape_show, NULL); } -static struct file_operations proc_viotape_operations = { +static const struct file_operations proc_viotape_operations = { .open = proc_viotape_open, .read = seq_read, .llseek = seq_lseek, diff --git a/drivers/char/vr41xx_giu.c b/drivers/char/vr41xx_giu.c index 073da48c092..1b9b1f1d4c4 100644 --- a/drivers/char/vr41xx_giu.c +++ b/drivers/char/vr41xx_giu.c @@ -605,7 +605,7 @@ static int gpio_release(struct inode *inode, struct file *file) return 0; } -static struct file_operations gpio_fops = { +static const struct file_operations gpio_fops = { .owner = THIS_MODULE, .read = gpio_read, .write = gpio_write, diff --git a/drivers/char/vt.c b/drivers/char/vt.c index 3ef823d7d25..da7e66a2a38 100644 --- a/drivers/char/vt.c +++ b/drivers/char/vt.c @@ -886,6 +886,7 @@ void vc_disallocate(unsigned int currcons) if (vc_cons_allocated(currcons)) { struct vc_data *vc = vc_cons[currcons].d; vc->vc_sw->con_deinit(vc); + module_put(vc->vc_sw->owner); if (vc->vc_kmalloced) kfree(vc->vc_screenbuf); if (currcons >= MIN_NR_CONSOLES) diff --git a/drivers/char/watchdog/acquirewdt.c b/drivers/char/watchdog/acquirewdt.c index 7289f4af93d..c77fe3cf285 100644 --- a/drivers/char/watchdog/acquirewdt.c +++ b/drivers/char/watchdog/acquirewdt.c @@ -231,7 +231,7 @@ static int acq_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations acq_fops = { +static const struct file_operations acq_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = acq_write, diff --git a/drivers/char/watchdog/advantechwdt.c b/drivers/char/watchdog/advantechwdt.c index 194a3fd36b9..8069be445ed 100644 --- a/drivers/char/watchdog/advantechwdt.c +++ b/drivers/char/watchdog/advantechwdt.c @@ -227,7 +227,7 @@ advwdt_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations advwdt_fops = { +static const struct file_operations advwdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = advwdt_write, diff --git a/drivers/char/watchdog/alim1535_wdt.c b/drivers/char/watchdog/alim1535_wdt.c index 8338ca300e2..c5c94e4c949 100644 --- a/drivers/char/watchdog/alim1535_wdt.c +++ b/drivers/char/watchdog/alim1535_wdt.c @@ -362,7 +362,7 @@ static int __init ali_find_watchdog(void) * Kernel Interfaces */ -static struct file_operations ali_fops = { +static const struct file_operations ali_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = ali_write, diff --git a/drivers/char/watchdog/alim7101_wdt.c b/drivers/char/watchdog/alim7101_wdt.c index c05ac188a4d..ffd7684f999 100644 --- a/drivers/char/watchdog/alim7101_wdt.c +++ b/drivers/char/watchdog/alim7101_wdt.c @@ -281,7 +281,7 @@ static int fop_ioctl(struct inode *inode, struct file *file, unsigned int cmd, u } } -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner= THIS_MODULE, .llseek= no_llseek, .write= fop_write, diff --git a/drivers/char/watchdog/at91_wdt.c b/drivers/char/watchdog/at91_wdt.c index f61dedc3c96..cc266715ea3 100644 --- a/drivers/char/watchdog/at91_wdt.c +++ b/drivers/char/watchdog/at91_wdt.c @@ -183,7 +183,7 @@ static ssize_t at91_wdt_write(struct file *file, const char *data, size_t len, l /* ......................................................................... */ -static struct file_operations at91wdt_fops = { +static const struct file_operations at91wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .ioctl = at91_wdt_ioctl, diff --git a/drivers/char/watchdog/booke_wdt.c b/drivers/char/watchdog/booke_wdt.c index 537f5c6729b..e3cefc538b4 100644 --- a/drivers/char/watchdog/booke_wdt.c +++ b/drivers/char/watchdog/booke_wdt.c @@ -145,7 +145,7 @@ static int booke_wdt_open (struct inode *inode, struct file *file) return 0; } -static struct file_operations booke_wdt_fops = { +static const struct file_operations booke_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = booke_wdt_write, diff --git a/drivers/char/watchdog/cpu5wdt.c b/drivers/char/watchdog/cpu5wdt.c index 3e8410b5a65..04c7e49918d 100644 --- a/drivers/char/watchdog/cpu5wdt.c +++ b/drivers/char/watchdog/cpu5wdt.c @@ -198,7 +198,7 @@ static ssize_t cpu5wdt_write(struct file *file, const char __user *buf, size_t c return count; } -static struct file_operations cpu5wdt_fops = { +static const struct file_operations cpu5wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .ioctl = cpu5wdt_ioctl, diff --git a/drivers/char/watchdog/ep93xx_wdt.c b/drivers/char/watchdog/ep93xx_wdt.c index 9021dbb7829..77c8a955ae9 100644 --- a/drivers/char/watchdog/ep93xx_wdt.c +++ b/drivers/char/watchdog/ep93xx_wdt.c @@ -187,7 +187,7 @@ static int ep93xx_wdt_release(struct inode *inode, struct file *file) return 0; } -static struct file_operations ep93xx_wdt_fops = { +static const struct file_operations ep93xx_wdt_fops = { .owner = THIS_MODULE, .write = ep93xx_wdt_write, .ioctl = ep93xx_wdt_ioctl, diff --git a/drivers/char/watchdog/eurotechwdt.c b/drivers/char/watchdog/eurotechwdt.c index ea670de4fab..62dbccb2f6d 100644 --- a/drivers/char/watchdog/eurotechwdt.c +++ b/drivers/char/watchdog/eurotechwdt.c @@ -356,7 +356,7 @@ static int eurwdt_notify_sys(struct notifier_block *this, unsigned long code, */ -static struct file_operations eurwdt_fops = { +static const struct file_operations eurwdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = eurwdt_write, diff --git a/drivers/char/watchdog/i6300esb.c b/drivers/char/watchdog/i6300esb.c index 93785f13242..870539eabbf 100644 --- a/drivers/char/watchdog/i6300esb.c +++ b/drivers/char/watchdog/i6300esb.c @@ -337,7 +337,7 @@ static int esb_notify_sys (struct notifier_block *this, unsigned long code, void * Kernel Interfaces */ -static struct file_operations esb_fops = { +static const struct file_operations esb_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = esb_write, diff --git a/drivers/char/watchdog/i8xx_tco.c b/drivers/char/watchdog/i8xx_tco.c index bfbdbbf3c2f..8385dd36eef 100644 --- a/drivers/char/watchdog/i8xx_tco.c +++ b/drivers/char/watchdog/i8xx_tco.c @@ -378,7 +378,7 @@ static int i8xx_tco_notify_sys (struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations i8xx_tco_fops = { +static const struct file_operations i8xx_tco_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = i8xx_tco_write, diff --git a/drivers/char/watchdog/ib700wdt.c b/drivers/char/watchdog/ib700wdt.c index a2e53c715b3..fd95f732779 100644 --- a/drivers/char/watchdog/ib700wdt.c +++ b/drivers/char/watchdog/ib700wdt.c @@ -255,7 +255,7 @@ ibwdt_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations ibwdt_fops = { +static const struct file_operations ibwdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = ibwdt_write, diff --git a/drivers/char/watchdog/ibmasr.c b/drivers/char/watchdog/ibmasr.c index b0741cbdc13..26ceee7a4df 100644 --- a/drivers/char/watchdog/ibmasr.c +++ b/drivers/char/watchdog/ibmasr.c @@ -322,7 +322,7 @@ static int asr_release(struct inode *inode, struct file *file) return 0; } -static struct file_operations asr_fops = { +static const struct file_operations asr_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = asr_write, diff --git a/drivers/char/watchdog/indydog.c b/drivers/char/watchdog/indydog.c index d387979b243..dacc1c20a31 100644 --- a/drivers/char/watchdog/indydog.c +++ b/drivers/char/watchdog/indydog.c @@ -154,7 +154,7 @@ static int indydog_notify_sys(struct notifier_block *this, unsigned long code, v return NOTIFY_DONE; } -static struct file_operations indydog_fops = { +static const struct file_operations indydog_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = indydog_write, diff --git a/drivers/char/watchdog/ixp2000_wdt.c b/drivers/char/watchdog/ixp2000_wdt.c index aa29a7d6875..692908819e2 100644 --- a/drivers/char/watchdog/ixp2000_wdt.c +++ b/drivers/char/watchdog/ixp2000_wdt.c @@ -168,7 +168,7 @@ ixp2000_wdt_release(struct inode *inode, struct file *file) } -static struct file_operations ixp2000_wdt_fops = +static const struct file_operations ixp2000_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/ixp4xx_wdt.c b/drivers/char/watchdog/ixp4xx_wdt.c index e6a3fe83fa0..9db5cf2c38c 100644 --- a/drivers/char/watchdog/ixp4xx_wdt.c +++ b/drivers/char/watchdog/ixp4xx_wdt.c @@ -162,7 +162,7 @@ ixp4xx_wdt_release(struct inode *inode, struct file *file) } -static struct file_operations ixp4xx_wdt_fops = +static const struct file_operations ixp4xx_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/machzwd.c b/drivers/char/watchdog/machzwd.c index b67b4878ae0..23734e07fb2 100644 --- a/drivers/char/watchdog/machzwd.c +++ b/drivers/char/watchdog/machzwd.c @@ -388,7 +388,7 @@ static int zf_notify_sys(struct notifier_block *this, unsigned long code, -static struct file_operations zf_fops = { +static const struct file_operations zf_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = zf_write, diff --git a/drivers/char/watchdog/mixcomwd.c b/drivers/char/watchdog/mixcomwd.c index 433c27f9815..ae943324d25 100644 --- a/drivers/char/watchdog/mixcomwd.c +++ b/drivers/char/watchdog/mixcomwd.c @@ -190,7 +190,7 @@ static int mixcomwd_ioctl(struct inode *inode, struct file *file, return 0; } -static struct file_operations mixcomwd_fops= +static const struct file_operations mixcomwd_fops= { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/mpc83xx_wdt.c b/drivers/char/watchdog/mpc83xx_wdt.c index dac1381af36..a480903ee1a 100644 --- a/drivers/char/watchdog/mpc83xx_wdt.c +++ b/drivers/char/watchdog/mpc83xx_wdt.c @@ -129,7 +129,7 @@ static int mpc83xx_wdt_ioctl(struct inode *inode, struct file *file, } } -static struct file_operations mpc83xx_wdt_fops = { +static const struct file_operations mpc83xx_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = mpc83xx_wdt_write, diff --git a/drivers/char/watchdog/mpc8xx_wdt.c b/drivers/char/watchdog/mpc8xx_wdt.c index 11f0ccd4c4d..35dd9e6e114 100644 --- a/drivers/char/watchdog/mpc8xx_wdt.c +++ b/drivers/char/watchdog/mpc8xx_wdt.c @@ -132,7 +132,7 @@ static int mpc8xx_wdt_ioctl(struct inode *inode, struct file *file, return 0; } -static struct file_operations mpc8xx_wdt_fops = { +static const struct file_operations mpc8xx_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = mpc8xx_wdt_write, diff --git a/drivers/char/watchdog/mpcore_wdt.c b/drivers/char/watchdog/mpcore_wdt.c index c2d492c852f..54b3c56ead0 100644 --- a/drivers/char/watchdog/mpcore_wdt.c +++ b/drivers/char/watchdog/mpcore_wdt.c @@ -297,7 +297,7 @@ static void mpcore_wdt_shutdown(struct platform_device *dev) /* * Kernel Interfaces */ -static struct file_operations mpcore_wdt_fops = { +static const struct file_operations mpcore_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = mpcore_wdt_write, diff --git a/drivers/char/watchdog/mv64x60_wdt.c b/drivers/char/watchdog/mv64x60_wdt.c index 20a6cbb0fbb..5c8fab345b4 100644 --- a/drivers/char/watchdog/mv64x60_wdt.c +++ b/drivers/char/watchdog/mv64x60_wdt.c @@ -166,7 +166,7 @@ static int mv64x60_wdt_ioctl(struct inode *inode, struct file *file, return 0; } -static struct file_operations mv64x60_wdt_fops = { +static const struct file_operations mv64x60_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = mv64x60_wdt_write, diff --git a/drivers/char/watchdog/pcwd.c b/drivers/char/watchdog/pcwd.c index 6d44ca68312..cd7d1b6a5d9 100644 --- a/drivers/char/watchdog/pcwd.c +++ b/drivers/char/watchdog/pcwd.c @@ -740,7 +740,7 @@ static int pcwd_notify_sys(struct notifier_block *this, unsigned long code, void * Kernel Interfaces */ -static struct file_operations pcwd_fops = { +static const struct file_operations pcwd_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = pcwd_write, @@ -755,7 +755,7 @@ static struct miscdevice pcwd_miscdev = { .fops = &pcwd_fops, }; -static struct file_operations pcwd_temp_fops = { +static const struct file_operations pcwd_temp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = pcwd_temp_read, diff --git a/drivers/char/watchdog/pcwd_pci.c b/drivers/char/watchdog/pcwd_pci.c index 1f40ecefbf7..c7cfd6dbfe1 100644 --- a/drivers/char/watchdog/pcwd_pci.c +++ b/drivers/char/watchdog/pcwd_pci.c @@ -625,7 +625,7 @@ static int pcipcwd_notify_sys(struct notifier_block *this, unsigned long code, v * Kernel Interfaces */ -static struct file_operations pcipcwd_fops = { +static const struct file_operations pcipcwd_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = pcipcwd_write, @@ -640,7 +640,7 @@ static struct miscdevice pcipcwd_miscdev = { .fops = &pcipcwd_fops, }; -static struct file_operations pcipcwd_temp_fops = { +static const struct file_operations pcipcwd_temp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = pcipcwd_temp_read, diff --git a/drivers/char/watchdog/pcwd_usb.c b/drivers/char/watchdog/pcwd_usb.c index 92bf8c1a0f0..b7ae73dcdd0 100644 --- a/drivers/char/watchdog/pcwd_usb.c +++ b/drivers/char/watchdog/pcwd_usb.c @@ -523,7 +523,7 @@ static int usb_pcwd_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations usb_pcwd_fops = { +static const struct file_operations usb_pcwd_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = usb_pcwd_write, @@ -538,7 +538,7 @@ static struct miscdevice usb_pcwd_miscdev = { .fops = &usb_pcwd_fops, }; -static struct file_operations usb_pcwd_temperature_fops = { +static const struct file_operations usb_pcwd_temperature_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = usb_pcwd_temperature_read, diff --git a/drivers/char/watchdog/s3c2410_wdt.c b/drivers/char/watchdog/s3c2410_wdt.c index f267dad2607..be978e8ed75 100644 --- a/drivers/char/watchdog/s3c2410_wdt.c +++ b/drivers/char/watchdog/s3c2410_wdt.c @@ -319,7 +319,7 @@ static int s3c2410wdt_ioctl(struct inode *inode, struct file *file, /* kernel interface */ -static struct file_operations s3c2410wdt_fops = { +static const struct file_operations s3c2410wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = s3c2410wdt_write, diff --git a/drivers/char/watchdog/sa1100_wdt.c b/drivers/char/watchdog/sa1100_wdt.c index b22e95c5470..1fc16d99578 100644 --- a/drivers/char/watchdog/sa1100_wdt.c +++ b/drivers/char/watchdog/sa1100_wdt.c @@ -135,7 +135,7 @@ static int sa1100dog_ioctl(struct inode *inode, struct file *file, return ret; } -static struct file_operations sa1100dog_fops = +static const struct file_operations sa1100dog_fops = { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/sbc60xxwdt.c b/drivers/char/watchdog/sbc60xxwdt.c index ed0bd55fbfc..4663c2fd53c 100644 --- a/drivers/char/watchdog/sbc60xxwdt.c +++ b/drivers/char/watchdog/sbc60xxwdt.c @@ -282,7 +282,7 @@ static int fop_ioctl(struct inode *inode, struct file *file, unsigned int cmd, } } -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = fop_write, diff --git a/drivers/char/watchdog/sbc8360.c b/drivers/char/watchdog/sbc8360.c index 6562aa910ac..1035be5b501 100644 --- a/drivers/char/watchdog/sbc8360.c +++ b/drivers/char/watchdog/sbc8360.c @@ -305,7 +305,7 @@ static int sbc8360_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations sbc8360_fops = { +static const struct file_operations sbc8360_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = sbc8360_write, diff --git a/drivers/char/watchdog/sbc_epx_c3.c b/drivers/char/watchdog/sbc_epx_c3.c index 09867fadc72..bfc475dabe6 100644 --- a/drivers/char/watchdog/sbc_epx_c3.c +++ b/drivers/char/watchdog/sbc_epx_c3.c @@ -154,7 +154,7 @@ static int epx_c3_notify_sys(struct notifier_block *this, unsigned long code, return NOTIFY_DONE; } -static struct file_operations epx_c3_fops = { +static const struct file_operations epx_c3_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = epx_c3_write, diff --git a/drivers/char/watchdog/sc1200wdt.c b/drivers/char/watchdog/sc1200wdt.c index 78ef6333c18..7c3cf293a5a 100644 --- a/drivers/char/watchdog/sc1200wdt.c +++ b/drivers/char/watchdog/sc1200wdt.c @@ -292,7 +292,7 @@ static struct notifier_block sc1200wdt_notifier = .notifier_call = sc1200wdt_notify_sys, }; -static struct file_operations sc1200wdt_fops = +static const struct file_operations sc1200wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/sc520_wdt.c b/drivers/char/watchdog/sc520_wdt.c index 4ee9974ad8c..2c7c9db71be 100644 --- a/drivers/char/watchdog/sc520_wdt.c +++ b/drivers/char/watchdog/sc520_wdt.c @@ -336,7 +336,7 @@ static int fop_ioctl(struct inode *inode, struct file *file, unsigned int cmd, } } -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = fop_write, diff --git a/drivers/char/watchdog/scx200_wdt.c b/drivers/char/watchdog/scx200_wdt.c index c0b4754e8de..c561299a553 100644 --- a/drivers/char/watchdog/scx200_wdt.c +++ b/drivers/char/watchdog/scx200_wdt.c @@ -194,7 +194,7 @@ static int scx200_wdt_ioctl(struct inode *inode, struct file *file, } } -static struct file_operations scx200_wdt_fops = { +static const struct file_operations scx200_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = scx200_wdt_write, diff --git a/drivers/char/watchdog/shwdt.c b/drivers/char/watchdog/shwdt.c index 803701b675c..1355038f104 100644 --- a/drivers/char/watchdog/shwdt.c +++ b/drivers/char/watchdog/shwdt.c @@ -344,7 +344,7 @@ static int sh_wdt_notify_sys(struct notifier_block *this, return NOTIFY_DONE; } -static struct file_operations sh_wdt_fops = { +static const struct file_operations sh_wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = sh_wdt_write, diff --git a/drivers/char/watchdog/softdog.c b/drivers/char/watchdog/softdog.c index 79ce5c65542..ef8da517545 100644 --- a/drivers/char/watchdog/softdog.c +++ b/drivers/char/watchdog/softdog.c @@ -243,7 +243,7 @@ static int softdog_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations softdog_fops = { +static const struct file_operations softdog_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = softdog_write, diff --git a/drivers/char/watchdog/w83627hf_wdt.c b/drivers/char/watchdog/w83627hf_wdt.c index d15ca9a3986..13f16d41c2f 100644 --- a/drivers/char/watchdog/w83627hf_wdt.c +++ b/drivers/char/watchdog/w83627hf_wdt.c @@ -274,7 +274,7 @@ wdt_notify_sys(struct notifier_block *this, unsigned long code, * Kernel Interfaces */ -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = wdt_write, diff --git a/drivers/char/watchdog/w83877f_wdt.c b/drivers/char/watchdog/w83877f_wdt.c index 52a8bd0a598..ccf6c091594 100644 --- a/drivers/char/watchdog/w83877f_wdt.c +++ b/drivers/char/watchdog/w83877f_wdt.c @@ -299,7 +299,7 @@ static int fop_ioctl(struct inode *inode, struct file *file, unsigned int cmd, } } -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = fop_write, diff --git a/drivers/char/watchdog/w83977f_wdt.c b/drivers/char/watchdog/w83977f_wdt.c index c31849e4c5c..98f4e17db70 100644 --- a/drivers/char/watchdog/w83977f_wdt.c +++ b/drivers/char/watchdog/w83977f_wdt.c @@ -449,7 +449,7 @@ static int wdt_notify_sys(struct notifier_block *this, unsigned long code, return NOTIFY_DONE; } -static struct file_operations wdt_fops= +static const struct file_operations wdt_fops= { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/wafer5823wdt.c b/drivers/char/watchdog/wafer5823wdt.c index 7cf6c9bbf48..2bb6a9d6ad2 100644 --- a/drivers/char/watchdog/wafer5823wdt.c +++ b/drivers/char/watchdog/wafer5823wdt.c @@ -222,7 +222,7 @@ static int wafwdt_notify_sys(struct notifier_block *this, unsigned long code, vo * Kernel Interfaces */ -static struct file_operations wafwdt_fops = { +static const struct file_operations wafwdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = wafwdt_write, diff --git a/drivers/char/watchdog/wdrtas.c b/drivers/char/watchdog/wdrtas.c index 3a462c34b92..5c38cdf4173 100644 --- a/drivers/char/watchdog/wdrtas.c +++ b/drivers/char/watchdog/wdrtas.c @@ -520,7 +520,7 @@ wdrtas_reboot(struct notifier_block *this, unsigned long code, void *ptr) /*** initialization stuff */ -static struct file_operations wdrtas_fops = { +static const struct file_operations wdrtas_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = wdrtas_write, @@ -535,7 +535,7 @@ static struct miscdevice wdrtas_miscdev = { .fops = &wdrtas_fops, }; -static struct file_operations wdrtas_temp_fops = { +static const struct file_operations wdrtas_temp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = wdrtas_temp_read, diff --git a/drivers/char/watchdog/wdt.c b/drivers/char/watchdog/wdt.c index a1d972c8f44..70be81e39a6 100644 --- a/drivers/char/watchdog/wdt.c +++ b/drivers/char/watchdog/wdt.c @@ -494,7 +494,7 @@ static int wdt_notify_sys(struct notifier_block *this, unsigned long code, */ -static struct file_operations wdt_fops = { +static const struct file_operations wdt_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = wdt_write, @@ -510,7 +510,7 @@ static struct miscdevice wdt_miscdev = { }; #ifdef CONFIG_WDT_501 -static struct file_operations wdt_temp_fops = { +static const struct file_operations wdt_temp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = wdt_temp_read, diff --git a/drivers/char/watchdog/wdt285.c b/drivers/char/watchdog/wdt285.c index 52825a1f177..6555fb844f2 100644 --- a/drivers/char/watchdog/wdt285.c +++ b/drivers/char/watchdog/wdt285.c @@ -178,7 +178,7 @@ watchdog_ioctl(struct inode *inode, struct file *file, unsigned int cmd, return ret; } -static struct file_operations watchdog_fops = { +static const struct file_operations watchdog_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = watchdog_write, diff --git a/drivers/char/watchdog/wdt977.c b/drivers/char/watchdog/wdt977.c index 3cde2b9bb76..a0935bc775f 100644 --- a/drivers/char/watchdog/wdt977.c +++ b/drivers/char/watchdog/wdt977.c @@ -418,7 +418,7 @@ static int wdt977_notify_sys(struct notifier_block *this, unsigned long code, return NOTIFY_DONE; } -static struct file_operations wdt977_fops= +static const struct file_operations wdt977_fops= { .owner = THIS_MODULE, .llseek = no_llseek, diff --git a/drivers/char/watchdog/wdt_pci.c b/drivers/char/watchdog/wdt_pci.c index 7529ecdbaba..5918ca2c9c3 100644 --- a/drivers/char/watchdog/wdt_pci.c +++ b/drivers/char/watchdog/wdt_pci.c @@ -543,7 +543,7 @@ static int wdtpci_notify_sys(struct notifier_block *this, unsigned long code, */ -static struct file_operations wdtpci_fops = { +static const struct file_operations wdtpci_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .write = wdtpci_write, @@ -559,7 +559,7 @@ static struct miscdevice wdtpci_miscdev = { }; #ifdef CONFIG_WDT_501_PCI -static struct file_operations wdtpci_temp_fops = { +static const struct file_operations wdtpci_temp_fops = { .owner = THIS_MODULE, .llseek = no_llseek, .read = wdtpci_temp_read, diff --git a/drivers/ide/ide-floppy.c b/drivers/ide/ide-floppy.c index 6ca3476d02c..adbe9f76a50 100644 --- a/drivers/ide/ide-floppy.c +++ b/drivers/ide/ide-floppy.c @@ -838,7 +838,7 @@ static ide_startstop_t idefloppy_pc_intr (ide_drive_t *drive) "transferred\n", pc->actually_transferred); clear_bit(PC_DMA_IN_PROGRESS, &pc->flags); - local_irq_enable(); + local_irq_enable_in_hardirq(); if (status.b.check || test_bit(PC_DMA_ERROR, &pc->flags)) { /* Error detected */ diff --git a/drivers/ide/ide-io.c b/drivers/ide/ide-io.c index 7dba9992ad3..fb6795236e7 100644 --- a/drivers/ide/ide-io.c +++ b/drivers/ide/ide-io.c @@ -693,7 +693,7 @@ static ide_startstop_t drive_cmd_intr (ide_drive_t *drive) u8 stat = hwif->INB(IDE_STATUS_REG); int retries = 10; - local_irq_enable(); + local_irq_enable_in_hardirq(); if ((stat & DRQ_STAT) && args && args[3]) { u8 io_32bit = drive->io_32bit; drive->io_32bit = 0; @@ -1286,7 +1286,7 @@ static void ide_do_request (ide_hwgroup_t *hwgroup, int masked_irq) if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq) disable_irq_nosync(hwif->irq); spin_unlock(&ide_lock); - local_irq_enable(); + local_irq_enable_in_hardirq(); /* allow other IRQs while we start this request */ startstop = start_request(drive, rq); spin_lock_irq(&ide_lock); @@ -1631,7 +1631,7 @@ irqreturn_t ide_intr (int irq, void *dev_id, struct pt_regs *regs) spin_unlock(&ide_lock); if (drive->unmask) - local_irq_enable(); + local_irq_enable_in_hardirq(); /* service this interrupt, may set handler for next interrupt */ startstop = handler(drive); spin_lock_irq(&ide_lock); @@ -1705,7 +1705,7 @@ int ide_do_drive_cmd (ide_drive_t *drive, struct request *rq, ide_action_t actio { unsigned long flags; ide_hwgroup_t *hwgroup = HWGROUP(drive); - DECLARE_COMPLETION(wait); + DECLARE_COMPLETION_ONSTACK(wait); int where = ELEVATOR_INSERT_BACK, err; int must_wait = (action == ide_wait || action == ide_head_wait); diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c index 04547eb0833..97a9244312f 100644 --- a/drivers/ide/ide-taskfile.c +++ b/drivers/ide/ide-taskfile.c @@ -222,7 +222,7 @@ ide_startstop_t task_no_data_intr (ide_drive_t *drive) ide_hwif_t *hwif = HWIF(drive); u8 stat; - local_irq_enable(); + local_irq_enable_in_hardirq(); if (!OK_STAT(stat = hwif->INB(IDE_STATUS_REG),READY_STAT,BAD_STAT)) { return ide_error(drive, "task_no_data_intr", stat); /* calls ide_end_drive_cmd */ diff --git a/drivers/ieee1394/hosts.c b/drivers/ieee1394/hosts.c index 2c669287f5b..4feead4a35c 100644 --- a/drivers/ieee1394/hosts.c +++ b/drivers/ieee1394/hosts.c @@ -107,6 +107,14 @@ static int alloc_hostnum_cb(struct hpsb_host *host, void *__data) */ static DEFINE_MUTEX(host_num_alloc); +/* + * The pending_packet_queue is special in that it's processed + * from hardirq context too (such as hpsb_bus_reset()). Hence + * split the lock class from the usual networking skb-head + * lock class by using a separate key for it: + */ +static struct lock_class_key pending_packet_queue_key; + struct hpsb_host *hpsb_alloc_host(struct hpsb_host_driver *drv, size_t extra, struct device *dev) { @@ -128,6 +136,8 @@ struct hpsb_host *hpsb_alloc_host(struct hpsb_host_driver *drv, size_t extra, h->driver = drv; skb_queue_head_init(&h->pending_packet_queue); + lockdep_set_class(&h->pending_packet_queue.lock, + &pending_packet_queue_key); INIT_LIST_HEAD(&h->addr_space); for (i = 2; i < 16; i++) diff --git a/drivers/input/serio/i8042-sparcio.h b/drivers/input/serio/i8042-sparcio.h index 7d9fafea961..54adba2d8ed 100644 --- a/drivers/input/serio/i8042-sparcio.h +++ b/drivers/input/serio/i8042-sparcio.h @@ -88,7 +88,7 @@ static struct of_device_id sparc_i8042_match[] = { }, {}, }; -MODULE_DEVICE_TABLE(of, i8042_match); +MODULE_DEVICE_TABLE(of, sparc_i8042_match); static struct of_platform_driver sparc_i8042_driver = { .name = "i8042", diff --git a/drivers/input/serio/libps2.c b/drivers/input/serio/libps2.c index 79c97f94bcb..61a6f977846 100644 --- a/drivers/input/serio/libps2.c +++ b/drivers/input/serio/libps2.c @@ -177,7 +177,7 @@ int ps2_command(struct ps2dev *ps2dev, unsigned char *param, int command) return -1; } - mutex_lock(&ps2dev->cmd_mutex); + mutex_lock_nested(&ps2dev->cmd_mutex, SINGLE_DEPTH_NESTING); serio_pause_rx(ps2dev->serio); ps2dev->flags = command == PS2_CMD_GETID ? PS2_FLAG_WAITID : 0; diff --git a/drivers/md/md.c b/drivers/md/md.c index 2fe32c26192..e4e161372a3 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -1404,7 +1404,7 @@ static int lock_rdev(mdk_rdev_t *rdev, dev_t dev) struct block_device *bdev; char b[BDEVNAME_SIZE]; - bdev = open_by_devnum(dev, FMODE_READ|FMODE_WRITE); + bdev = open_partition_by_devnum(dev, FMODE_READ|FMODE_WRITE); if (IS_ERR(bdev)) { printk(KERN_ERR "md: could not open %s.\n", __bdevname(dev, b)); @@ -1414,7 +1414,7 @@ static int lock_rdev(mdk_rdev_t *rdev, dev_t dev) if (err) { printk(KERN_ERR "md: could not bd_claim %s.\n", bdevname(bdev, b)); - blkdev_put(bdev); + blkdev_put_partition(bdev); return err; } rdev->bdev = bdev; @@ -1428,7 +1428,7 @@ static void unlock_rdev(mdk_rdev_t *rdev) if (!bdev) MD_BUG(); bd_release(bdev); - blkdev_put(bdev); + blkdev_put_partition(bdev); } void md_autodetect_dev(dev_t dev); diff --git a/drivers/mmc/mmc.c b/drivers/mmc/mmc.c index 247ff2f23ac..33525bdf2ab 100644 --- a/drivers/mmc/mmc.c +++ b/drivers/mmc/mmc.c @@ -128,7 +128,7 @@ static void mmc_wait_done(struct mmc_request *mrq) int mmc_wait_for_req(struct mmc_host *host, struct mmc_request *mrq) { - DECLARE_COMPLETION(complete); + DECLARE_COMPLETION_ONSTACK(complete); mrq->done_data = &complete; mrq->done = mmc_wait_done; diff --git a/drivers/mmc/sdhci.c b/drivers/mmc/sdhci.c index 74134699cce..893319108ba 100644 --- a/drivers/mmc/sdhci.c +++ b/drivers/mmc/sdhci.c @@ -32,9 +32,39 @@ static unsigned int debug_nodma = 0; static unsigned int debug_forcedma = 0; static unsigned int debug_quirks = 0; +#define SDHCI_QUIRK_CLOCK_BEFORE_RESET (1<<0) +#define SDHCI_QUIRK_FORCE_DMA (1<<1) + static const struct pci_device_id pci_ids[] __devinitdata = { - /* handle any SD host controller */ - {PCI_DEVICE_CLASS((PCI_CLASS_SYSTEM_SDHCI << 8), 0xFFFF00)}, + { + .vendor = PCI_VENDOR_ID_RICOH, + .device = PCI_DEVICE_ID_RICOH_R5C822, + .subvendor = PCI_VENDOR_ID_IBM, + .subdevice = PCI_ANY_ID, + .driver_data = SDHCI_QUIRK_CLOCK_BEFORE_RESET | + SDHCI_QUIRK_FORCE_DMA, + }, + + { + .vendor = PCI_VENDOR_ID_RICOH, + .device = PCI_DEVICE_ID_RICOH_R5C822, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .driver_data = SDHCI_QUIRK_FORCE_DMA, + }, + + { + .vendor = PCI_VENDOR_ID_TI, + .device = PCI_DEVICE_ID_TI_XX21_XX11_SD, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .driver_data = SDHCI_QUIRK_FORCE_DMA, + }, + + { /* Generic SD host controller */ + PCI_DEVICE_CLASS((PCI_CLASS_SYSTEM_SDHCI << 8), 0xFFFF00) + }, + { /* end: all zeroes */ }, }; @@ -440,9 +470,7 @@ static void sdhci_finish_data(struct sdhci_host *host) "though there were blocks left. Please report this " "to " BUGMAIL ".\n", mmc_hostname(host->mmc)); data->error = MMC_ERR_FAILED; - } - - if (host->size != 0) { + } else if (host->size != 0) { printk(KERN_ERR "%s: %d bytes were left untransferred. " "Please report this to " BUGMAIL ".\n", mmc_hostname(host->mmc), host->size); @@ -808,6 +836,19 @@ static void sdhci_tasklet_finish(unsigned long param) if ((mrq->cmd->error != MMC_ERR_NONE) || (mrq->data && ((mrq->data->error != MMC_ERR_NONE) || (mrq->data->stop && (mrq->data->stop->error != MMC_ERR_NONE))))) { + + /* Some controllers need this kick or reset won't work here */ + if (host->chip->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET) { + unsigned int clock; + + /* This is to force an update */ + clock = host->clock; + host->clock = 0; + sdhci_set_clock(host, clock); + } + + /* Spec says we should do both at the same time, but Ricoh + controllers do not like that. */ sdhci_reset(host, SDHCI_RESET_CMD); sdhci_reset(host, SDHCI_RESET_DATA); } @@ -1165,7 +1206,9 @@ static int __devinit sdhci_probe_slot(struct pci_dev *pdev, int slot) else if (debug_forcedma) { DBG("DMA forced on\n"); host->flags |= SDHCI_USE_DMA; - } else if ((pdev->class & 0x0000FF) != PCI_SDHCI_IFDMA) + } else if (chip->quirks & SDHCI_QUIRK_FORCE_DMA) + host->flags |= SDHCI_USE_DMA; + else if ((pdev->class & 0x0000FF) != PCI_SDHCI_IFDMA) DBG("Controller doesn't have DMA interface\n"); else if (!(caps & SDHCI_CAN_DO_DMA)) DBG("Controller doesn't have DMA capability\n"); diff --git a/drivers/net/3c59x.c b/drivers/net/3c59x.c index 8ab03b4a885..2819de79442 100644 --- a/drivers/net/3c59x.c +++ b/drivers/net/3c59x.c @@ -1897,7 +1897,7 @@ vortex_timer(unsigned long data) printk(KERN_DEBUG "dev->watchdog_timeo=%d\n", dev->watchdog_timeo); } - disable_irq(dev->irq); + disable_irq_lockdep(dev->irq); old_window = ioread16(ioaddr + EL3_CMD) >> 13; EL3WINDOW(4); media_status = ioread16(ioaddr + Wn4_Media); @@ -1978,7 +1978,7 @@ leave_media_alone: dev->name, media_tbl[dev->if_port].name); EL3WINDOW(old_window); - enable_irq(dev->irq); + enable_irq_lockdep(dev->irq); mod_timer(&vp->timer, RUN_AT(next_tick)); if (vp->deferred) iowrite16(FakeIntr, ioaddr + EL3_CMD); diff --git a/drivers/net/8390.c b/drivers/net/8390.c index 86be96af9c8..d2935ae3981 100644 --- a/drivers/net/8390.c +++ b/drivers/net/8390.c @@ -249,7 +249,7 @@ void ei_tx_timeout(struct net_device *dev) /* Ugly but a reset can be slow, yet must be protected */ - disable_irq_nosync(dev->irq); + disable_irq_nosync_lockdep(dev->irq); spin_lock(&ei_local->page_lock); /* Try to restart the card. Perhaps the user has fixed something. */ @@ -257,7 +257,7 @@ void ei_tx_timeout(struct net_device *dev) NS8390_init(dev, 1); spin_unlock(&ei_local->page_lock); - enable_irq(dev->irq); + enable_irq_lockdep(dev->irq); netif_wake_queue(dev); } diff --git a/drivers/net/forcedeth.c b/drivers/net/forcedeth.c index 3c90003f423..037d870712f 100644 --- a/drivers/net/forcedeth.c +++ b/drivers/net/forcedeth.c @@ -2735,21 +2735,21 @@ static void nv_do_nic_poll(unsigned long data) if (!using_multi_irqs(dev)) { if (np->msi_flags & NV_MSI_X_ENABLED) - disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector); + disable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector); else - disable_irq(dev->irq); + disable_irq_lockdep(dev->irq); mask = np->irqmask; } else { if (np->nic_poll_irq & NVREG_IRQ_RX_ALL) { - disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector); + disable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector); mask |= NVREG_IRQ_RX_ALL; } if (np->nic_poll_irq & NVREG_IRQ_TX_ALL) { - disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector); + disable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector); mask |= NVREG_IRQ_TX_ALL; } if (np->nic_poll_irq & NVREG_IRQ_OTHER) { - disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector); + disable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector); mask |= NVREG_IRQ_OTHER; } } @@ -2761,23 +2761,23 @@ static void nv_do_nic_poll(unsigned long data) pci_push(base); if (!using_multi_irqs(dev)) { - nv_nic_irq((int) 0, (void *) data, (struct pt_regs *) NULL); + nv_nic_irq(0, dev, NULL); if (np->msi_flags & NV_MSI_X_ENABLED) - enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector); + enable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector); else - enable_irq(dev->irq); + enable_irq_lockdep(dev->irq); } else { if (np->nic_poll_irq & NVREG_IRQ_RX_ALL) { - nv_nic_irq_rx((int) 0, (void *) data, (struct pt_regs *) NULL); - enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector); + nv_nic_irq_rx(0, dev, NULL); + enable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector); } if (np->nic_poll_irq & NVREG_IRQ_TX_ALL) { - nv_nic_irq_tx((int) 0, (void *) data, (struct pt_regs *) NULL); - enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector); + nv_nic_irq_tx(0, dev, NULL); + enable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector); } if (np->nic_poll_irq & NVREG_IRQ_OTHER) { - nv_nic_irq_other((int) 0, (void *) data, (struct pt_regs *) NULL); - enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector); + nv_nic_irq_other(0, dev, NULL); + enable_irq_lockdep(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector); } } } diff --git a/drivers/net/wireless/hostap/hostap_hw.c b/drivers/net/wireless/hostap/hostap_hw.c index b764cfda6e8..dafaa5ff5aa 100644 --- a/drivers/net/wireless/hostap/hostap_hw.c +++ b/drivers/net/wireless/hostap/hostap_hw.c @@ -3095,6 +3095,14 @@ static void prism2_clear_set_tim_queue(local_info_t *local) } +/* + * HostAP uses two layers of net devices, where the inner + * layer gets called all the time from the outer layer. + * This is a natural nesting, which needs a split lock type. + */ +static struct lock_class_key hostap_netdev_xmit_lock_key; + + static struct net_device * prism2_init_local_data(struct prism2_helper_functions *funcs, int card_idx, struct device *sdev) @@ -3259,6 +3267,8 @@ while (0) SET_NETDEV_DEV(dev, sdev); if (ret >= 0) ret = register_netdevice(dev); + + lockdep_set_class(&dev->_xmit_lock, &hostap_netdev_xmit_lock_key); rtnl_unlock(); if (ret < 0) { printk(KERN_WARNING "%s: register netdevice failed!\n", diff --git a/drivers/pnp/pnpacpi/rsparser.c b/drivers/pnp/pnpacpi/rsparser.c index 3a4a644c268..21226888185 100644 --- a/drivers/pnp/pnpacpi/rsparser.c +++ b/drivers/pnp/pnpacpi/rsparser.c @@ -74,7 +74,7 @@ static void decode_irq_flags(int flag, int *triggering, int *polarity) static void pnpacpi_parse_allocated_irqresource(struct pnp_resource_table *res, u32 gsi, - int triggering, int polarity) + int triggering, int polarity, int shareable) { int i = 0; int irq; @@ -95,6 +95,9 @@ pnpacpi_parse_allocated_irqresource(struct pnp_resource_table *res, u32 gsi, return; } + if (shareable) + res->irq_resource[i].flags |= IORESOURCE_IRQ_SHAREABLE; + res->irq_resource[i].start = irq; res->irq_resource[i].end = irq; pcibios_penalize_isa_irq(irq, 1); @@ -194,7 +197,8 @@ static acpi_status pnpacpi_allocated_resource(struct acpi_resource *res, pnpacpi_parse_allocated_irqresource(res_table, res->data.irq.interrupts[i], res->data.irq.triggering, - res->data.irq.polarity); + res->data.irq.polarity, + res->data.irq.sharable); } break; @@ -255,7 +259,8 @@ static acpi_status pnpacpi_allocated_resource(struct acpi_resource *res, pnpacpi_parse_allocated_irqresource(res_table, res->data.extended_irq.interrupts[i], res->data.extended_irq.triggering, - res->data.extended_irq.polarity); + res->data.extended_irq.polarity, + res->data.extended_irq.sharable); } break; diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 4138564402b..985d1613baa 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -383,6 +383,7 @@ void sclp_sync_wait(void) { unsigned long psw_mask; + unsigned long flags; unsigned long cr0, cr0_sync; u64 timeout; @@ -395,9 +396,11 @@ sclp_sync_wait(void) sclp_tod_from_jiffies(sclp_request_timer.expires - jiffies); } + local_irq_save(flags); /* Prevent bottom half from executing once we force interrupts open */ local_bh_disable(); /* Enable service-signal interruption, disable timer interrupts */ + trace_hardirqs_on(); __ctl_store(cr0, 0, 0); cr0_sync = cr0; cr0_sync |= 0x00000200; @@ -415,11 +418,10 @@ sclp_sync_wait(void) barrier(); cpu_relax(); } - /* Restore interrupt settings */ - asm volatile ("SSM 0(%0)" - : : "a" (&psw_mask) : "memory"); + local_irq_disable(); __ctl_load(cr0, 0, 0); - __local_bh_enable(); + _local_bh_enable(); + local_irq_restore(flags); } EXPORT_SYMBOL(sclp_sync_wait); diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index a3423267467..6fec90eab00 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -147,7 +147,7 @@ cio_tpi(void) sch->driver->irq(&sch->dev); spin_unlock(&sch->lock); irq_exit (); - __local_bh_enable(); + _local_bh_enable(); return 1; } diff --git a/drivers/s390/net/qeth_main.c b/drivers/s390/net/qeth_main.c index 36733b9823c..8e8963f1573 100644 --- a/drivers/s390/net/qeth_main.c +++ b/drivers/s390/net/qeth_main.c @@ -84,6 +84,8 @@ static debug_info_t *qeth_dbf_qerr = NULL; DEFINE_PER_CPU(char[256], qeth_dbf_txt_buf); +static struct lock_class_key qdio_out_skb_queue_key; + /** * some more definitions and declarations */ @@ -3229,6 +3231,9 @@ qeth_alloc_qdio_buffers(struct qeth_card *card) &card->qdio.out_qs[i]->qdio_bufs[j]; skb_queue_head_init(&card->qdio.out_qs[i]->bufs[j]. skb_list); + lockdep_set_class( + &card->qdio.out_qs[i]->bufs[j].skb_list.lock, + &qdio_out_skb_queue_key); INIT_LIST_HEAD(&card->qdio.out_qs[i]->bufs[j].ctx_list); } } @@ -5272,6 +5277,7 @@ qeth_free_vlan_buffer(struct qeth_card *card, struct qeth_qdio_out_buffer *buf, struct sk_buff_head tmp_list; skb_queue_head_init(&tmp_list); + lockdep_set_class(&tmp_list.lock, &qdio_out_skb_queue_key); for(i = 0; i < QETH_MAX_BUFFER_ELEMENTS(card); ++i){ while ((skb = skb_dequeue(&buf->skb_list))){ if (vlan_tx_tag_present(skb) && diff --git a/drivers/s390/s390mach.c b/drivers/s390/s390mach.c index 432136f96e6..ffb3677e354 100644 --- a/drivers/s390/s390mach.c +++ b/drivers/s390/s390mach.c @@ -378,6 +378,8 @@ s390_do_machine_check(struct pt_regs *regs) struct mcck_struct *mcck; int umode; + lockdep_off(); + mci = (struct mci *) &S390_lowcore.mcck_interruption_code; mcck = &__get_cpu_var(cpu_mcck); umode = user_mode(regs); @@ -482,6 +484,7 @@ s390_do_machine_check(struct pt_regs *regs) mcck->warning = 1; set_thread_flag(TIF_MCCK_PENDING); } + lockdep_on(); } /* diff --git a/drivers/scsi/libata-core.c b/drivers/scsi/libata-core.c index 82caba46429..1c960ac1617 100644 --- a/drivers/scsi/libata-core.c +++ b/drivers/scsi/libata-core.c @@ -1001,7 +1001,7 @@ unsigned ata_exec_internal(struct ata_device *dev, struct ata_queued_cmd *qc; unsigned int tag, preempted_tag; u32 preempted_sactive, preempted_qc_active; - DECLARE_COMPLETION(wait); + DECLARE_COMPLETION_ONSTACK(wait); unsigned long flags; unsigned int err_mask; int rc; diff --git a/drivers/serial/8250_pnp.c b/drivers/serial/8250_pnp.c index 739bc84f91e..632f62d6ec7 100644 --- a/drivers/serial/8250_pnp.c +++ b/drivers/serial/8250_pnp.c @@ -431,6 +431,8 @@ serial_pnp_probe(struct pnp_dev *dev, const struct pnp_device_id *dev_id) #endif port.flags |= UPF_SKIP_TEST | UPF_BOOT_AUTOCONF; + if (pnp_irq_flags(dev, 0) & IORESOURCE_IRQ_SHAREABLE) + port.flags |= UPF_SHARE_IRQ; port.uartclk = 1843200; port.dev = &dev->dev; diff --git a/drivers/serial/serial_core.c b/drivers/serial/serial_core.c index c54af877439..95831808334 100644 --- a/drivers/serial/serial_core.c +++ b/drivers/serial/serial_core.c @@ -49,6 +49,12 @@ */ static DEFINE_MUTEX(port_mutex); +/* + * lockdep: port->lock is initialized in two places, but we + * want only one lock-class: + */ +static struct lock_class_key port_lock_key; + #define HIGH_BITS_OFFSET ((sizeof(long)-sizeof(int))*8) #define uart_users(state) ((state)->count + ((state)->info ? (state)->info->blocked_open : 0)) @@ -1865,6 +1871,7 @@ uart_set_options(struct uart_port *port, struct console *co, * early. */ spin_lock_init(&port->lock); + lockdep_set_class(&port->lock, &port_lock_key); memset(&termios, 0, sizeof(struct termios)); @@ -2247,8 +2254,10 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *port) * If this port is a console, then the spinlock is already * initialised. */ - if (!(uart_console(port) && (port->cons->flags & CON_ENABLED))) + if (!(uart_console(port) && (port->cons->flags & CON_ENABLED))) { spin_lock_init(&port->lock); + lockdep_set_class(&port->lock, &port_lock_key); + } uart_configure_port(drv, state, port); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index ed1cdf6ac8f..146298ad737 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -510,7 +510,7 @@ static void spi_complete(void *arg) */ int spi_sync(struct spi_device *spi, struct spi_message *message) { - DECLARE_COMPLETION(done); + DECLARE_COMPLETION_ONSTACK(done); int status; message->complete = spi_complete; diff --git a/drivers/usb/core/inode.c b/drivers/usb/core/inode.c index e47e3a8ed6e..f48c3dbc367 100644 --- a/drivers/usb/core/inode.c +++ b/drivers/usb/core/inode.c @@ -200,7 +200,7 @@ static void update_sb(struct super_block *sb) if (!root) return; - mutex_lock(&root->d_inode->i_mutex); + mutex_lock_nested(&root->d_inode->i_mutex, I_MUTEX_PARENT); list_for_each_entry(bus, &root->d_subdirs, d_u.d_child) { if (bus->d_inode) { @@ -527,7 +527,7 @@ static void fs_remove_file (struct dentry *dentry) if (!parent || !parent->d_inode) return; - mutex_lock(&parent->d_inode->i_mutex); + mutex_lock_nested(&parent->d_inode->i_mutex, I_MUTEX_PARENT); if (usbfs_positive(dentry)) { if (dentry->d_inode) { if (S_ISDIR(dentry->d_inode->i_mode)) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 17de4c84db6..3badb48d662 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -1557,6 +1557,21 @@ config FB_S3C2410_DEBUG Turn on debugging messages. Note that you can set/unset at run time through sysfs +config FB_PNX4008_DUM + tristate "Display Update Module support on Philips PNX4008 board" + depends on FB && ARCH_PNX4008 + ---help--- + Say Y here to enable support for PNX4008 Display Update Module (DUM) + +config FB_PNX4008_DUM_RGB + tristate "RGB Framebuffer support on Philips PNX4008 board" + depends on FB_PNX4008_DUM + select FB_CFB_FILLRECT + select FB_CFB_COPYAREA + select FB_CFB_IMAGEBLIT + ---help--- + Say Y here to enable support for PNX4008 RGB Framebuffer + config FB_VIRTUAL tristate "Virtual Frame Buffer support (ONLY FOR TESTING!)" depends on FB diff --git a/drivers/video/Makefile b/drivers/video/Makefile index c335e9bc3b2..6283d015f8f 100644 --- a/drivers/video/Makefile +++ b/drivers/video/Makefile @@ -94,6 +94,8 @@ obj-$(CONFIG_FB_TX3912) += tx3912fb.o obj-$(CONFIG_FB_S1D13XXX) += s1d13xxxfb.o obj-$(CONFIG_FB_IMX) += imxfb.o obj-$(CONFIG_FB_S3C2410) += s3c2410fb.o +obj-$(CONFIG_FB_PNX4008_DUM) += pnx4008/ +obj-$(CONFIG_FB_PNX4008_DUM_RGB) += pnx4008/ # Platform or fallback drivers go here obj-$(CONFIG_FB_VESA) += vesafb.o diff --git a/drivers/video/pnx4008/Makefile b/drivers/video/pnx4008/Makefile new file mode 100644 index 00000000000..636aaccf01f --- /dev/null +++ b/drivers/video/pnx4008/Makefile @@ -0,0 +1,7 @@ +# +# Makefile for the new PNX4008 framebuffer device driver +# + +obj-$(CONFIG_FB_PNX4008_DUM) += sdum.o +obj-$(CONFIG_FB_PNX4008_DUM_RGB) += pnxrgbfb.o + diff --git a/drivers/video/pnx4008/dum.h b/drivers/video/pnx4008/dum.h new file mode 100644 index 00000000000..d80a614d89e --- /dev/null +++ b/drivers/video/pnx4008/dum.h @@ -0,0 +1,211 @@ +/* + * linux/drivers/video/pnx4008/dum.h + * + * Internal header for SDUM + * + * 2005 (c) Koninklijke Philips N.V. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ + +#ifndef __PNX008_DUM_H__ +#define __PNX008_DUM_H__ + +#include <asm/arch/platform.h> + +#define PNX4008_DUMCONF_VA_BASE IO_ADDRESS(PNX4008_DUMCONF_BASE) +#define PNX4008_DUM_MAIN_VA_BASE IO_ADDRESS(PNX4008_DUM_MAINCFG_BASE) + +/* DUM CFG ADDRESSES */ +#define DUM_CH_BASE_ADR (PNX4008_DUMCONF_VA_BASE + 0x00) +#define DUM_CH_MIN_ADR (PNX4008_DUMCONF_VA_BASE + 0x00) +#define DUM_CH_MAX_ADR (PNX4008_DUMCONF_VA_BASE + 0x04) +#define DUM_CH_CONF_ADR (PNX4008_DUMCONF_VA_BASE + 0x08) +#define DUM_CH_STAT_ADR (PNX4008_DUMCONF_VA_BASE + 0x0C) +#define DUM_CH_CTRL_ADR (PNX4008_DUMCONF_VA_BASE + 0x10) + +#define CH_MARG (0x100 / sizeof(u32)) +#define DUM_CH_MIN(i) (*((volatile u32 *)DUM_CH_MIN_ADR + (i) * CH_MARG)) +#define DUM_CH_MAX(i) (*((volatile u32 *)DUM_CH_MAX_ADR + (i) * CH_MARG)) +#define DUM_CH_CONF(i) (*((volatile u32 *)DUM_CH_CONF_ADR + (i) * CH_MARG)) +#define DUM_CH_STAT(i) (*((volatile u32 *)DUM_CH_STAT_ADR + (i) * CH_MARG)) +#define DUM_CH_CTRL(i) (*((volatile u32 *)DUM_CH_CTRL_ADR + (i) * CH_MARG)) + +#define DUM_CONF_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x00) +#define DUM_CTRL_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x04) +#define DUM_STAT_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x08) +#define DUM_DECODE_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x0C) +#define DUM_COM_BASE_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x10) +#define DUM_SYNC_C_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x14) +#define DUM_CLK_DIV_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x18) +#define DUM_DIRTY_LOW_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x20) +#define DUM_DIRTY_HIGH_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x24) +#define DUM_FORMAT_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x28) +#define DUM_WTCFG1_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x30) +#define DUM_RTCFG1_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x34) +#define DUM_WTCFG2_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x38) +#define DUM_RTCFG2_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x3C) +#define DUM_TCFG_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x40) +#define DUM_OUTP_FORMAT1_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x44) +#define DUM_OUTP_FORMAT2_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x48) +#define DUM_SYNC_MODE_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x4C) +#define DUM_SYNC_OUT_C_ADR (PNX4008_DUM_MAIN_VA_BASE + 0x50) + +#define DUM_CONF (*(volatile u32 *)(DUM_CONF_ADR)) +#define DUM_CTRL (*(volatile u32 *)(DUM_CTRL_ADR)) +#define DUM_STAT (*(volatile u32 *)(DUM_STAT_ADR)) +#define DUM_DECODE (*(volatile u32 *)(DUM_DECODE_ADR)) +#define DUM_COM_BASE (*(volatile u32 *)(DUM_COM_BASE_ADR)) +#define DUM_SYNC_C (*(volatile u32 *)(DUM_SYNC_C_ADR)) +#define DUM_CLK_DIV (*(volatile u32 *)(DUM_CLK_DIV_ADR)) +#define DUM_DIRTY_LOW (*(volatile u32 *)(DUM_DIRTY_LOW_ADR)) +#define DUM_DIRTY_HIGH (*(volatile u32 *)(DUM_DIRTY_HIGH_ADR)) +#define DUM_FORMAT (*(volatile u32 *)(DUM_FORMAT_ADR)) +#define DUM_WTCFG1 (*(volatile u32 *)(DUM_WTCFG1_ADR)) +#define DUM_RTCFG1 (*(volatile u32 *)(DUM_RTCFG1_ADR)) +#define DUM_WTCFG2 (*(volatile u32 *)(DUM_WTCFG2_ADR)) +#define DUM_RTCFG2 (*(volatile u32 *)(DUM_RTCFG2_ADR)) +#define DUM_TCFG (*(volatile u32 *)(DUM_TCFG_ADR)) +#define DUM_OUTP_FORMAT1 (*(volatile u32 *)(DUM_OUTP_FORMAT1_ADR)) +#define DUM_OUTP_FORMAT2 (*(volatile u32 *)(DUM_OUTP_FORMAT2_ADR)) +#define DUM_SYNC_MODE (*(volatile u32 *)(DUM_SYNC_MODE_ADR)) +#define DUM_SYNC_OUT_C (*(volatile u32 *)(DUM_SYNC_OUT_C_ADR)) + +/* DUM SLAVE ADDRESSES */ +#define DUM_SLAVE_WRITE_ADR (PNX4008_DUM_MAINCFG_BASE + 0x0000000) +#define DUM_SLAVE_READ1_I_ADR (PNX4008_DUM_MAINCFG_BASE + 0x1000000) +#define DUM_SLAVE_READ1_R_ADR (PNX4008_DUM_MAINCFG_BASE + 0x1000004) +#define DUM_SLAVE_READ2_I_ADR (PNX4008_DUM_MAINCFG_BASE + 0x1000008) +#define DUM_SLAVE_READ2_R_ADR (PNX4008_DUM_MAINCFG_BASE + 0x100000C) + +#define DUM_SLAVE_WRITE_W ((volatile u32 *)(DUM_SLAVE_WRITE_ADR)) +#define DUM_SLAVE_WRITE_HW ((volatile u16 *)(DUM_SLAVE_WRITE_ADR)) +#define DUM_SLAVE_READ1_I ((volatile u8 *)(DUM_SLAVE_READ1_I_ADR)) +#define DUM_SLAVE_READ1_R ((volatile u16 *)(DUM_SLAVE_READ1_R_ADR)) +#define DUM_SLAVE_READ2_I ((volatile u8 *)(DUM_SLAVE_READ2_I_ADR)) +#define DUM_SLAVE_READ2_R ((volatile u16 *)(DUM_SLAVE_READ2_R_ADR)) + +/* Sony display register addresses */ +#define DISP_0_REG (0x00) +#define DISP_1_REG (0x01) +#define DISP_CAL_REG (0x20) +#define DISP_ID_REG (0x2A) +#define DISP_XMIN_L_REG (0x30) +#define DISP_XMIN_H_REG (0x31) +#define DISP_YMIN_REG (0x32) +#define DISP_XMAX_L_REG (0x34) +#define DISP_XMAX_H_REG (0x35) +#define DISP_YMAX_REG (0x36) +#define DISP_SYNC_EN_REG (0x38) +#define DISP_SYNC_RISE_L_REG (0x3C) +#define DISP_SYNC_RISE_H_REG (0x3D) +#define DISP_SYNC_FALL_L_REG (0x3E) +#define DISP_SYNC_FALL_H_REG (0x3F) +#define DISP_PIXEL_REG (0x0B) +#define DISP_DUMMY1_REG (0x28) +#define DISP_DUMMY2_REG (0x29) +#define DISP_TIMING_REG (0x98) +#define DISP_DUMP_REG (0x99) + +/* Sony display constants */ +#define SONY_ID1 (0x22) +#define SONY_ID2 (0x23) + +/* Philips display register addresses */ +#define PH_DISP_ORIENT_REG (0x003) +#define PH_DISP_YPOINT_REG (0x200) +#define PH_DISP_XPOINT_REG (0x201) +#define PH_DISP_PIXEL_REG (0x202) +#define PH_DISP_YMIN_REG (0x406) +#define PH_DISP_YMAX_REG (0x407) +#define PH_DISP_XMIN_REG (0x408) +#define PH_DISP_XMAX_REG (0x409) + +/* Misc constants */ +#define NO_VALID_DISPLAY_FOUND (0) +#define DISPLAY2_IS_NOT_CONNECTED (0) + +/* register values */ +#define V_BAC_ENABLE (BIT(0)) +#define V_BAC_DISABLE_IDLE (BIT(1)) +#define V_BAC_DISABLE_TRIG (BIT(2)) +#define V_DUM_RESET (BIT(3)) +#define V_MUX_RESET (BIT(4)) +#define BAC_ENABLED (BIT(0)) +#define BAC_DISABLED 0 + +/* Sony LCD commands */ +#define V_LCD_STANDBY_OFF ((BIT(25)) | (0 << 16) | DISP_0_REG) +#define V_LCD_USE_9BIT_BUS ((BIT(25)) | (2 << 16) | DISP_1_REG) +#define V_LCD_SYNC_RISE_L ((BIT(25)) | (0 << 16) | DISP_SYNC_RISE_L_REG) +#define V_LCD_SYNC_RISE_H ((BIT(25)) | (0 << 16) | DISP_SYNC_RISE_H_REG) +#define V_LCD_SYNC_FALL_L ((BIT(25)) | (160 << 16) | DISP_SYNC_FALL_L_REG) +#define V_LCD_SYNC_FALL_H ((BIT(25)) | (0 << 16) | DISP_SYNC_FALL_H_REG) +#define V_LCD_SYNC_ENABLE ((BIT(25)) | (128 << 16) | DISP_SYNC_EN_REG) +#define V_LCD_DISPLAY_ON ((BIT(25)) | (64 << 16) | DISP_0_REG) + +enum { + PAD_NONE, + PAD_512, + PAD_1024 +}; + +enum { + RGB888, + RGB666, + RGB565, + BGR565, + ARGB1555, + ABGR1555, + ARGB4444, + ABGR4444 +}; + +struct dum_setup { + int sync_neg_edge; + int round_robin; + int mux_int; + int synced_dirty_flag_int; + int dirty_flag_int; + int error_int; + int pf_empty_int; + int sf_empty_int; + int bac_dis_int; + u32 dirty_base_adr; + u32 command_base_adr; + u32 sync_clk_div; + int sync_output; + u32 sync_restart_val; + u32 set_sync_high; + u32 set_sync_low; +}; + +struct dum_ch_setup { + int disp_no; + u32 xmin; + u32 ymin; + u32 xmax; + u32 ymax; + int xmirror; + int ymirror; + int rotate; + u32 minadr; + u32 maxadr; + u32 dirtybuffer; + int pad; + int format; + int hwdirty; + int slave_trans; +}; + +struct disp_window { + u32 xmin_l; + u32 xmin_h; + u32 ymin; + u32 xmax_l; + u32 xmax_h; + u32 ymax; +}; + +#endif /* #ifndef __PNX008_DUM_H__ */ diff --git a/drivers/video/pnx4008/fbcommon.h b/drivers/video/pnx4008/fbcommon.h new file mode 100644 index 00000000000..4ebc87dafaf --- /dev/null +++ b/drivers/video/pnx4008/fbcommon.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2005 Philips Semiconductors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA, or http://www.gnu.org/licenses/gpl.html +*/ + +#define QCIF_W (176) +#define QCIF_H (144) + +#define CIF_W (352) +#define CIF_H (288) + +#define LCD_X_RES 208 +#define LCD_Y_RES 320 +#define LCD_X_PAD 256 +#define LCD_BBP 4 /* Bytes Per Pixel */ + +#define DISP_MAX_X_SIZE (320) +#define DISP_MAX_Y_SIZE (208) + +#define RETURNVAL_BASE (0x400) + +enum fb_ioctl_returntype { + ENORESOURCESLEFT = RETURNVAL_BASE, + ERESOURCESNOTFREED, + EPROCNOTOWNER, + EFBNOTOWNER, + ECOPYFAILED, + EIOREMAPFAILED, +}; diff --git a/drivers/video/pnx4008/pnxrgbfb.c b/drivers/video/pnx4008/pnxrgbfb.c new file mode 100644 index 00000000000..7d9453c91a4 --- /dev/null +++ b/drivers/video/pnx4008/pnxrgbfb.c @@ -0,0 +1,213 @@ +/* + * drivers/video/pnx4008/pnxrgbfb.c + * + * PNX4008's framebuffer support + * + * Author: Grigory Tolstolytkin <gtolstolytkin@ru.mvista.com> + * Based on Philips Semiconductors's code + * + * Copyrght (c) 2005 MontaVista Software, Inc. + * Copyright (c) 2005 Philips Semiconductors + * This file is licensed under the terms of the GNU General Public License + * version 2. This program is licensed "as is" without any warranty of any + * kind, whether express or implied. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/fb.h> +#include <linux/init.h> +#include <linux/platform_device.h> + +#include <asm/uaccess.h> +#include "sdum.h" +#include "fbcommon.h" + +static u32 colreg[16]; + +static struct fb_var_screeninfo rgbfb_var __initdata = { + .xres = LCD_X_RES, + .yres = LCD_Y_RES, + .xres_virtual = LCD_X_RES, + .yres_virtual = LCD_Y_RES, + .bits_per_pixel = 32, + .red.offset = 16, + .red.length = 8, + .green.offset = 8, + .green.length = 8, + .blue.offset = 0, + .blue.length = 8, + .left_margin = 0, + .right_margin = 0, + .upper_margin = 0, + .lower_margin = 0, + .vmode = FB_VMODE_NONINTERLACED, +}; +static struct fb_fix_screeninfo rgbfb_fix __initdata = { + .id = "RGBFB", + .line_length = LCD_X_RES * LCD_BBP, + .type = FB_TYPE_PACKED_PIXELS, + .visual = FB_VISUAL_TRUECOLOR, + .xpanstep = 0, + .ypanstep = 0, + .ywrapstep = 0, + .accel = FB_ACCEL_NONE, +}; + +static int channel_owned; + +static int no_cursor(struct fb_info *info, struct fb_cursor *cursor) +{ + return 0; +} + +static int rgbfb_setcolreg(u_int regno, u_int red, u_int green, u_int blue, + u_int transp, struct fb_info *info) +{ + if (regno > 15) + return 1; + + colreg[regno] = ((red & 0xff00) << 8) | (green & 0xff00) | + ((blue & 0xff00) >> 8); + return 0; +} + +static int rgbfb_mmap(struct fb_info *info, struct vm_area_struct *vma) +{ + return pnx4008_sdum_mmap(info, vma, NULL); +} + +static struct fb_ops rgbfb_ops = { + .fb_mmap = rgbfb_mmap, + .fb_setcolreg = rgbfb_setcolreg, + .fb_fillrect = cfb_fillrect, + .fb_copyarea = cfb_copyarea, + .fb_imageblit = cfb_imageblit, +}; + +static int rgbfb_remove(struct platform_device *pdev) +{ + struct fb_info *info = platform_get_drvdata(pdev); + + if (info) { + unregister_framebuffer(info); + fb_dealloc_cmap(&info->cmap); + framebuffer_release(info); + platform_set_drvdata(pdev, NULL); + kfree(info); + } + + pnx4008_free_dum_channel(channel_owned, pdev->id); + pnx4008_set_dum_exit_notification(pdev->id); + + return 0; +} + +static int __devinit rgbfb_probe(struct platform_device *pdev) +{ + struct fb_info *info; + struct dumchannel_uf chan_uf; + int ret; + char *option; + + info = framebuffer_alloc(sizeof(u32) * 16, &pdev->dev); + if (!info) { + ret = -ENOMEM; + goto err; + } + + pnx4008_get_fb_addresses(FB_TYPE_RGB, (void **)&info->screen_base, + (dma_addr_t *) &rgbfb_fix.smem_start, + &rgbfb_fix.smem_len); + + if ((ret = pnx4008_alloc_dum_channel(pdev->id)) < 0) + goto err0; + else { + channel_owned = ret; + chan_uf.channelnr = channel_owned; + chan_uf.dirty = (u32 *) NULL; + chan_uf.source = (u32 *) rgbfb_fix.smem_start; + chan_uf.x_offset = 0; + chan_uf.y_offset = 0; + chan_uf.width = LCD_X_RES; + chan_uf.height = LCD_Y_RES; + + if ((ret = pnx4008_put_dum_channel_uf(chan_uf, pdev->id))< 0) + goto err1; + + if ((ret = + pnx4008_set_dum_channel_sync(channel_owned, CONF_SYNC_ON, + pdev->id)) < 0) + goto err1; + + if ((ret = + pnx4008_set_dum_channel_dirty_detect(channel_owned, + CONF_DIRTYDETECTION_ON, + pdev->id)) < 0) + goto err1; + } + + if (!fb_get_options("pnxrgbfb", &option) && !strcmp(option, "nocursor")) + rgbfb_ops.fb_cursor = no_cursor; + + info->node = -1; + info->flags = FBINFO_FLAG_DEFAULT; + info->fbops = &rgbfb_ops; + info->fix = rgbfb_fix; + info->var = rgbfb_var; + info->screen_size = rgbfb_fix.smem_len; + info->pseudo_palette = info->par; + info->par = NULL; + + ret = fb_alloc_cmap(&info->cmap, 256, 0); + if (ret < 0) + goto err2; + + ret = register_framebuffer(info); + if (ret < 0) + goto err3; + platform_set_drvdata(pdev, info); + + return 0; + +err3: + fb_dealloc_cmap(&info->cmap); +err2: + framebuffer_release(info); +err1: + pnx4008_free_dum_channel(channel_owned, pdev->id); +err0: + kfree(info); +err: + return ret; +} + +static struct platform_driver rgbfb_driver = { + .driver = { + .name = "rgbfb", + }, + .probe = rgbfb_probe, + .remove = rgbfb_remove, +}; + +static int __init rgbfb_init(void) +{ + return platform_driver_register(&rgbfb_driver); +} + +static void __exit rgbfb_exit(void) +{ + platform_driver_unregister(&rgbfb_driver); +} + +module_init(rgbfb_init); +module_exit(rgbfb_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/video/pnx4008/sdum.c b/drivers/video/pnx4008/sdum.c new file mode 100644 index 00000000000..51f0ecc2a51 --- /dev/null +++ b/drivers/video/pnx4008/sdum.c @@ -0,0 +1,872 @@ +/* + * drivers/video/pnx4008/sdum.c + * + * Display Update Master support + * + * Authors: Grigory Tolstolytkin <gtolstolytkin@ru.mvista.com> + * Vitaly Wool <vitalywool@gmail.com> + * Based on Philips Semiconductors's code + * + * Copyrght (c) 2005-2006 MontaVista Software, Inc. + * Copyright (c) 2005 Philips Semiconductors + * This file is licensed under the terms of the GNU General Public License + * version 2. This program is licensed "as is" without any warranty of any + * kind, whether express or implied. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/tty.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/fb.h> +#include <linux/init.h> +#include <linux/dma-mapping.h> +#include <linux/clk.h> +#include <asm/uaccess.h> +#include <asm/arch/gpio.h> + +#include "sdum.h" +#include "fbcommon.h" +#include "dum.h" + +/* Framebuffers we have */ + +static struct pnx4008_fb_addr { + int fb_type; + long addr_offset; + long fb_length; +} fb_addr[] = { + [0] = { + FB_TYPE_YUV, 0, 0xB0000 + }, + [1] = { + FB_TYPE_RGB, 0xB0000, 0x50000 + }, +}; + +static struct dum_data { + u32 lcd_phys_start; + u32 lcd_virt_start; + u32 slave_phys_base; + u32 *slave_virt_base; + int fb_owning_channel[MAX_DUM_CHANNELS]; + struct dumchannel_uf chan_uf_store[MAX_DUM_CHANNELS]; +} dum_data; + +/* Different local helper functions */ + +static u32 nof_pixels_dx(struct dum_ch_setup *ch_setup) +{ + return (ch_setup->xmax - ch_setup->xmin + 1); +} + +static u32 nof_pixels_dy(struct dum_ch_setup *ch_setup) +{ + return (ch_setup->ymax - ch_setup->ymin + 1); +} + +static u32 nof_pixels_dxy(struct dum_ch_setup *ch_setup) +{ + return (nof_pixels_dx(ch_setup) * nof_pixels_dy(ch_setup)); +} + +static u32 nof_bytes(struct dum_ch_setup *ch_setup) +{ + u32 r = nof_pixels_dxy(ch_setup); + switch (ch_setup->format) { + case RGB888: + case RGB666: + r *= 4; + break; + + default: + r *= 2; + break; + } + return r; +} + +static u32 build_command(int disp_no, u32 reg, u32 val) +{ + return ((disp_no << 26) | BIT(25) | (val << 16) | (disp_no << 10) | + (reg << 0)); +} + +static u32 build_double_index(int disp_no, u32 val) +{ + return ((disp_no << 26) | (val << 16) | (disp_no << 10) | (val << 0)); +} + +static void build_disp_window(struct dum_ch_setup * ch_setup, struct disp_window * dw) +{ + dw->ymin = ch_setup->ymin; + dw->ymax = ch_setup->ymax; + dw->xmin_l = ch_setup->xmin & 0xFF; + dw->xmin_h = (ch_setup->xmin & BIT(8)) >> 8; + dw->xmax_l = ch_setup->xmax & 0xFF; + dw->xmax_h = (ch_setup->xmax & BIT(8)) >> 8; +} + +static int put_channel(struct dumchannel chan) +{ + int i = chan.channelnr; + + if (i < 0 || i > MAX_DUM_CHANNELS) + return -EINVAL; + else { + DUM_CH_MIN(i) = chan.dum_ch_min; + DUM_CH_MAX(i) = chan.dum_ch_max; + DUM_CH_CONF(i) = chan.dum_ch_conf; + DUM_CH_CTRL(i) = chan.dum_ch_ctrl; + } + + return 0; +} + +static void clear_channel(int channr) +{ + struct dumchannel chan; + + chan.channelnr = channr; + chan.dum_ch_min = 0; + chan.dum_ch_max = 0; + chan.dum_ch_conf = 0; + chan.dum_ch_ctrl = 0; + + put_channel(chan); +} + +static int put_cmd_string(struct cmdstring cmds) +{ + u16 *cmd_str_virtaddr; + u32 *cmd_ptr0_virtaddr; + u32 cmd_str_physaddr; + + int i = cmds.channelnr; + + if (i < 0 || i > MAX_DUM_CHANNELS) + return -EINVAL; + else if ((cmd_ptr0_virtaddr = + (int *)ioremap_nocache(DUM_COM_BASE, + sizeof(int) * MAX_DUM_CHANNELS)) == + NULL) + return -EIOREMAPFAILED; + else { + cmd_str_physaddr = ioread32(&cmd_ptr0_virtaddr[cmds.channelnr]); + if ((cmd_str_virtaddr = + (u16 *) ioremap_nocache(cmd_str_physaddr, + sizeof(cmds))) == NULL) { + iounmap(cmd_ptr0_virtaddr); + return -EIOREMAPFAILED; + } else { + int t; + for (t = 0; t < 8; t++) + iowrite16(*((u16 *)&cmds.prestringlen + t), + cmd_str_virtaddr + t); + + for (t = 0; t < cmds.prestringlen / 2; t++) + iowrite16(*((u16 *)&cmds.precmd + t), + cmd_str_virtaddr + t + 8); + + for (t = 0; t < cmds.poststringlen / 2; t++) + iowrite16(*((u16 *)&cmds.postcmd + t), + cmd_str_virtaddr + t + 8 + + cmds.prestringlen / 2); + + iounmap(cmd_ptr0_virtaddr); + iounmap(cmd_str_virtaddr); + } + } + + return 0; +} + +static u32 dum_ch_setup(int ch_no, struct dum_ch_setup * ch_setup) +{ + struct cmdstring cmds_c; + struct cmdstring *cmds = &cmds_c; + struct disp_window dw; + int standard; + u32 orientation = 0; + struct dumchannel chan = { 0 }; + int ret; + + if ((ch_setup->xmirror) || (ch_setup->ymirror) || (ch_setup->rotate)) { + standard = 0; + + orientation = BIT(1); /* always set 9-bit-bus */ + if (ch_setup->xmirror) + orientation |= BIT(4); + if (ch_setup->ymirror) + orientation |= BIT(3); + if (ch_setup->rotate) + orientation |= BIT(0); + } else + standard = 1; + + cmds->channelnr = ch_no; + + /* build command string header */ + if (standard) { + cmds->prestringlen = 32; + cmds->poststringlen = 0; + } else { + cmds->prestringlen = 48; + cmds->poststringlen = 16; + } + + cmds->format = + (u16) ((ch_setup->disp_no << 4) | (BIT(3)) | (ch_setup->format)); + cmds->reserved = 0x0; + cmds->startaddr_low = (ch_setup->minadr & 0xFFFF); + cmds->startaddr_high = (ch_setup->minadr >> 16); + + if ((ch_setup->minadr == 0) && (ch_setup->maxadr == 0) + && (ch_setup->xmin == 0) + && (ch_setup->ymin == 0) && (ch_setup->xmax == 0) + && (ch_setup->ymax == 0)) { + cmds->pixdatlen_low = 0; + cmds->pixdatlen_high = 0; + } else { + u32 nbytes = nof_bytes(ch_setup); + cmds->pixdatlen_low = (nbytes & 0xFFFF); + cmds->pixdatlen_high = (nbytes >> 16); + } + + if (ch_setup->slave_trans) + cmds->pixdatlen_high |= BIT(15); + + /* build pre-string */ + build_disp_window(ch_setup, &dw); + + if (standard) { + cmds->precmd[0] = + build_command(ch_setup->disp_no, DISP_XMIN_L_REG, 0x99); + cmds->precmd[1] = + build_command(ch_setup->disp_no, DISP_XMIN_L_REG, + dw.xmin_l); + cmds->precmd[2] = + build_command(ch_setup->disp_no, DISP_XMIN_H_REG, + dw.xmin_h); + cmds->precmd[3] = + build_command(ch_setup->disp_no, DISP_YMIN_REG, dw.ymin); + cmds->precmd[4] = + build_command(ch_setup->disp_no, DISP_XMAX_L_REG, + dw.xmax_l); + cmds->precmd[5] = + build_command(ch_setup->disp_no, DISP_XMAX_H_REG, + dw.xmax_h); + cmds->precmd[6] = + build_command(ch_setup->disp_no, DISP_YMAX_REG, dw.ymax); + cmds->precmd[7] = + build_double_index(ch_setup->disp_no, DISP_PIXEL_REG); + } else { + if (dw.xmin_l == ch_no) + cmds->precmd[0] = + build_command(ch_setup->disp_no, DISP_XMIN_L_REG, + 0x99); + else + cmds->precmd[0] = + build_command(ch_setup->disp_no, DISP_XMIN_L_REG, + ch_no); + + cmds->precmd[1] = + build_command(ch_setup->disp_no, DISP_XMIN_L_REG, + dw.xmin_l); + cmds->precmd[2] = + build_command(ch_setup->disp_no, DISP_XMIN_H_REG, + dw.xmin_h); + cmds->precmd[3] = + build_command(ch_setup->disp_no, DISP_YMIN_REG, dw.ymin); + cmds->precmd[4] = + build_command(ch_setup->disp_no, DISP_XMAX_L_REG, + dw.xmax_l); + cmds->precmd[5] = + build_command(ch_setup->disp_no, DISP_XMAX_H_REG, + dw.xmax_h); + cmds->precmd[6] = + build_command(ch_setup->disp_no, DISP_YMAX_REG, dw.ymax); + cmds->precmd[7] = + build_command(ch_setup->disp_no, DISP_1_REG, orientation); + cmds->precmd[8] = + build_double_index(ch_setup->disp_no, DISP_PIXEL_REG); + cmds->precmd[9] = + build_double_index(ch_setup->disp_no, DISP_PIXEL_REG); + cmds->precmd[0xA] = + build_double_index(ch_setup->disp_no, DISP_PIXEL_REG); + cmds->precmd[0xB] = + build_double_index(ch_setup->disp_no, DISP_PIXEL_REG); + cmds->postcmd[0] = + build_command(ch_setup->disp_no, DISP_1_REG, BIT(1)); + cmds->postcmd[1] = + build_command(ch_setup->disp_no, DISP_DUMMY1_REG, 1); + cmds->postcmd[2] = + build_command(ch_setup->disp_no, DISP_DUMMY1_REG, 2); + cmds->postcmd[3] = + build_command(ch_setup->disp_no, DISP_DUMMY1_REG, 3); + } + + if ((ret = put_cmd_string(cmds_c)) != 0) { + return ret; + } + + chan.channelnr = cmds->channelnr; + chan.dum_ch_min = ch_setup->dirtybuffer + ch_setup->minadr; + chan.dum_ch_max = ch_setup->dirtybuffer + ch_setup->maxadr; + chan.dum_ch_conf = 0x002; + chan.dum_ch_ctrl = 0x04; + + put_channel(chan); + + return 0; +} + +static u32 display_open(int ch_no, int auto_update, u32 * dirty_buffer, + u32 * frame_buffer, u32 xpos, u32 ypos, u32 w, u32 h) +{ + + struct dum_ch_setup k; + int ret; + + /* keep width & height within display area */ + if ((xpos + w) > DISP_MAX_X_SIZE) + w = DISP_MAX_X_SIZE - xpos; + + if ((ypos + h) > DISP_MAX_Y_SIZE) + h = DISP_MAX_Y_SIZE - ypos; + + /* assume 1 display only */ + k.disp_no = 0; + k.xmin = xpos; + k.ymin = ypos; + k.xmax = xpos + (w - 1); + k.ymax = ypos + (h - 1); + + /* adjust min and max values if necessary */ + if (k.xmin > DISP_MAX_X_SIZE - 1) + k.xmin = DISP_MAX_X_SIZE - 1; + if (k.ymin > DISP_MAX_Y_SIZE - 1) + k.ymin = DISP_MAX_Y_SIZE - 1; + + if (k.xmax > DISP_MAX_X_SIZE - 1) + k.xmax = DISP_MAX_X_SIZE - 1; + if (k.ymax > DISP_MAX_Y_SIZE - 1) + k.ymax = DISP_MAX_Y_SIZE - 1; + + k.xmirror = 0; + k.ymirror = 0; + k.rotate = 0; + k.minadr = (u32) frame_buffer; + k.maxadr = (u32) frame_buffer + (((w - 1) << 10) | ((h << 2) - 2)); + k.pad = PAD_1024; + k.dirtybuffer = (u32) dirty_buffer; + k.format = RGB888; + k.hwdirty = 0; + k.slave_trans = 0; + + ret = dum_ch_setup(ch_no, &k); + + return ret; +} + +static void lcd_reset(void) +{ + u32 *dum_pio_base = (u32 *)IO_ADDRESS(PNX4008_PIO_BASE); + + udelay(1); + iowrite32(BIT(19), &dum_pio_base[2]); + udelay(1); + iowrite32(BIT(19), &dum_pio_base[1]); + udelay(1); +} + +static int dum_init(struct platform_device *pdev) +{ + struct clk *clk; + + /* enable DUM clock */ + clk = clk_get(&pdev->dev, "dum_ck"); + if (IS_ERR(clk)) { + printk(KERN_ERR "pnx4008_dum: Unable to access DUM clock\n"); + return PTR_ERR(clk); + } + + clk_set_rate(clk, 1); + clk_put(clk); + + DUM_CTRL = V_DUM_RESET; + + /* set priority to "round-robin". All other params to "false" */ + DUM_CONF = BIT(9); + + /* Display 1 */ + DUM_WTCFG1 = PNX4008_DUM_WT_CFG; + DUM_RTCFG1 = PNX4008_DUM_RT_CFG; + DUM_TCFG = PNX4008_DUM_T_CFG; + + return 0; +} + +static void dum_chan_init(void) +{ + int i = 0, ch = 0; + u32 *cmdptrs; + u32 *cmdstrings; + + DUM_COM_BASE = + CMDSTRING_BASEADDR + BYTES_PER_CMDSTRING * NR_OF_CMDSTRINGS; + + if ((cmdptrs = + (u32 *) ioremap_nocache(DUM_COM_BASE, + sizeof(u32) * NR_OF_CMDSTRINGS)) == NULL) + return; + + for (ch = 0; ch < NR_OF_CMDSTRINGS; ch++) + iowrite32(CMDSTRING_BASEADDR + BYTES_PER_CMDSTRING * ch, + cmdptrs + ch); + + for (ch = 0; ch < MAX_DUM_CHANNELS; ch++) + clear_channel(ch); + + /* Clear the cmdstrings */ + cmdstrings = + (u32 *)ioremap_nocache(*cmdptrs, + BYTES_PER_CMDSTRING * NR_OF_CMDSTRINGS); + + if (!cmdstrings) + goto out; + + for (i = 0; i < NR_OF_CMDSTRINGS * BYTES_PER_CMDSTRING / sizeof(u32); + i++) + iowrite32(0, cmdstrings + i); + + iounmap((u32 *)cmdstrings); + +out: + iounmap((u32 *)cmdptrs); +} + +static void lcd_init(void) +{ + lcd_reset(); + + DUM_OUTP_FORMAT1 = 0; /* RGB666 */ + + udelay(1); + iowrite32(V_LCD_STANDBY_OFF, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_USE_9BIT_BUS, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_SYNC_RISE_L, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_SYNC_RISE_H, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_SYNC_FALL_L, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_SYNC_FALL_H, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_SYNC_ENABLE, dum_data.slave_virt_base); + udelay(1); + iowrite32(V_LCD_DISPLAY_ON, dum_data.slave_virt_base); + udelay(1); +} + +/* Interface exported to framebuffer drivers */ + +int pnx4008_get_fb_addresses(int fb_type, void **virt_addr, + dma_addr_t *phys_addr, int *fb_length) +{ + int i; + int ret = -1; + for (i = 0; i < ARRAY_SIZE(fb_addr); i++) + if (fb_addr[i].fb_type == fb_type) { + *virt_addr = (void *)(dum_data.lcd_virt_start + + fb_addr[i].addr_offset); + *phys_addr = + dum_data.lcd_phys_start + fb_addr[i].addr_offset; + *fb_length = fb_addr[i].fb_length; + ret = 0; + break; + } + + return ret; +} + +EXPORT_SYMBOL(pnx4008_get_fb_addresses); + +int pnx4008_alloc_dum_channel(int dev_id) +{ + int i = 0; + + while ((i < MAX_DUM_CHANNELS) && (dum_data.fb_owning_channel[i] != -1)) + i++; + + if (i == MAX_DUM_CHANNELS) + return -ENORESOURCESLEFT; + else { + dum_data.fb_owning_channel[i] = dev_id; + return i; + } +} + +EXPORT_SYMBOL(pnx4008_alloc_dum_channel); + +int pnx4008_free_dum_channel(int channr, int dev_id) +{ + if (channr < 0 || channr > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[channr] != dev_id) + return -EFBNOTOWNER; + else { + clear_channel(channr); + dum_data.fb_owning_channel[channr] = -1; + } + + return 0; +} + +EXPORT_SYMBOL(pnx4008_free_dum_channel); + +int pnx4008_put_dum_channel_uf(struct dumchannel_uf chan_uf, int dev_id) +{ + int i = chan_uf.channelnr; + int ret; + + if (i < 0 || i > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[i] != dev_id) + return -EFBNOTOWNER; + else if ((ret = + display_open(chan_uf.channelnr, 0, chan_uf.dirty, + chan_uf.source, chan_uf.y_offset, + chan_uf.x_offset, chan_uf.height, + chan_uf.width)) != 0) + return ret; + else { + dum_data.chan_uf_store[i].dirty = chan_uf.dirty; + dum_data.chan_uf_store[i].source = chan_uf.source; + dum_data.chan_uf_store[i].x_offset = chan_uf.x_offset; + dum_data.chan_uf_store[i].y_offset = chan_uf.y_offset; + dum_data.chan_uf_store[i].width = chan_uf.width; + dum_data.chan_uf_store[i].height = chan_uf.height; + } + + return 0; +} + +EXPORT_SYMBOL(pnx4008_put_dum_channel_uf); + +int pnx4008_set_dum_channel_sync(int channr, int val, int dev_id) +{ + if (channr < 0 || channr > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[channr] != dev_id) + return -EFBNOTOWNER; + else { + if (val == CONF_SYNC_ON) { + DUM_CH_CONF(channr) |= CONF_SYNCENABLE; + DUM_CH_CONF(channr) |= DUM_CHANNEL_CFG_SYNC_MASK | + DUM_CHANNEL_CFG_SYNC_MASK_SET; + } else if (val == CONF_SYNC_OFF) + DUM_CH_CONF(channr) &= ~CONF_SYNCENABLE; + else + return -EINVAL; + } + + return 0; +} + +EXPORT_SYMBOL(pnx4008_set_dum_channel_sync); + +int pnx4008_set_dum_channel_dirty_detect(int channr, int val, int dev_id) +{ + if (channr < 0 || channr > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[channr] != dev_id) + return -EFBNOTOWNER; + else { + if (val == CONF_DIRTYDETECTION_ON) + DUM_CH_CONF(channr) |= CONF_DIRTYENABLE; + else if (val == CONF_DIRTYDETECTION_OFF) + DUM_CH_CONF(channr) &= ~CONF_DIRTYENABLE; + else + return -EINVAL; + } + + return 0; +} + +EXPORT_SYMBOL(pnx4008_set_dum_channel_dirty_detect); + +#if 0 /* Functions not used currently, but likely to be used in future */ + +static int get_channel(struct dumchannel *p_chan) +{ + int i = p_chan->channelnr; + + if (i < 0 || i > MAX_DUM_CHANNELS) + return -EINVAL; + else { + p_chan->dum_ch_min = DUM_CH_MIN(i); + p_chan->dum_ch_max = DUM_CH_MAX(i); + p_chan->dum_ch_conf = DUM_CH_CONF(i); + p_chan->dum_ch_stat = DUM_CH_STAT(i); + p_chan->dum_ch_ctrl = 0; /* WriteOnly control register */ + } + + return 0; +} + +int pnx4008_get_dum_channel_uf(struct dumchannel_uf *p_chan_uf, int dev_id) +{ + int i = p_chan_uf->channelnr; + + if (i < 0 || i > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[i] != dev_id) + return -EFBNOTOWNER; + else { + p_chan_uf->dirty = dum_data.chan_uf_store[i].dirty; + p_chan_uf->source = dum_data.chan_uf_store[i].source; + p_chan_uf->x_offset = dum_data.chan_uf_store[i].x_offset; + p_chan_uf->y_offset = dum_data.chan_uf_store[i].y_offset; + p_chan_uf->width = dum_data.chan_uf_store[i].width; + p_chan_uf->height = dum_data.chan_uf_store[i].height; + } + + return 0; +} + +EXPORT_SYMBOL(pnx4008_get_dum_channel_uf); + +int pnx4008_get_dum_channel_config(int channr, int dev_id) +{ + int ret; + struct dumchannel chan; + + if (channr < 0 || channr > MAX_DUM_CHANNELS) + return -EINVAL; + else if (dum_data.fb_owning_channel[channr] != dev_id) + return -EFBNOTOWNER; + else { + chan.channelnr = channr; + if ((ret = get_channel(&chan)) != 0) + return ret; + } + + return (chan.dum_ch_conf & DUM_CHANNEL_CFG_MASK); +} + +EXPORT_SYMBOL(pnx4008_get_dum_channel_config); + +int pnx4008_force_update_dum_channel(int channr, int dev_id) +{ + if (channr < 0 || channr > MAX_DUM_CHANNELS) + return -EINVAL; + + else if (dum_data.fb_owning_channel[channr] != dev_id) + return -EFBNOTOWNER; + else + DUM_CH_CTRL(channr) = CTRL_SETDIRTY; + + return 0; +} + +EXPORT_SYMBOL(pnx4008_force_update_dum_channel); + +#endif + +int pnx4008_sdum_mmap(struct fb_info *info, struct vm_area_struct *vma, + struct device *dev) +{ + unsigned long off = vma->vm_pgoff << PAGE_SHIFT; + + if (off < info->fix.smem_len) { + vma->vm_pgoff += 1; + return dma_mmap_writecombine(dev, vma, + (void *)dum_data.lcd_virt_start, + dum_data.lcd_phys_start, + FB_DMA_SIZE); + } + return -EINVAL; +} + +EXPORT_SYMBOL(pnx4008_sdum_mmap); + +int pnx4008_set_dum_exit_notification(int dev_id) +{ + int i; + + for (i = 0; i < MAX_DUM_CHANNELS; i++) + if (dum_data.fb_owning_channel[i] == dev_id) + return -ERESOURCESNOTFREED; + + return 0; +} + +EXPORT_SYMBOL(pnx4008_set_dum_exit_notification); + +/* Platform device driver for DUM */ + +static int sdum_suspend(struct platform_device *pdev, pm_message_t state) +{ + int retval = 0; + struct clk *clk; + + clk = clk_get(0, "dum_ck"); + if (!IS_ERR(clk)) { + clk_set_rate(clk, 0); + clk_put(clk); + } else + retval = PTR_ERR(clk); + + /* disable BAC */ + DUM_CTRL = V_BAC_DISABLE_IDLE; + + /* LCD standby & turn off display */ + lcd_reset(); + + return retval; +} + +static int sdum_resume(struct platform_device *pdev) +{ + int retval = 0; + struct clk *clk; + + clk = clk_get(0, "dum_ck"); + if (!IS_ERR(clk)) { + clk_set_rate(clk, 1); + clk_put(clk); + } else + retval = PTR_ERR(clk); + + /* wait for BAC disable */ + DUM_CTRL = V_BAC_DISABLE_TRIG; + + while (DUM_CTRL & BAC_ENABLED) + udelay(10); + + /* re-init LCD */ + lcd_init(); + + /* enable BAC and reset MUX */ + DUM_CTRL = V_BAC_ENABLE; + udelay(1); + DUM_CTRL = V_MUX_RESET; + return 0; +} + +static int __devinit sdum_probe(struct platform_device *pdev) +{ + int ret = 0, i = 0; + + /* map frame buffer */ + dum_data.lcd_virt_start = (u32) dma_alloc_writecombine(&pdev->dev, + FB_DMA_SIZE, + &dum_data.lcd_phys_start, + GFP_KERNEL); + + if (!dum_data.lcd_virt_start) { + ret = -ENOMEM; + goto out_3; + } + + /* map slave registers */ + dum_data.slave_phys_base = PNX4008_DUM_SLAVE_BASE; + dum_data.slave_virt_base = + (u32 *) ioremap_nocache(dum_data.slave_phys_base, sizeof(u32)); + + if (dum_data.slave_virt_base == NULL) { + ret = -ENOMEM; + goto out_2; + } + + /* initialize DUM and LCD display */ + ret = dum_init(pdev); + if (ret) + goto out_1; + + dum_chan_init(); + lcd_init(); + + DUM_CTRL = V_BAC_ENABLE; + udelay(1); + DUM_CTRL = V_MUX_RESET; + + /* set decode address and sync clock divider */ + DUM_DECODE = dum_data.lcd_phys_start & DUM_DECODE_MASK; + DUM_CLK_DIV = PNX4008_DUM_CLK_DIV; + + for (i = 0; i < MAX_DUM_CHANNELS; i++) + dum_data.fb_owning_channel[i] = -1; + + /*setup wakeup interrupt */ + start_int_set_rising_edge(SE_DISP_SYNC_INT); + start_int_ack(SE_DISP_SYNC_INT); + start_int_umask(SE_DISP_SYNC_INT); + + return 0; + +out_1: + iounmap((void *)dum_data.slave_virt_base); +out_2: + dma_free_writecombine(&pdev->dev, FB_DMA_SIZE, + (void *)dum_data.lcd_virt_start, + dum_data.lcd_phys_start); +out_3: + return ret; +} + +static int sdum_remove(struct platform_device *pdev) +{ + struct clk *clk; + + start_int_mask(SE_DISP_SYNC_INT); + + clk = clk_get(0, "dum_ck"); + if (!IS_ERR(clk)) { + clk_set_rate(clk, 0); + clk_put(clk); + } + + iounmap((void *)dum_data.slave_virt_base); + + dma_free_writecombine(&pdev->dev, FB_DMA_SIZE, + (void *)dum_data.lcd_virt_start, + dum_data.lcd_phys_start); + + return 0; +} + +static struct platform_driver sdum_driver = { + .driver = { + .name = "sdum", + }, + .probe = sdum_probe, + .remove = sdum_remove, + .suspend = sdum_suspend, + .resume = sdum_resume, +}; + +int __init sdum_init(void) +{ + return platform_driver_register(&sdum_driver); +} + +static void __exit sdum_exit(void) +{ + platform_driver_unregister(&sdum_driver); +}; + +module_init(sdum_init); +module_exit(sdum_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/video/pnx4008/sdum.h b/drivers/video/pnx4008/sdum.h new file mode 100644 index 00000000000..e8c5dcdd881 --- /dev/null +++ b/drivers/video/pnx4008/sdum.h @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2005 Philips Semiconductors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA, or http://www.gnu.org/licenses/gpl.html +*/ + +#define MAX_DUM_CHANNELS 64 + +#define RGB_MEM_WINDOW(x) (0x10000000 + (x)*0x00100000) + +#define QCIF_OFFSET(x) (((x) == 0) ? 0x00000: ((x) == 1) ? 0x30000: -1) +#define CIF_OFFSET(x) (((x) == 0) ? 0x00000: ((x) == 1) ? 0x60000: -1) + +#define CTRL_SETDIRTY (0x00000001) +#define CONF_DIRTYENABLE (0x00000020) +#define CONF_SYNCENABLE (0x00000004) + +#define DIRTY_ENABLED(conf) ((conf) & 0x0020) +#define SYNC_ENABLED(conf) ((conf) & 0x0004) + +/* Display 1 & 2 Write Timing Configuration */ +#define PNX4008_DUM_WT_CFG 0x00372000 + +/* Display 1 & 2 Read Timing Configuration */ +#define PNX4008_DUM_RT_CFG 0x00003A47 + +/* DUM Transit State Timing Configuration */ +#define PNX4008_DUM_T_CFG 0x1D /* 29 HCLK cycles */ + +/* DUM Sync count clock divider */ +#define PNX4008_DUM_CLK_DIV 0x02DD + +/* Memory size for framebuffer, allocated through dma_alloc_writecombine(). + * Must be PAGE aligned + */ +#define FB_DMA_SIZE (PAGE_ALIGN(SZ_1M + PAGE_SIZE)) + +#define OFFSET_RGBBUFFER (0xB0000) +#define OFFSET_YUVBUFFER (0x00000) + +#define YUVBUFFER (lcd_video_start + OFFSET_YUVBUFFER) +#define RGBBUFFER (lcd_video_start + OFFSET_RGBBUFFER) + +#define CMDSTRING_BASEADDR (0x00C000) /* iram */ +#define BYTES_PER_CMDSTRING (0x80) +#define NR_OF_CMDSTRINGS (64) + +#define MAX_NR_PRESTRINGS (0x40) +#define MAX_NR_POSTSTRINGS (0x40) + +/* various mask definitions */ +#define DUM_CLK_ENABLE 0x01 +#define DUM_CLK_DISABLE 0 +#define DUM_DECODE_MASK 0x1FFFFFFF +#define DUM_CHANNEL_CFG_MASK 0x01FF +#define DUM_CHANNEL_CFG_SYNC_MASK 0xFFFE00FF +#define DUM_CHANNEL_CFG_SYNC_MASK_SET 0x0CA00 + +#define SDUM_RETURNVAL_BASE (0x500) + +#define CONF_SYNC_OFF (0x602) +#define CONF_SYNC_ON (0x603) + +#define CONF_DIRTYDETECTION_OFF (0x600) +#define CONF_DIRTYDETECTION_ON (0x601) + +/* Set the corresponding bit. */ +#define BIT(n) (0x1U << (n)) + +struct dumchannel_uf { + int channelnr; + u32 *dirty; + u32 *source; + u32 x_offset; + u32 y_offset; + u32 width; + u32 height; +}; + +enum { + FB_TYPE_YUV, + FB_TYPE_RGB +}; + +struct cmdstring { + int channelnr; + uint16_t prestringlen; + uint16_t poststringlen; + uint16_t format; + uint16_t reserved; + uint16_t startaddr_low; + uint16_t startaddr_high; + uint16_t pixdatlen_low; + uint16_t pixdatlen_high; + u32 precmd[MAX_NR_PRESTRINGS]; + u32 postcmd[MAX_NR_POSTSTRINGS]; + +}; + +struct dumchannel { + int channelnr; + int dum_ch_min; + int dum_ch_max; + int dum_ch_conf; + int dum_ch_stat; + int dum_ch_ctrl; +}; + +int pnx4008_alloc_dum_channel(int dev_id); +int pnx4008_free_dum_channel(int channr, int dev_id); + +int pnx4008_get_dum_channel_uf(struct dumchannel_uf *pChan_uf, int dev_id); +int pnx4008_put_dum_channel_uf(struct dumchannel_uf chan_uf, int dev_id); + +int pnx4008_set_dum_channel_sync(int channr, int val, int dev_id); +int pnx4008_set_dum_channel_dirty_detect(int channr, int val, int dev_id); + +int pnx4008_force_dum_update_channel(int channr, int dev_id); + +int pnx4008_get_dum_channel_config(int channr, int dev_id); + +int pnx4008_sdum_mmap(struct fb_info *info, struct vm_area_struct *vma, struct device *dev); +int pnx4008_set_dum_exit_notification(int dev_id); + +int pnx4008_get_fb_addresses(int fb_type, void **virt_addr, + dma_addr_t * phys_addr, int *fb_length); diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index d0434406eae..f42e64210ee 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -84,7 +84,7 @@ static struct linux_binfmt elf_format = { .min_coredump = ELF_EXEC_PAGESIZE }; -#define BAD_ADDR(x) ((unsigned long)(x) > TASK_SIZE) +#define BAD_ADDR(x) ((unsigned long)(x) >= TASK_SIZE) static int set_brk(unsigned long start, unsigned long end) { @@ -394,7 +394,7 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, * <= p_memsize so it's only necessary to check p_memsz. */ k = load_addr + eppnt->p_vaddr; - if (k > TASK_SIZE || + if (BAD_ADDR(k) || eppnt->p_filesz > eppnt->p_memsz || eppnt->p_memsz > TASK_SIZE || TASK_SIZE - eppnt->p_memsz < k) { @@ -887,7 +887,7 @@ static int load_elf_binary(struct linux_binprm *bprm, struct pt_regs *regs) * allowed task size. Note that p_filesz must always be * <= p_memsz so it is only necessary to check p_memsz. */ - if (k > TASK_SIZE || elf_ppnt->p_filesz > elf_ppnt->p_memsz || + if (BAD_ADDR(k) || elf_ppnt->p_filesz > elf_ppnt->p_memsz || elf_ppnt->p_memsz > TASK_SIZE || TASK_SIZE - elf_ppnt->p_memsz < k) { /* set_brk can never work. Avoid overflows. */ @@ -941,10 +941,9 @@ static int load_elf_binary(struct linux_binprm *bprm, struct pt_regs *regs) interpreter, &interp_load_addr); if (BAD_ADDR(elf_entry)) { - printk(KERN_ERR "Unable to load interpreter %.128s\n", - elf_interpreter); force_sig(SIGSEGV, current); - retval = -ENOEXEC; /* Nobody gets to see this, but.. */ + retval = IS_ERR((void *)elf_entry) ? + (int)elf_entry : -EINVAL; goto out_free_dentry; } reloc_func_desc = interp_load_addr; @@ -955,8 +954,8 @@ static int load_elf_binary(struct linux_binprm *bprm, struct pt_regs *regs) } else { elf_entry = loc->elf_ex.e_entry; if (BAD_ADDR(elf_entry)) { - send_sig(SIGSEGV, current, 0); - retval = -ENOEXEC; /* Nobody gets to see this, but.. */ + force_sig(SIGSEGV, current); + retval = -EINVAL; goto out_free_dentry; } } diff --git a/fs/block_dev.c b/fs/block_dev.c index 9633a490dab..37534573960 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -739,7 +739,7 @@ static int bd_claim_by_kobject(struct block_device *bdev, void *holder, if (!bo) return -ENOMEM; - mutex_lock(&bdev->bd_mutex); + mutex_lock_nested(&bdev->bd_mutex, BD_MUTEX_PARTITION); res = bd_claim(bdev, holder); if (res || !add_bd_holder(bdev, bo)) free_bd_holder(bo); @@ -764,7 +764,7 @@ static void bd_release_from_kobject(struct block_device *bdev, if (!kobj) return; - mutex_lock(&bdev->bd_mutex); + mutex_lock_nested(&bdev->bd_mutex, BD_MUTEX_PARTITION); bd_release(bdev); if ((bo = del_bd_holder(bdev, kobj))) free_bd_holder(bo); @@ -822,6 +822,22 @@ struct block_device *open_by_devnum(dev_t dev, unsigned mode) EXPORT_SYMBOL(open_by_devnum); +static int +blkdev_get_partition(struct block_device *bdev, mode_t mode, unsigned flags); + +struct block_device *open_partition_by_devnum(dev_t dev, unsigned mode) +{ + struct block_device *bdev = bdget(dev); + int err = -ENOMEM; + int flags = mode & FMODE_WRITE ? O_RDWR : O_RDONLY; + if (bdev) + err = blkdev_get_partition(bdev, mode, flags); + return err ? ERR_PTR(err) : bdev; +} + +EXPORT_SYMBOL(open_partition_by_devnum); + + /* * This routine checks whether a removable media has been changed, * and invalidates all buffer-cache-entries in that case. This @@ -868,7 +884,11 @@ void bd_set_size(struct block_device *bdev, loff_t size) } EXPORT_SYMBOL(bd_set_size); -static int do_open(struct block_device *bdev, struct file *file) +static int +blkdev_get_whole(struct block_device *bdev, mode_t mode, unsigned flags); + +static int +do_open(struct block_device *bdev, struct file *file, unsigned int subclass) { struct module *owner = NULL; struct gendisk *disk; @@ -885,7 +905,8 @@ static int do_open(struct block_device *bdev, struct file *file) } owner = disk->fops->owner; - mutex_lock(&bdev->bd_mutex); + mutex_lock_nested(&bdev->bd_mutex, subclass); + if (!bdev->bd_openers) { bdev->bd_disk = disk; bdev->bd_contains = bdev; @@ -912,11 +933,11 @@ static int do_open(struct block_device *bdev, struct file *file) ret = -ENOMEM; if (!whole) goto out_first; - ret = blkdev_get(whole, file->f_mode, file->f_flags); + ret = blkdev_get_whole(whole, file->f_mode, file->f_flags); if (ret) goto out_first; bdev->bd_contains = whole; - mutex_lock(&whole->bd_mutex); + mutex_lock_nested(&whole->bd_mutex, BD_MUTEX_WHOLE); whole->bd_part_count++; p = disk->part[part - 1]; bdev->bd_inode->i_data.backing_dev_info = @@ -944,7 +965,8 @@ static int do_open(struct block_device *bdev, struct file *file) if (bdev->bd_invalidated) rescan_partitions(bdev->bd_disk, bdev); } else { - mutex_lock(&bdev->bd_contains->bd_mutex); + mutex_lock_nested(&bdev->bd_contains->bd_mutex, + BD_MUTEX_PARTITION); bdev->bd_contains->bd_part_count++; mutex_unlock(&bdev->bd_contains->bd_mutex); } @@ -985,11 +1007,49 @@ int blkdev_get(struct block_device *bdev, mode_t mode, unsigned flags) fake_file.f_dentry = &fake_dentry; fake_dentry.d_inode = bdev->bd_inode; - return do_open(bdev, &fake_file); + return do_open(bdev, &fake_file, BD_MUTEX_NORMAL); } EXPORT_SYMBOL(blkdev_get); +static int +blkdev_get_whole(struct block_device *bdev, mode_t mode, unsigned flags) +{ + /* + * This crockload is due to bad choice of ->open() type. + * It will go away. + * For now, block device ->open() routine must _not_ + * examine anything in 'inode' argument except ->i_rdev. + */ + struct file fake_file = {}; + struct dentry fake_dentry = {}; + fake_file.f_mode = mode; + fake_file.f_flags = flags; + fake_file.f_dentry = &fake_dentry; + fake_dentry.d_inode = bdev->bd_inode; + + return do_open(bdev, &fake_file, BD_MUTEX_WHOLE); +} + +static int +blkdev_get_partition(struct block_device *bdev, mode_t mode, unsigned flags) +{ + /* + * This crockload is due to bad choice of ->open() type. + * It will go away. + * For now, block device ->open() routine must _not_ + * examine anything in 'inode' argument except ->i_rdev. + */ + struct file fake_file = {}; + struct dentry fake_dentry = {}; + fake_file.f_mode = mode; + fake_file.f_flags = flags; + fake_file.f_dentry = &fake_dentry; + fake_dentry.d_inode = bdev->bd_inode; + + return do_open(bdev, &fake_file, BD_MUTEX_PARTITION); +} + static int blkdev_open(struct inode * inode, struct file * filp) { struct block_device *bdev; @@ -1005,7 +1065,7 @@ static int blkdev_open(struct inode * inode, struct file * filp) bdev = bd_acquire(inode); - res = do_open(bdev, filp); + res = do_open(bdev, filp, BD_MUTEX_NORMAL); if (res) return res; @@ -1019,13 +1079,13 @@ static int blkdev_open(struct inode * inode, struct file * filp) return res; } -int blkdev_put(struct block_device *bdev) +static int __blkdev_put(struct block_device *bdev, unsigned int subclass) { int ret = 0; struct inode *bd_inode = bdev->bd_inode; struct gendisk *disk = bdev->bd_disk; - mutex_lock(&bdev->bd_mutex); + mutex_lock_nested(&bdev->bd_mutex, subclass); lock_kernel(); if (!--bdev->bd_openers) { sync_blockdev(bdev); @@ -1035,7 +1095,8 @@ int blkdev_put(struct block_device *bdev) if (disk->fops->release) ret = disk->fops->release(bd_inode, NULL); } else { - mutex_lock(&bdev->bd_contains->bd_mutex); + mutex_lock_nested(&bdev->bd_contains->bd_mutex, + subclass + 1); bdev->bd_contains->bd_part_count--; mutex_unlock(&bdev->bd_contains->bd_mutex); } @@ -1051,9 +1112,8 @@ int blkdev_put(struct block_device *bdev) } bdev->bd_disk = NULL; bdev->bd_inode->i_data.backing_dev_info = &default_backing_dev_info; - if (bdev != bdev->bd_contains) { - blkdev_put(bdev->bd_contains); - } + if (bdev != bdev->bd_contains) + __blkdev_put(bdev->bd_contains, subclass + 1); bdev->bd_contains = NULL; } unlock_kernel(); @@ -1062,8 +1122,20 @@ int blkdev_put(struct block_device *bdev) return ret; } +int blkdev_put(struct block_device *bdev) +{ + return __blkdev_put(bdev, BD_MUTEX_NORMAL); +} + EXPORT_SYMBOL(blkdev_put); +int blkdev_put_partition(struct block_device *bdev) +{ + return __blkdev_put(bdev, BD_MUTEX_PARTITION); +} + +EXPORT_SYMBOL(blkdev_put_partition); + static int blkdev_close(struct inode * inode, struct file * filp) { struct block_device *bdev = I_BDEV(filp->f_mapping->host); diff --git a/fs/dcache.c b/fs/dcache.c index c6e3535be19..1b4a3a34ec5 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -38,7 +38,7 @@ int sysctl_vfs_cache_pressure __read_mostly = 100; EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); -static seqlock_t rename_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; +static __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); EXPORT_SYMBOL(dcache_lock); @@ -1339,10 +1339,10 @@ void d_move(struct dentry * dentry, struct dentry * target) */ if (target < dentry) { spin_lock(&target->d_lock); - spin_lock(&dentry->d_lock); + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); } else { spin_lock(&dentry->d_lock); - spin_lock(&target->d_lock); + spin_lock_nested(&target->d_lock, DENTRY_D_LOCK_NESTED); } /* Move the dentry to the target hash queue, if on different bucket */ diff --git a/fs/direct-io.c b/fs/direct-io.c index 538fb0418fb..5981e17f46f 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -220,7 +220,8 @@ static void dio_complete(struct dio *dio, loff_t offset, ssize_t bytes) if (dio->end_io && dio->result) dio->end_io(dio->iocb, offset, bytes, dio->map_bh.b_private); if (dio->lock_type == DIO_LOCKING) - up_read(&dio->inode->i_alloc_sem); + /* lockdep: non-owner release */ + up_read_non_owner(&dio->inode->i_alloc_sem); } /* @@ -1261,7 +1262,8 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, } if (dio_lock_type == DIO_LOCKING) - down_read(&inode->i_alloc_sem); + /* lockdep: not the owner will release it */ + down_read_non_owner(&inode->i_alloc_sem); } /* diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 9c677bbd0b0..19ffb043abb 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -120,7 +120,7 @@ struct epoll_filefd { */ struct wake_task_node { struct list_head llink; - task_t *task; + struct task_struct *task; wait_queue_head_t *wq; }; @@ -413,7 +413,7 @@ static void ep_poll_safewake(struct poll_safewake *psw, wait_queue_head_t *wq) { int wake_nests = 0; unsigned long flags; - task_t *this_task = current; + struct task_struct *this_task = current; struct list_head *lsthead = &psw->wake_task_list, *lnk; struct wake_task_node *tncur; struct wake_task_node tnode; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 9f43879d6d6..f2702cda977 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -1157,7 +1157,7 @@ static ssize_t ext2_quota_write(struct super_block *sb, int type, struct buffer_head tmp_bh; struct buffer_head *bh; - mutex_lock(&inode->i_mutex); + mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA); while (towrite > 0) { tocopy = sb->s_blocksize - offset < towrite ? sb->s_blocksize - offset : towrite; diff --git a/fs/ext3/super.c b/fs/ext3/super.c index f2dd7133661..813d589cc6c 100644 --- a/fs/ext3/super.c +++ b/fs/ext3/super.c @@ -2614,7 +2614,7 @@ static ssize_t ext3_quota_write(struct super_block *sb, int type, struct buffer_head *bh; handle_t *handle = journal_current_handle(); - mutex_lock(&inode->i_mutex); + mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA); while (towrite > 0) { tocopy = sb->s_blocksize - offset < towrite ? sb->s_blocksize - offset : towrite; diff --git a/fs/namei.c b/fs/namei.c index c784e8bb57a..c9750d755af 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1423,7 +1423,7 @@ struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) struct dentry *p; if (p1 == p2) { - mutex_lock(&p1->d_inode->i_mutex); + mutex_lock_nested(&p1->d_inode->i_mutex, I_MUTEX_PARENT); return NULL; } @@ -1431,22 +1431,22 @@ struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) for (p = p1; p->d_parent != p; p = p->d_parent) { if (p->d_parent == p2) { - mutex_lock(&p2->d_inode->i_mutex); - mutex_lock(&p1->d_inode->i_mutex); + mutex_lock_nested(&p2->d_inode->i_mutex, I_MUTEX_PARENT); + mutex_lock_nested(&p1->d_inode->i_mutex, I_MUTEX_CHILD); return p; } } for (p = p2; p->d_parent != p; p = p->d_parent) { if (p->d_parent == p1) { - mutex_lock(&p1->d_inode->i_mutex); - mutex_lock(&p2->d_inode->i_mutex); + mutex_lock_nested(&p1->d_inode->i_mutex, I_MUTEX_PARENT); + mutex_lock_nested(&p2->d_inode->i_mutex, I_MUTEX_CHILD); return p; } } - mutex_lock(&p1->d_inode->i_mutex); - mutex_lock(&p2->d_inode->i_mutex); + mutex_lock_nested(&p1->d_inode->i_mutex, I_MUTEX_PARENT); + mutex_lock_nested(&p2->d_inode->i_mutex, I_MUTEX_CHILD); return NULL; } @@ -1751,7 +1751,7 @@ struct dentry *lookup_create(struct nameidata *nd, int is_dir) { struct dentry *dentry = ERR_PTR(-EEXIST); - mutex_lock(&nd->dentry->d_inode->i_mutex); + mutex_lock_nested(&nd->dentry->d_inode->i_mutex, I_MUTEX_PARENT); /* * Yucky last component or no last component at all? * (foo/., foo/.., /////) @@ -2008,7 +2008,7 @@ static long do_rmdir(int dfd, const char __user *pathname) error = -EBUSY; goto exit1; } - mutex_lock(&nd.dentry->d_inode->i_mutex); + mutex_lock_nested(&nd.dentry->d_inode->i_mutex, I_MUTEX_PARENT); dentry = lookup_hash(&nd); error = PTR_ERR(dentry); if (!IS_ERR(dentry)) { @@ -2082,7 +2082,7 @@ static long do_unlinkat(int dfd, const char __user *pathname) error = -EISDIR; if (nd.last_type != LAST_NORM) goto exit1; - mutex_lock(&nd.dentry->d_inode->i_mutex); + mutex_lock_nested(&nd.dentry->d_inode->i_mutex, I_MUTEX_PARENT); dentry = lookup_hash(&nd); error = PTR_ERR(dentry); if (!IS_ERR(dentry)) { diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 4c86b7e1d1e..d313f356e66 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -367,6 +367,12 @@ static void ntfs_destroy_extent_inode(ntfs_inode *ni) kmem_cache_free(ntfs_inode_cache, ni); } +/* + * The attribute runlist lock has separate locking rules from the + * normal runlist lock, so split the two lock-classes: + */ +static struct lock_class_key attr_list_rl_lock_class; + /** * __ntfs_init_inode - initialize ntfs specific part of an inode * @sb: super block of mounted volume @@ -394,6 +400,8 @@ void __ntfs_init_inode(struct super_block *sb, ntfs_inode *ni) ni->attr_list_size = 0; ni->attr_list = NULL; ntfs_init_runlist(&ni->attr_list_rl); + lockdep_set_class(&ni->attr_list_rl.lock, + &attr_list_rl_lock_class); ni->itype.index.bmp_ino = NULL; ni->itype.index.block_size = 0; ni->itype.index.vcn_size = 0; @@ -405,6 +413,13 @@ void __ntfs_init_inode(struct super_block *sb, ntfs_inode *ni) ni->ext.base_ntfs_ino = NULL; } +/* + * Extent inodes get MFT-mapped in a nested way, while the base inode + * is still mapped. Teach this nesting to the lock validator by creating + * a separate class for nested inode's mrec_lock's: + */ +static struct lock_class_key extent_inode_mrec_lock_key; + inline ntfs_inode *ntfs_new_extent_inode(struct super_block *sb, unsigned long mft_no) { @@ -413,6 +428,7 @@ inline ntfs_inode *ntfs_new_extent_inode(struct super_block *sb, ntfs_debug("Entering."); if (likely(ni != NULL)) { __ntfs_init_inode(sb, ni); + lockdep_set_class(&ni->mrec_lock, &extent_inode_mrec_lock_key); ni->mft_no = mft_no; ni->type = AT_UNUSED; ni->name = NULL; @@ -1722,6 +1738,15 @@ err_out: return err; } +/* + * The MFT inode has special locking, so teach the lock validator + * about this by splitting off the locking rules of the MFT from + * the locking rules of other inodes. The MFT inode can never be + * accessed from the VFS side (or even internally), only by the + * map_mft functions. + */ +static struct lock_class_key mft_ni_runlist_lock_key, mft_ni_mrec_lock_key; + /** * ntfs_read_inode_mount - special read_inode for mount time use only * @vi: inode to read @@ -2148,6 +2173,14 @@ int ntfs_read_inode_mount(struct inode *vi) ntfs_attr_put_search_ctx(ctx); ntfs_debug("Done."); ntfs_free(m); + + /* + * Split the locking rules of the MFT inode from the + * locking rules of other inodes: + */ + lockdep_set_class(&ni->runlist.lock, &mft_ni_runlist_lock_key); + lockdep_set_class(&ni->mrec_lock, &mft_ni_mrec_lock_key); + return 0; em_put_err_out: diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 0e14acea3f8..74e0ee8fce7 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -1724,6 +1724,14 @@ upcase_failed: return FALSE; } +/* + * The lcn and mft bitmap inodes are NTFS-internal inodes with + * their own special locking rules: + */ +static struct lock_class_key + lcnbmp_runlist_lock_key, lcnbmp_mrec_lock_key, + mftbmp_runlist_lock_key, mftbmp_mrec_lock_key; + /** * load_system_files - open the system files using normal functions * @vol: ntfs super block describing device whose system files to load @@ -1780,6 +1788,10 @@ static BOOL load_system_files(ntfs_volume *vol) ntfs_error(sb, "Failed to load $MFT/$BITMAP attribute."); goto iput_mirr_err_out; } + lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->runlist.lock, + &mftbmp_runlist_lock_key); + lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->mrec_lock, + &mftbmp_mrec_lock_key); /* Read upcase table and setup @vol->upcase and @vol->upcase_len. */ if (!load_and_init_upcase(vol)) goto iput_mftbmp_err_out; @@ -1802,6 +1814,11 @@ static BOOL load_system_files(ntfs_volume *vol) iput(vol->lcnbmp_ino); goto bitmap_failed; } + lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->runlist.lock, + &lcnbmp_runlist_lock_key); + lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->mrec_lock, + &lcnbmp_mrec_lock_key); + NInoSetSparseDisabled(NTFS_I(vol->lcnbmp_ino)); if ((vol->nr_clusters + 7) >> 3 > i_size_read(vol->lcnbmp_ino)) { iput(vol->lcnbmp_ino); @@ -2743,6 +2760,17 @@ static int ntfs_fill_super(struct super_block *sb, void *opt, const int silent) struct inode *tmp_ino; int blocksize, result; + /* + * We do a pretty difficult piece of bootstrap by reading the + * MFT (and other metadata) from disk into memory. We'll only + * release this metadata during umount, so the locking patterns + * observed during bootstrap do not count. So turn off the + * observation of locking patterns (strictly for this context + * only) while mounting NTFS. [The validator is still active + * otherwise, even for this context: it will for example record + * lock class registrations.] + */ + lockdep_off(); ntfs_debug("Entering."); #ifndef NTFS_RW sb->s_flags |= MS_RDONLY; @@ -2754,6 +2782,7 @@ static int ntfs_fill_super(struct super_block *sb, void *opt, const int silent) if (!silent) ntfs_error(sb, "Allocation of NTFS volume structure " "failed. Aborting mount..."); + lockdep_on(); return -ENOMEM; } /* Initialize ntfs_volume structure. */ @@ -2940,6 +2969,7 @@ static int ntfs_fill_super(struct super_block *sb, void *opt, const int silent) mutex_unlock(&ntfs_lock); sb->s_export_op = &ntfs_export_ops; lock_kernel(); + lockdep_on(); return 0; } ntfs_error(sb, "Failed to allocate root directory."); @@ -3059,6 +3089,7 @@ err_out_now: sb->s_fs_info = NULL; kfree(vol); ntfs_debug("Failed, returning -EINVAL."); + lockdep_on(); return -EINVAL; } diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index 28eb3c88603..5567328f104 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -2203,7 +2203,7 @@ static ssize_t reiserfs_quota_write(struct super_block *sb, int type, size_t towrite = len; struct buffer_head tmp_bh, *bh; - mutex_lock(&inode->i_mutex); + mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA); while (towrite > 0) { tocopy = sb->s_blocksize - offset < towrite ? sb->s_blocksize - offset : towrite; diff --git a/fs/super.c b/fs/super.c index 9b780c42d84..6d4e8174b6d 100644 --- a/fs/super.c +++ b/fs/super.c @@ -53,7 +53,7 @@ DEFINE_SPINLOCK(sb_lock); * Allocates and initializes a new &struct super_block. alloc_super() * returns a pointer new superblock or %NULL if allocation had failed. */ -static struct super_block *alloc_super(void) +static struct super_block *alloc_super(struct file_system_type *type) { struct super_block *s = kzalloc(sizeof(struct super_block), GFP_USER); static struct super_operations default_op; @@ -72,6 +72,13 @@ static struct super_block *alloc_super(void) INIT_LIST_HEAD(&s->s_inodes); init_rwsem(&s->s_umount); mutex_init(&s->s_lock); + lockdep_set_class(&s->s_umount, &type->s_umount_key); + /* + * The locking rules for s_lock are up to the + * filesystem. For example ext3fs has different + * lock ordering than usbfs: + */ + lockdep_set_class(&s->s_lock, &type->s_lock_key); down_write(&s->s_umount); s->s_count = S_BIAS; atomic_set(&s->s_active, 1); @@ -295,7 +302,7 @@ retry: } if (!s) { spin_unlock(&sb_lock); - s = alloc_super(); + s = alloc_super(type); if (!s) return ERR_PTR(-ENOMEM); goto retry; diff --git a/fs/ufs/super.c b/fs/ufs/super.c index 19a99726e58..992ee0b87cc 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1326,7 +1326,7 @@ static ssize_t ufs_quota_write(struct super_block *sb, int type, size_t towrite = len; struct buffer_head *bh; - mutex_lock(&inode->i_mutex); + mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA); while (towrite > 0) { tocopy = sb->s_blocksize - offset < towrite ? sb->s_blocksize - offset : towrite; diff --git a/include/asm-alpha/rwsem.h b/include/asm-alpha/rwsem.h index fafdd4f7010..1570c0b5433 100644 --- a/include/asm-alpha/rwsem.h +++ b/include/asm-alpha/rwsem.h @@ -36,20 +36,11 @@ struct rw_semaphore { #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif }; -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) __RWSEM_DEBUG_INIT } + LIST_HEAD_INIT((name).wait_list) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -59,9 +50,6 @@ static inline void init_rwsem(struct rw_semaphore *sem) sem->count = RWSEM_UNLOCKED_VALUE; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } static inline void __down_read(struct rw_semaphore *sem) diff --git a/include/asm-arm/floppy.h b/include/asm-arm/floppy.h index aa0c8d28d8d..54b5ae44ed9 100644 --- a/include/asm-arm/floppy.h +++ b/include/asm-arm/floppy.h @@ -25,7 +25,7 @@ #define fd_inb(port) inb((port)) #define fd_request_irq() request_irq(IRQ_FLOPPYDISK,floppy_interrupt,\ - SA_INTERRUPT,"floppy",NULL) + IRQF_DISABLED,"floppy",NULL) #define fd_free_irq() free_irq(IRQ_FLOPPYDISK,NULL) #define fd_disable_irq() disable_irq(IRQ_FLOPPYDISK) #define fd_enable_irq() enable_irq(IRQ_FLOPPYDISK) diff --git a/include/asm-arm/hw_irq.h b/include/asm-arm/hw_irq.h index f1a08a50060..ea856971989 100644 --- a/include/asm-arm/hw_irq.h +++ b/include/asm-arm/hw_irq.h @@ -6,4 +6,15 @@ #include <asm/mach/irq.h> +#if defined(CONFIG_NO_IDLE_HZ) +# include <asm/dyntick.h> +# define handle_dynamic_tick(action) \ + if (!(action->flags & IRQF_TIMER) && system_timer->dyn_tick) { \ + write_seqlock(&xtime_lock); \ + if (system_timer->dyn_tick->state & DYN_TICK_ENABLED) \ + system_timer->dyn_tick->handler(irq, 0, regs); \ + write_sequnlock(&xtime_lock); \ + } +#endif + #endif diff --git a/include/asm-arm/mach/time.h b/include/asm-arm/mach/time.h index 9f28073559e..dee0bc336fe 100644 --- a/include/asm-arm/mach/time.h +++ b/include/asm-arm/mach/time.h @@ -69,6 +69,7 @@ extern void timer_tick(struct pt_regs *); /* * Kernel time keeping support. */ +struct timespec; extern int (*set_rtc)(void); extern void save_time_delta(struct timespec *delta, struct timespec *rtc); extern void restore_time_delta(struct timespec *delta, struct timespec *rtc); diff --git a/include/asm-arm/signal.h b/include/asm-arm/signal.h index ced69161917..d0fb487aba4 100644 --- a/include/asm-arm/signal.h +++ b/include/asm-arm/signal.h @@ -82,7 +82,6 @@ typedef unsigned long sigset_t; * is running in 26-bit. * SA_ONSTACK allows alternate signal stacks (see sigaltstack(2)). * SA_RESTART flag to get restarting signals (which were the default long ago) - * SA_INTERRUPT is a no-op, but left due to historical reasons. Use the * SA_NODEFER prevents the current signal from being masked in the handler. * SA_RESETHAND clears the handler when the signal is delivered. * @@ -101,7 +100,6 @@ typedef unsigned long sigset_t; #define SA_NOMASK SA_NODEFER #define SA_ONESHOT SA_RESETHAND -#define SA_INTERRUPT 0x20000000 /* dummy -- ignored */ /* @@ -113,10 +111,6 @@ typedef unsigned long sigset_t; #define MINSIGSTKSZ 2048 #define SIGSTKSZ 8192 -#ifdef __KERNEL__ -#define SA_TIMER 0x40000000 -#endif - #include <asm-generic/signal.h> #ifdef __KERNEL__ diff --git a/include/asm-generic/mutex-null.h b/include/asm-generic/mutex-null.h index 5cf8b7ce0c4..254a126ede5 100644 --- a/include/asm-generic/mutex-null.h +++ b/include/asm-generic/mutex-null.h @@ -10,15 +10,10 @@ #ifndef _ASM_GENERIC_MUTEX_NULL_H #define _ASM_GENERIC_MUTEX_NULL_H -/* extra parameter only needed for mutex debugging: */ -#ifndef __IP__ -# define __IP__ -#endif - -#define __mutex_fastpath_lock(count, fail_fn) fail_fn(count __RET_IP__) -#define __mutex_fastpath_lock_retval(count, fail_fn) fail_fn(count __RET_IP__) -#define __mutex_fastpath_unlock(count, fail_fn) fail_fn(count __RET_IP__) -#define __mutex_fastpath_trylock(count, fail_fn) fail_fn(count) -#define __mutex_slowpath_needs_to_unlock() 1 +#define __mutex_fastpath_lock(count, fail_fn) fail_fn(count) +#define __mutex_fastpath_lock_retval(count, fail_fn) fail_fn(count) +#define __mutex_fastpath_unlock(count, fail_fn) fail_fn(count) +#define __mutex_fastpath_trylock(count, fail_fn) fail_fn(count) +#define __mutex_slowpath_needs_to_unlock() 1 #endif diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index c7452115746..e160e04290f 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -7,6 +7,8 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; +#define per_cpu_offset(x) (__per_cpu_offset[x]) + /* Separate out the type, so (int[3], foo) works. */ #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name diff --git a/include/asm-i386/irqflags.h b/include/asm-i386/irqflags.h new file mode 100644 index 00000000000..e1bdb97c07f --- /dev/null +++ b/include/asm-i386/irqflags.h @@ -0,0 +1,127 @@ +/* + * include/asm-i386/irqflags.h + * + * IRQ flags handling + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * raw_local_irq_*() functions from the lowlevel headers. + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +#ifndef __ASSEMBLY__ + +static inline unsigned long __raw_local_save_flags(void) +{ + unsigned long flags; + + __asm__ __volatile__( + "pushfl ; popl %0" + : "=g" (flags) + : /* no input */ + ); + + return flags; +} + +#define raw_local_save_flags(flags) \ + do { (flags) = __raw_local_save_flags(); } while (0) + +static inline void raw_local_irq_restore(unsigned long flags) +{ + __asm__ __volatile__( + "pushl %0 ; popfl" + : /* no output */ + :"g" (flags) + :"memory", "cc" + ); +} + +static inline void raw_local_irq_disable(void) +{ + __asm__ __volatile__("cli" : : : "memory"); +} + +static inline void raw_local_irq_enable(void) +{ + __asm__ __volatile__("sti" : : : "memory"); +} + +/* + * Used in the idle loop; sti takes one instruction cycle + * to complete: + */ +static inline void raw_safe_halt(void) +{ + __asm__ __volatile__("sti; hlt" : : : "memory"); +} + +/* + * Used when interrupts are already enabled or to + * shutdown the processor: + */ +static inline void halt(void) +{ + __asm__ __volatile__("hlt": : :"memory"); +} + +static inline int raw_irqs_disabled_flags(unsigned long flags) +{ + return !(flags & (1 << 9)); +} + +static inline int raw_irqs_disabled(void) +{ + unsigned long flags = __raw_local_save_flags(); + + return raw_irqs_disabled_flags(flags); +} + +/* + * For spinlocks, etc: + */ +static inline unsigned long __raw_local_irq_save(void) +{ + unsigned long flags = __raw_local_save_flags(); + + raw_local_irq_disable(); + + return flags; +} + +#define raw_local_irq_save(flags) \ + do { (flags) = __raw_local_irq_save(); } while (0) + +#endif /* __ASSEMBLY__ */ + +/* + * Do the CPU's IRQ-state tracing from assembly code. We call a + * C function, so save all the C-clobbered registers: + */ +#ifdef CONFIG_TRACE_IRQFLAGS + +# define TRACE_IRQS_ON \ + pushl %eax; \ + pushl %ecx; \ + pushl %edx; \ + call trace_hardirqs_on; \ + popl %edx; \ + popl %ecx; \ + popl %eax; + +# define TRACE_IRQS_OFF \ + pushl %eax; \ + pushl %ecx; \ + pushl %edx; \ + call trace_hardirqs_off; \ + popl %edx; \ + popl %ecx; \ + popl %eax; + +#else +# define TRACE_IRQS_ON +# define TRACE_IRQS_OFF +#endif + +#endif diff --git a/include/asm-i386/rwsem.h b/include/asm-i386/rwsem.h index be4ab859238..2f07601562e 100644 --- a/include/asm-i386/rwsem.h +++ b/include/asm-i386/rwsem.h @@ -40,6 +40,7 @@ #include <linux/list.h> #include <linux/spinlock.h> +#include <linux/lockdep.h> struct rwsem_waiter; @@ -61,36 +62,34 @@ struct rw_semaphore { #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; #endif }; -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname } #else -#define __RWSEM_DEBUG_INIT /* */ +# define __RWSEM_DEP_MAP_INIT(lockname) #endif + #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } + __RWSEM_DEP_MAP_INIT(name) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif -} +extern void __init_rwsem(struct rw_semaphore *sem, const char *name, + struct lock_class_key *key); + +#define init_rwsem(sem) \ +do { \ + static struct lock_class_key __key; \ + \ + __init_rwsem((sem), #sem, &__key); \ +} while (0) /* * lock for reading @@ -143,7 +142,7 @@ LOCK_PREFIX " cmpxchgl %2,%0\n\t" /* * lock for writing */ -static inline void __down_write(struct rw_semaphore *sem) +static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) { int tmp; @@ -167,6 +166,11 @@ LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the : "memory", "cc"); } +static inline void __down_write(struct rw_semaphore *sem) +{ + __down_write_nested(sem, 0); +} + /* * trylock for writing -- returns 1 if successful, 0 if contention */ diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index 04ba30234c4..87c40f83065 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -31,6 +31,11 @@ "jmp 1b\n" \ "3:\n\t" +/* + * NOTE: there's an irqs-on section here, which normally would have to be + * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use + * __raw_spin_lock_string_flags(). + */ #define __raw_spin_lock_string_flags \ "\n1:\t" \ "lock ; decb %0\n\t" \ @@ -63,6 +68,12 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock) "=m" (lock->slock) : : "memory"); } +/* + * It is easier for the lock validator if interrupts are not re-enabled + * in the middle of a lock-acquire. This is a performance feature anyway + * so we turn it off: + */ +#ifndef CONFIG_PROVE_LOCKING static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) { alternative_smp( @@ -70,6 +81,7 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla __raw_spin_lock_string_up, "=m" (lock->slock) : "r" (flags) : "memory"); } +#endif static inline int __raw_spin_trylock(raw_spinlock_t *lock) { diff --git a/include/asm-i386/system.h b/include/asm-i386/system.h index cab0180567f..db398d88b1d 100644 --- a/include/asm-i386/system.h +++ b/include/asm-i386/system.h @@ -456,25 +456,7 @@ static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long l #define set_wmb(var, value) do { var = value; wmb(); } while (0) -/* interrupt control.. */ -#define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */); } while (0) -#define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0) -#define local_irq_disable() __asm__ __volatile__("cli": : :"memory") -#define local_irq_enable() __asm__ __volatile__("sti": : :"memory") -/* used in the idle loop; sti takes one instruction cycle to complete */ -#define safe_halt() __asm__ __volatile__("sti; hlt": : :"memory") -/* used when interrupts are already enabled or to shutdown the processor */ -#define halt() __asm__ __volatile__("hlt": : :"memory") - -#define irqs_disabled() \ -({ \ - unsigned long flags; \ - local_save_flags(flags); \ - !(flags & (1<<9)); \ -}) - -/* For spinlocks etc */ -#define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") +#include <linux/irqflags.h> /* * disable hlt during certain critical i/o operations diff --git a/include/asm-ia64/irq.h b/include/asm-ia64/irq.h index 8acb00190d5..79479e2c696 100644 --- a/include/asm-ia64/irq.h +++ b/include/asm-ia64/irq.h @@ -14,8 +14,6 @@ #define NR_IRQS 256 #define NR_IRQ_VECTORS NR_IRQS -#define IRQF_PERCPU 0x02000000 - static __inline__ int irq_canonicalize (int irq) { diff --git a/include/asm-ia64/percpu.h b/include/asm-ia64/percpu.h index 24d898b650c..fbe5cf3ab8d 100644 --- a/include/asm-ia64/percpu.h +++ b/include/asm-ia64/percpu.h @@ -36,6 +36,7 @@ #ifdef CONFIG_SMP extern unsigned long __per_cpu_offset[NR_CPUS]; +#define per_cpu_offset(x) (__per_cpu_offset(x)) /* Equal to __per_cpu_offset[smp_processor_id()], but faster to access: */ DECLARE_PER_CPU(unsigned long, local_per_cpu_offset); diff --git a/include/asm-ia64/rwsem.h b/include/asm-ia64/rwsem.h index 1327c91ea39..2d1640cc240 100644 --- a/include/asm-ia64/rwsem.h +++ b/include/asm-ia64/rwsem.h @@ -33,9 +33,6 @@ struct rw_semaphore { signed long count; spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif }; #define RWSEM_UNLOCKED_VALUE __IA64_UL_CONST(0x0000000000000000) @@ -45,19 +42,9 @@ struct rw_semaphore { #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) -/* - * initialization - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } + LIST_HEAD_INIT((name).wait_list) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -73,9 +60,6 @@ init_rwsem (struct rw_semaphore *sem) sem->count = RWSEM_UNLOCKED_VALUE; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } /* diff --git a/include/asm-ia64/thread_info.h b/include/asm-ia64/thread_info.h index 8bc9869e576..8adcde0934c 100644 --- a/include/asm-ia64/thread_info.h +++ b/include/asm-ia64/thread_info.h @@ -68,7 +68,7 @@ struct thread_info { #define end_of_stack(p) (unsigned long *)((void *)(p) + IA64_RBS_OFFSET) #define __HAVE_ARCH_TASK_STRUCT_ALLOCATOR -#define alloc_task_struct() ((task_t *)__get_free_pages(GFP_KERNEL | __GFP_COMP, KERNEL_STACK_SIZE_ORDER)) +#define alloc_task_struct() ((struct task_struct *)__get_free_pages(GFP_KERNEL | __GFP_COMP, KERNEL_STACK_SIZE_ORDER)) #define free_task_struct(tsk) free_pages((unsigned long) (tsk), KERNEL_STACK_SIZE_ORDER) #endif /* !__ASSEMBLY */ diff --git a/include/asm-m32r/system.h b/include/asm-m32r/system.h index 66c4742f09e..311cebf44ef 100644 --- a/include/asm-m32r/system.h +++ b/include/asm-m32r/system.h @@ -18,7 +18,7 @@ * switch_to(prev, next) should switch from task `prev' to `next' * `prev' will never be the same as `next'. * - * `next' and `prev' should be task_t, but it isn't always defined + * `next' and `prev' should be struct task_struct, but it isn't always defined */ #define switch_to(prev, next, last) do { \ diff --git a/include/asm-powerpc/irqflags.h b/include/asm-powerpc/irqflags.h new file mode 100644 index 00000000000..7970cbaeaa5 --- /dev/null +++ b/include/asm-powerpc/irqflags.h @@ -0,0 +1,31 @@ +/* + * include/asm-powerpc/irqflags.h + * + * IRQ flags handling + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * raw_local_irq_*() macros from the lowlevel headers. + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +/* + * Get definitions for raw_local_save_flags(x), etc. + */ +#include <asm-powerpc/hw_irq.h> + +/* + * Do the CPU's IRQ-state tracing from assembly code. We call a + * C function, so save all the C-clobbered registers: + */ +#ifdef CONFIG_TRACE_IRQFLAGS + +#error No support on PowerPC yet for CONFIG_TRACE_IRQFLAGS + +#else +# define TRACE_IRQS_ON +# define TRACE_IRQS_OFF +#endif + +#endif diff --git a/include/asm-powerpc/percpu.h b/include/asm-powerpc/percpu.h index faa1fc70305..2f2e3024fa6 100644 --- a/include/asm-powerpc/percpu.h +++ b/include/asm-powerpc/percpu.h @@ -14,6 +14,7 @@ #define __per_cpu_offset(cpu) (paca[cpu].data_offset) #define __my_cpu_offset() get_paca()->data_offset +#define per_cpu_offset(x) (__per_cpu_offset(x)) /* Separate out the type, so (int[3], foo) works. */ #define DEFINE_PER_CPU(type, name) \ diff --git a/include/asm-powerpc/rwsem.h b/include/asm-powerpc/rwsem.h index 2c2fe964759..e929145e1e4 100644 --- a/include/asm-powerpc/rwsem.h +++ b/include/asm-powerpc/rwsem.h @@ -28,24 +28,11 @@ struct rw_semaphore { #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif }; -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } + LIST_HEAD_INIT((name).wait_list) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -60,9 +47,6 @@ static inline void init_rwsem(struct rw_semaphore *sem) sem->count = RWSEM_UNLOCKED_VALUE; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } /* diff --git a/include/asm-s390/irqflags.h b/include/asm-s390/irqflags.h new file mode 100644 index 00000000000..65f4db627e7 --- /dev/null +++ b/include/asm-s390/irqflags.h @@ -0,0 +1,50 @@ +/* + * include/asm-s390/irqflags.h + * + * Copyright (C) IBM Corp. 2006 + * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> + */ + +#ifndef __ASM_IRQFLAGS_H +#define __ASM_IRQFLAGS_H + +#ifdef __KERNEL__ + +/* interrupt control.. */ +#define raw_local_irq_enable() ({ \ + unsigned long __dummy; \ + __asm__ __volatile__ ( \ + "stosm 0(%1),0x03" \ + : "=m" (__dummy) : "a" (&__dummy) : "memory" ); \ + }) + +#define raw_local_irq_disable() ({ \ + unsigned long __flags; \ + __asm__ __volatile__ ( \ + "stnsm 0(%1),0xfc" : "=m" (__flags) : "a" (&__flags) ); \ + __flags; \ + }) + +#define raw_local_save_flags(x) \ + __asm__ __volatile__("stosm 0(%1),0" : "=m" (x) : "a" (&x), "m" (x) ) + +#define raw_local_irq_restore(x) \ + __asm__ __volatile__("ssm 0(%0)" : : "a" (&x), "m" (x) : "memory") + +#define raw_irqs_disabled() \ +({ \ + unsigned long flags; \ + local_save_flags(flags); \ + !((flags >> __FLAG_SHIFT) & 3); \ +}) + +static inline int raw_irqs_disabled_flags(unsigned long flags) +{ + return !((flags >> __FLAG_SHIFT) & 3); +} + +/* For spinlocks etc */ +#define raw_local_irq_save(x) ((x) = raw_local_irq_disable()) + +#endif /* __KERNEL__ */ +#endif /* __ASM_IRQFLAGS_H */ diff --git a/include/asm-s390/percpu.h b/include/asm-s390/percpu.h index d9a8cca9b65..28b3517e787 100644 --- a/include/asm-s390/percpu.h +++ b/include/asm-s390/percpu.h @@ -42,6 +42,7 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define __get_cpu_var(var) __reloc_hide(var,S390_lowcore.percpu_offset) #define __raw_get_cpu_var(var) __reloc_hide(var,S390_lowcore.percpu_offset) #define per_cpu(var,cpu) __reloc_hide(var,__per_cpu_offset[cpu]) +#define per_cpu_offset(x) (__per_cpu_offset[x]) /* A macro to avoid #include hell... */ #define percpu_modcopy(pcpudst, src, size) \ diff --git a/include/asm-s390/rwsem.h b/include/asm-s390/rwsem.h index 0422a085dd5..13ec1696515 100644 --- a/include/asm-s390/rwsem.h +++ b/include/asm-s390/rwsem.h @@ -61,6 +61,9 @@ struct rw_semaphore { signed long count; spinlock_t wait_lock; struct list_head wait_list; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif }; #ifndef __s390x__ @@ -80,8 +83,16 @@ struct rw_semaphore { /* * initialisation */ + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname } +#else +# define __RWSEM_DEP_MAP_INIT(lockname) +#endif + #define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) } +{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) \ + __RWSEM_DEP_MAP_INIT(name) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -93,6 +104,17 @@ static inline void init_rwsem(struct rw_semaphore *sem) INIT_LIST_HEAD(&sem->wait_list); } +extern void __init_rwsem(struct rw_semaphore *sem, const char *name, + struct lock_class_key *key); + +#define init_rwsem(sem) \ +do { \ + static struct lock_class_key __key; \ + \ + __init_rwsem((sem), #sem, &__key); \ +} while (0) + + /* * lock for reading */ @@ -155,7 +177,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) /* * lock for writing */ -static inline void __down_write(struct rw_semaphore *sem) +static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) { signed long old, new, tmp; @@ -181,6 +203,11 @@ static inline void __down_write(struct rw_semaphore *sem) rwsem_down_write_failed(sem); } +static inline void __down_write(struct rw_semaphore *sem) +{ + __down_write_nested(sem, 0); +} + /* * trylock for writing -- returns 1 if successful, 0 if contention */ diff --git a/include/asm-s390/semaphore.h b/include/asm-s390/semaphore.h index 702cf436698..32cdc69f39f 100644 --- a/include/asm-s390/semaphore.h +++ b/include/asm-s390/semaphore.h @@ -37,7 +37,8 @@ struct semaphore { static inline void sema_init (struct semaphore *sem, int val) { - *sem = (struct semaphore) __SEMAPHORE_INITIALIZER((*sem),val); + atomic_set(&sem->count, val); + init_waitqueue_head(&sem->wait); } static inline void init_MUTEX (struct semaphore *sem) diff --git a/include/asm-s390/system.h b/include/asm-s390/system.h index 71a0732cd51..9ab186ffde2 100644 --- a/include/asm-s390/system.h +++ b/include/asm-s390/system.h @@ -301,34 +301,6 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) #define set_mb(var, value) do { var = value; mb(); } while (0) #define set_wmb(var, value) do { var = value; wmb(); } while (0) -/* interrupt control.. */ -#define local_irq_enable() ({ \ - unsigned long __dummy; \ - __asm__ __volatile__ ( \ - "stosm 0(%1),0x03" \ - : "=m" (__dummy) : "a" (&__dummy) : "memory" ); \ - }) - -#define local_irq_disable() ({ \ - unsigned long __flags; \ - __asm__ __volatile__ ( \ - "stnsm 0(%1),0xfc" : "=m" (__flags) : "a" (&__flags) ); \ - __flags; \ - }) - -#define local_save_flags(x) \ - __asm__ __volatile__("stosm 0(%1),0" : "=m" (x) : "a" (&x), "m" (x) ) - -#define local_irq_restore(x) \ - __asm__ __volatile__("ssm 0(%0)" : : "a" (&x), "m" (x) : "memory") - -#define irqs_disabled() \ -({ \ - unsigned long flags; \ - local_save_flags(flags); \ - !((flags >> __FLAG_SHIFT) & 3); \ -}) - #ifdef __s390x__ #define __ctl_load(array, low, high) ({ \ @@ -442,8 +414,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) }) #endif /* __s390x__ */ -/* For spinlocks etc */ -#define local_irq_save(x) ((x) = local_irq_disable()) +#include <linux/irqflags.h> /* * Use to set psw mask except for the first byte which @@ -482,4 +453,3 @@ extern void (*_machine_power_off)(void); #endif /* __KERNEL__ */ #endif - diff --git a/include/asm-sh/rwsem.h b/include/asm-sh/rwsem.h index 0262d3d1e5e..9d2aea5e848 100644 --- a/include/asm-sh/rwsem.h +++ b/include/asm-sh/rwsem.h @@ -25,24 +25,11 @@ struct rw_semaphore { #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif }; -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } + LIST_HEAD_INIT((name).wait_list) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -57,9 +44,6 @@ static inline void init_rwsem(struct rw_semaphore *sem) sem->count = RWSEM_UNLOCKED_VALUE; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } /* diff --git a/include/asm-sh/system.h b/include/asm-sh/system.h index b752e5cbb83..ce2e60664a8 100644 --- a/include/asm-sh/system.h +++ b/include/asm-sh/system.h @@ -12,7 +12,7 @@ */ #define switch_to(prev, next, last) do { \ - task_t *__last; \ + struct task_struct *__last; \ register unsigned long *__ts1 __asm__ ("r1") = &prev->thread.sp; \ register unsigned long *__ts2 __asm__ ("r2") = &prev->thread.pc; \ register unsigned long *__ts4 __asm__ ("r4") = (unsigned long *)prev; \ diff --git a/include/asm-sparc64/percpu.h b/include/asm-sparc64/percpu.h index a6ece06b83d..ced8cbde046 100644 --- a/include/asm-sparc64/percpu.h +++ b/include/asm-sparc64/percpu.h @@ -11,6 +11,7 @@ extern unsigned long __per_cpu_base; extern unsigned long __per_cpu_shift; #define __per_cpu_offset(__cpu) \ (__per_cpu_base + ((unsigned long)(__cpu) << __per_cpu_shift)) +#define per_cpu_offset(x) (__per_cpu_offset(x)) /* Separate out the type, so (int[3], foo) works. */ #define DEFINE_PER_CPU(type, name) \ diff --git a/include/asm-x86_64/irqflags.h b/include/asm-x86_64/irqflags.h new file mode 100644 index 00000000000..cce6937e87c --- /dev/null +++ b/include/asm-x86_64/irqflags.h @@ -0,0 +1,141 @@ +/* + * include/asm-x86_64/irqflags.h + * + * IRQ flags handling + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * raw_local_irq_*() functions from the lowlevel headers. + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +#ifndef __ASSEMBLY__ +/* + * Interrupt control: + */ + +static inline unsigned long __raw_local_save_flags(void) +{ + unsigned long flags; + + __asm__ __volatile__( + "# __raw_save_flags\n\t" + "pushfq ; popq %q0" + : "=g" (flags) + : /* no input */ + : "memory" + ); + + return flags; +} + +#define raw_local_save_flags(flags) \ + do { (flags) = __raw_local_save_flags(); } while (0) + +static inline void raw_local_irq_restore(unsigned long flags) +{ + __asm__ __volatile__( + "pushq %0 ; popfq" + : /* no output */ + :"g" (flags) + :"memory", "cc" + ); +} + +#ifdef CONFIG_X86_VSMP + +/* + * Interrupt control for the VSMP architecture: + */ + +static inline void raw_local_irq_disable(void) +{ + unsigned long flags = __raw_local_save_flags(); + + raw_local_irq_restore((flags & ~(1 << 9)) | (1 << 18)); +} + +static inline void raw_local_irq_enable(void) +{ + unsigned long flags = __raw_local_save_flags(); + + raw_local_irq_restore((flags | (1 << 9)) & ~(1 << 18)); +} + +static inline int raw_irqs_disabled_flags(unsigned long flags) +{ + return !(flags & (1<<9)) || (flags & (1 << 18)); +} + +#else /* CONFIG_X86_VSMP */ + +static inline void raw_local_irq_disable(void) +{ + __asm__ __volatile__("cli" : : : "memory"); +} + +static inline void raw_local_irq_enable(void) +{ + __asm__ __volatile__("sti" : : : "memory"); +} + +static inline int raw_irqs_disabled_flags(unsigned long flags) +{ + return !(flags & (1 << 9)); +} + +#endif + +/* + * For spinlocks, etc.: + */ + +static inline unsigned long __raw_local_irq_save(void) +{ + unsigned long flags = __raw_local_save_flags(); + + raw_local_irq_disable(); + + return flags; +} + +#define raw_local_irq_save(flags) \ + do { (flags) = __raw_local_irq_save(); } while (0) + +static inline int raw_irqs_disabled(void) +{ + unsigned long flags = __raw_local_save_flags(); + + return raw_irqs_disabled_flags(flags); +} + +/* + * Used in the idle loop; sti takes one instruction cycle + * to complete: + */ +static inline void raw_safe_halt(void) +{ + __asm__ __volatile__("sti; hlt" : : : "memory"); +} + +/* + * Used when interrupts are already enabled or to + * shutdown the processor: + */ +static inline void halt(void) +{ + __asm__ __volatile__("hlt": : :"memory"); +} + +#else /* __ASSEMBLY__: */ +# ifdef CONFIG_TRACE_IRQFLAGS +# define TRACE_IRQS_ON call trace_hardirqs_on_thunk +# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk +# else +# define TRACE_IRQS_ON +# define TRACE_IRQS_OFF +# endif +#endif + +#endif diff --git a/include/asm-x86_64/kdebug.h b/include/asm-x86_64/kdebug.h index cd52c7f33bc..2b0c088e295 100644 --- a/include/asm-x86_64/kdebug.h +++ b/include/asm-x86_64/kdebug.h @@ -49,7 +49,7 @@ static inline int notify_die(enum die_val val, const char *str, return atomic_notifier_call_chain(&die_chain, val, &args); } -extern int printk_address(unsigned long address); +extern void printk_address(unsigned long address); extern void die(const char *,struct pt_regs *,long); extern void __die(const char *,struct pt_regs *,long); extern void show_registers(struct pt_regs *regs); diff --git a/include/asm-x86_64/percpu.h b/include/asm-x86_64/percpu.h index 549eb929b2c..08dd9f9dda8 100644 --- a/include/asm-x86_64/percpu.h +++ b/include/asm-x86_64/percpu.h @@ -14,6 +14,8 @@ #define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset) #define __my_cpu_offset() read_pda(data_offset) +#define per_cpu_offset(x) (__per_cpu_offset(x)) + /* Separate out the type, so (int[3], foo) works. */ #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h index 68e559f3631..f67f2873a92 100644 --- a/include/asm-x86_64/system.h +++ b/include/asm-x86_64/system.h @@ -244,43 +244,7 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, #define warn_if_not_ulong(x) do { unsigned long foo; (void) (&(x) == &foo); } while (0) -/* interrupt control.. */ -#define local_save_flags(x) do { warn_if_not_ulong(x); __asm__ __volatile__("# save_flags \n\t pushfq ; popq %q0":"=g" (x): /* no input */ :"memory"); } while (0) -#define local_irq_restore(x) __asm__ __volatile__("# restore_flags \n\t pushq %0 ; popfq": /* no output */ :"g" (x):"memory", "cc") - -#ifdef CONFIG_X86_VSMP -/* Interrupt control for VSMP architecture */ -#define local_irq_disable() do { unsigned long flags; local_save_flags(flags); local_irq_restore((flags & ~(1 << 9)) | (1 << 18)); } while (0) -#define local_irq_enable() do { unsigned long flags; local_save_flags(flags); local_irq_restore((flags | (1 << 9)) & ~(1 << 18)); } while (0) - -#define irqs_disabled() \ -({ \ - unsigned long flags; \ - local_save_flags(flags); \ - (flags & (1<<18)) || !(flags & (1<<9)); \ -}) - -/* For spinlocks etc */ -#define local_irq_save(x) do { local_save_flags(x); local_irq_restore((x & ~(1 << 9)) | (1 << 18)); } while (0) -#else /* CONFIG_X86_VSMP */ -#define local_irq_disable() __asm__ __volatile__("cli": : :"memory") -#define local_irq_enable() __asm__ __volatile__("sti": : :"memory") - -#define irqs_disabled() \ -({ \ - unsigned long flags; \ - local_save_flags(flags); \ - !(flags & (1<<9)); \ -}) - -/* For spinlocks etc */ -#define local_irq_save(x) do { warn_if_not_ulong(x); __asm__ __volatile__("# local_irq_save \n\t pushfq ; popq %0 ; cli":"=g" (x): /* no input */ :"memory"); } while (0) -#endif - -/* used in the idle loop; sti takes one instruction cycle to complete */ -#define safe_halt() __asm__ __volatile__("sti; hlt": : :"memory") -/* used when interrupts are already enabled or to shutdown the processor */ -#define halt() __asm__ __volatile__("hlt": : :"memory") +#include <linux/irqflags.h> void cpu_idle_wait(void); diff --git a/include/asm-xtensa/rwsem.h b/include/asm-xtensa/rwsem.h index abcd86dc5ab..0aad3a58755 100644 --- a/include/asm-xtensa/rwsem.h +++ b/include/asm-xtensa/rwsem.h @@ -31,24 +31,11 @@ struct rw_semaphore { #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif }; -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - #define __RWSEM_INITIALIZER(name) \ { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } + LIST_HEAD_INIT((name).wait_list) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) @@ -63,9 +50,6 @@ static inline void init_rwsem(struct rw_semaphore *sem) sem->count = RWSEM_UNLOCKED_VALUE; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } /* diff --git a/include/linux/completion.h b/include/linux/completion.h index 90663ad217f..251c41e3ddd 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -21,6 +21,18 @@ struct completion { #define DECLARE_COMPLETION(work) \ struct completion work = COMPLETION_INITIALIZER(work) +/* + * Lockdep needs to run a non-constant initializer for on-stack + * completions - so we use the _ONSTACK() variant for those that + * are on the kernel stack: + */ +#ifdef CONFIG_LOCKDEP +# define DECLARE_COMPLETION_ONSTACK(work) \ + struct completion work = ({ init_completion(&work); work; }) +#else +# define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work) +#endif + static inline void init_completion(struct completion *x) { x->done = 0; diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 0dd1610a94a..471781ffeab 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -114,6 +114,18 @@ struct dentry { unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */ }; +/* + * dentry->d_lock spinlock nesting subclasses: + * + * 0: normal + * 1: nested + */ +enum dentry_d_lock_class +{ + DENTRY_D_LOCK_NORMAL, /* implicitly used by plain spin_lock() APIs. */ + DENTRY_D_LOCK_NESTED +}; + struct dentry_operations { int (*d_revalidate)(struct dentry *, struct nameidata *); int (*d_hash) (struct dentry *, struct qstr *); diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h new file mode 100644 index 00000000000..6a7047851e4 --- /dev/null +++ b/include/linux/debug_locks.h @@ -0,0 +1,69 @@ +#ifndef __LINUX_DEBUG_LOCKING_H +#define __LINUX_DEBUG_LOCKING_H + +extern int debug_locks; +extern int debug_locks_silent; + +/* + * Generic 'turn off all lock debugging' function: + */ +extern int debug_locks_off(void); + +/* + * In the debug case we carry the caller's instruction pointer into + * other functions, but we dont want the function argument overhead + * in the nondebug case - hence these macros: + */ +#define _RET_IP_ (unsigned long)__builtin_return_address(0) +#define _THIS_IP_ ({ __label__ __here; __here: (unsigned long)&&__here; }) + +#define DEBUG_LOCKS_WARN_ON(c) \ +({ \ + int __ret = 0; \ + \ + if (unlikely(c)) { \ + if (debug_locks_off()) \ + WARN_ON(1); \ + __ret = 1; \ + } \ + __ret; \ +}) + +#ifdef CONFIG_SMP +# define SMP_DEBUG_LOCKS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) +#else +# define SMP_DEBUG_LOCKS_WARN_ON(c) do { } while (0) +#endif + +#ifdef CONFIG_DEBUG_LOCKING_API_SELFTESTS + extern void locking_selftest(void); +#else +# define locking_selftest() do { } while (0) +#endif + +#ifdef CONFIG_LOCKDEP +extern void debug_show_all_locks(void); +extern void debug_show_held_locks(struct task_struct *task); +extern void debug_check_no_locks_freed(const void *from, unsigned long len); +extern void debug_check_no_locks_held(struct task_struct *task); +#else +static inline void debug_show_all_locks(void) +{ +} + +static inline void debug_show_held_locks(struct task_struct *task) +{ +} + +static inline void +debug_check_no_locks_freed(const void *from, unsigned long len) +{ +} + +static inline void +debug_check_no_locks_held(struct task_struct *task) +{ +} +#endif + +#endif diff --git a/include/linux/fs.h b/include/linux/fs.h index e04a5cfe874..134b3206824 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -436,6 +436,21 @@ struct block_device { }; /* + * bdev->bd_mutex nesting subclasses for the lock validator: + * + * 0: normal + * 1: 'whole' + * 2: 'partition' + */ +enum bdev_bd_mutex_lock_class +{ + BD_MUTEX_NORMAL, + BD_MUTEX_WHOLE, + BD_MUTEX_PARTITION +}; + + +/* * Radix-tree tags, for tagging dirty and writeback pages within the pagecache * radix trees */ @@ -543,6 +558,25 @@ struct inode { }; /* + * inode->i_mutex nesting subclasses for the lock validator: + * + * 0: the object of the current VFS operation + * 1: parent + * 2: child/target + * 3: quota file + * + * The locking order between these classes is + * parent -> child -> normal -> quota + */ +enum inode_i_mutex_lock_class +{ + I_MUTEX_NORMAL, + I_MUTEX_PARENT, + I_MUTEX_CHILD, + I_MUTEX_QUOTA +}; + +/* * NOTE: in a 32bit arch with a preemptable kernel and * an UP compile the i_size_read/write must be atomic * with respect to the local cpu (unlike with preempt disabled), @@ -1276,6 +1310,8 @@ struct file_system_type { struct module *owner; struct file_system_type * next; struct list_head fs_supers; + struct lock_class_key s_lock_key; + struct lock_class_key s_umount_key; }; extern int get_sb_bdev(struct file_system_type *fs_type, @@ -1404,6 +1440,7 @@ extern void bd_set_size(struct block_device *, loff_t size); extern void bd_forget(struct inode *inode); extern void bdput(struct block_device *); extern struct block_device *open_by_devnum(dev_t, unsigned); +extern struct block_device *open_partition_by_devnum(dev_t, unsigned); extern const struct file_operations def_blk_fops; extern const struct address_space_operations def_blk_aops; extern const struct file_operations def_chr_fops; @@ -1414,6 +1451,7 @@ extern int blkdev_ioctl(struct inode *, struct file *, unsigned, unsigned long); extern long compat_blkdev_ioctl(struct file *, unsigned, unsigned long); extern int blkdev_get(struct block_device *, mode_t, unsigned); extern int blkdev_put(struct block_device *); +extern int blkdev_put_partition(struct block_device *); extern int bd_claim(struct block_device *, void *); extern void bd_release(struct block_device *); #ifdef CONFIG_SYSFS diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 114ae583cca..50d8b5744cf 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -3,6 +3,7 @@ #include <linux/preempt.h> #include <linux/smp_lock.h> +#include <linux/lockdep.h> #include <asm/hardirq.h> #include <asm/system.h> @@ -86,9 +87,6 @@ extern void synchronize_irq(unsigned int irq); # define synchronize_irq(irq) barrier() #endif -#define nmi_enter() irq_enter() -#define nmi_exit() sub_preempt_count(HARDIRQ_OFFSET) - struct task_struct; #ifndef CONFIG_VIRT_CPU_ACCOUNTING @@ -97,12 +95,35 @@ static inline void account_system_vtime(struct task_struct *tsk) } #endif +/* + * It is safe to do non-atomic ops on ->hardirq_context, + * because NMI handlers may not preempt and the ops are + * always balanced, so the interrupted value of ->hardirq_context + * will always be restored. + */ #define irq_enter() \ do { \ account_system_vtime(current); \ add_preempt_count(HARDIRQ_OFFSET); \ + trace_hardirq_enter(); \ + } while (0) + +/* + * Exit irq context without processing softirqs: + */ +#define __irq_exit() \ + do { \ + trace_hardirq_exit(); \ + account_system_vtime(current); \ + sub_preempt_count(HARDIRQ_OFFSET); \ } while (0) +/* + * Exit irq context and process softirqs if needed: + */ extern void irq_exit(void); +#define nmi_enter() do { lockdep_off(); irq_enter(); } while (0) +#define nmi_exit() do { __irq_exit(); lockdep_on(); } while (0) + #endif /* LINUX_HARDIRQ_H */ diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 07d7305f131..e4bccbcc275 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -91,6 +91,7 @@ struct hrtimer_base { ktime_t (*get_softirq_time)(void); struct hrtimer *curr_timer; ktime_t softirq_time; + struct lock_class_key lock_key; }; /* diff --git a/include/linux/ide.h b/include/linux/ide.h index 285316c836b..dc7abef1096 100644 --- a/include/linux/ide.h +++ b/include/linux/ide.h @@ -1359,7 +1359,7 @@ extern struct semaphore ide_cfg_sem; * ide_drive_t->hwif: constant, no locking */ -#define local_irq_set(flags) do { local_save_flags((flags)); local_irq_enable(); } while (0) +#define local_irq_set(flags) do { local_save_flags((flags)); local_irq_enable_in_hardirq(); } while (0) extern struct bus_type ide_bus_type; diff --git a/include/linux/idr.h b/include/linux/idr.h index f559a719dbe..826803449db 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -66,7 +66,7 @@ struct idr { .id_free = NULL, \ .layers = 0, \ .id_free_cnt = 0, \ - .lock = SPIN_LOCK_UNLOCKED, \ + .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ } #define DEFINE_IDR(name) struct idr name = IDR_INIT(name) diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 3a256957fb5..60aac2cea0c 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -3,6 +3,8 @@ #include <linux/file.h> #include <linux/rcupdate.h> +#include <linux/irqflags.h> +#include <linux/lockdep.h> #define INIT_FDTABLE \ { \ @@ -21,7 +23,7 @@ .count = ATOMIC_INIT(1), \ .fdt = &init_files.fdtab, \ .fdtab = INIT_FDTABLE, \ - .file_lock = SPIN_LOCK_UNLOCKED, \ + .file_lock = __SPIN_LOCK_UNLOCKED(init_task.file_lock), \ .next_fd = 0, \ .close_on_exec_init = { { 0, } }, \ .open_fds_init = { { 0, } }, \ @@ -36,7 +38,7 @@ .user_id = 0, \ .next = NULL, \ .wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.wait), \ - .ctx_lock = SPIN_LOCK_UNLOCKED, \ + .ctx_lock = __SPIN_LOCK_UNLOCKED(name.ctx_lock), \ .reqs_active = 0U, \ .max_reqs = ~0U, \ } @@ -48,7 +50,7 @@ .mm_users = ATOMIC_INIT(2), \ .mm_count = ATOMIC_INIT(1), \ .mmap_sem = __RWSEM_INITIALIZER(name.mmap_sem), \ - .page_table_lock = SPIN_LOCK_UNLOCKED, \ + .page_table_lock = __SPIN_LOCK_UNLOCKED(name.page_table_lock), \ .mmlist = LIST_HEAD_INIT(name.mmlist), \ .cpu_vm_mask = CPU_MASK_ALL, \ } @@ -69,7 +71,7 @@ #define INIT_SIGHAND(sighand) { \ .count = ATOMIC_INIT(1), \ .action = { { { .sa_handler = NULL, } }, }, \ - .siglock = SPIN_LOCK_UNLOCKED, \ + .siglock = __SPIN_LOCK_UNLOCKED(sighand.siglock), \ } extern struct group_info init_groups; @@ -119,12 +121,13 @@ extern struct group_info init_groups; .list = LIST_HEAD_INIT(tsk.pending.list), \ .signal = {{0}}}, \ .blocked = {{0}}, \ - .alloc_lock = SPIN_LOCK_UNLOCKED, \ + .alloc_lock = __SPIN_LOCK_UNLOCKED(tsk.alloc_lock), \ .journal_info = NULL, \ .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ .fs_excl = ATOMIC_INIT(0), \ .pi_lock = SPIN_LOCK_UNLOCKED, \ - INIT_RT_MUTEXES(tsk) \ + INIT_TRACE_IRQFLAGS \ + INIT_LOCKDEP \ } diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index cf682a73a6f..d5afee95fd4 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -10,6 +10,7 @@ #include <linux/irqreturn.h> #include <linux/hardirq.h> #include <linux/sched.h> +#include <linux/irqflags.h> #include <asm/atomic.h> #include <asm/ptrace.h> #include <asm/system.h> @@ -45,6 +46,7 @@ #define IRQF_SHARED 0x00000080 #define IRQF_PROBE_SHARED 0x00000100 #define IRQF_TIMER 0x00000200 +#define IRQF_PERCPU 0x00000400 /* * Migration helpers. Scheduled for removal in 1/2007 @@ -54,6 +56,7 @@ #define SA_SAMPLE_RANDOM IRQF_SAMPLE_RANDOM #define SA_SHIRQ IRQF_SHARED #define SA_PROBEIRQ IRQF_PROBE_SHARED +#define SA_PERCPU IRQF_PERCPU #define SA_TRIGGER_LOW IRQF_TRIGGER_LOW #define SA_TRIGGER_HIGH IRQF_TRIGGER_HIGH @@ -78,12 +81,64 @@ extern int request_irq(unsigned int, unsigned long, const char *, void *); extern void free_irq(unsigned int, void *); +/* + * On lockdep we dont want to enable hardirqs in hardirq + * context. Use local_irq_enable_in_hardirq() to annotate + * kernel code that has to do this nevertheless (pretty much + * the only valid case is for old/broken hardware that is + * insanely slow). + * + * NOTE: in theory this might break fragile code that relies + * on hardirq delivery - in practice we dont seem to have such + * places left. So the only effect should be slightly increased + * irqs-off latencies. + */ +#ifdef CONFIG_LOCKDEP +# define local_irq_enable_in_hardirq() do { } while (0) +#else +# define local_irq_enable_in_hardirq() local_irq_enable() +#endif #ifdef CONFIG_GENERIC_HARDIRQS extern void disable_irq_nosync(unsigned int irq); extern void disable_irq(unsigned int irq); extern void enable_irq(unsigned int irq); +/* + * Special lockdep variants of irq disabling/enabling. + * These should be used for locking constructs that + * know that a particular irq context which is disabled, + * and which is the only irq-context user of a lock, + * that it's safe to take the lock in the irq-disabled + * section without disabling hardirqs. + * + * On !CONFIG_LOCKDEP they are equivalent to the normal + * irq disable/enable methods. + */ +static inline void disable_irq_nosync_lockdep(unsigned int irq) +{ + disable_irq_nosync(irq); +#ifdef CONFIG_LOCKDEP + local_irq_disable(); +#endif +} + +static inline void disable_irq_lockdep(unsigned int irq) +{ + disable_irq(irq); +#ifdef CONFIG_LOCKDEP + local_irq_disable(); +#endif +} + +static inline void enable_irq_lockdep(unsigned int irq) +{ +#ifdef CONFIG_LOCKDEP + local_irq_enable(); +#endif + enable_irq(irq); +} + /* IRQ wakeup (PM) control: */ extern int set_irq_wake(unsigned int irq, unsigned int on); @@ -97,7 +152,19 @@ static inline int disable_irq_wake(unsigned int irq) return set_irq_wake(irq, 0); } -#endif +#else /* !CONFIG_GENERIC_HARDIRQS */ +/* + * NOTE: non-genirq architectures, if they want to support the lock + * validator need to define the methods below in their asm/irq.h + * files, under an #ifdef CONFIG_LOCKDEP section. + */ +# ifndef CONFIG_LOCKDEP +# define disable_irq_nosync_lockdep(irq) disable_irq_nosync(irq) +# define disable_irq_lockdep(irq) disable_irq(irq) +# define enable_irq_lockdep(irq) enable_irq(irq) +# endif + +#endif /* CONFIG_GENERIC_HARDIRQS */ #ifndef __ARCH_SET_SOFTIRQ_PENDING #define set_softirq_pending(x) (local_softirq_pending() = (x)) @@ -133,13 +200,11 @@ static inline void __deprecated save_and_cli(unsigned long *x) #define save_and_cli(x) save_and_cli(&x) #endif /* CONFIG_SMP */ -/* SoftIRQ primitives. */ -#define local_bh_disable() \ - do { add_preempt_count(SOFTIRQ_OFFSET); barrier(); } while (0) -#define __local_bh_enable() \ - do { barrier(); sub_preempt_count(SOFTIRQ_OFFSET); } while (0) - +extern void local_bh_disable(void); +extern void __local_bh_enable(void); +extern void _local_bh_enable(void); extern void local_bh_enable(void); +extern void local_bh_enable_ip(unsigned long ip); /* PLEASE, avoid to allocate new softirqs, if you need not _really_ high frequency threaded job scheduling. For almost all the purposes diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 87a9fc039b4..5612dfeeae5 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -55,6 +55,7 @@ struct resource_list { #define IORESOURCE_IRQ_LOWEDGE (1<<1) #define IORESOURCE_IRQ_HIGHLEVEL (1<<2) #define IORESOURCE_IRQ_LOWLEVEL (1<<3) +#define IORESOURCE_IRQ_SHAREABLE (1<<4) /* ISA PnP DMA specific bits (IORESOURCE_BITS) */ #define IORESOURCE_DMA_TYPE_MASK (3<<0) diff --git a/include/linux/irq.h b/include/linux/irq.h index 95d7aa7954d..b48eae32dc6 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -182,6 +182,10 @@ extern int setup_irq(unsigned int irq, struct irqaction *new); #ifdef CONFIG_GENERIC_HARDIRQS +#ifndef handle_dynamic_tick +# define handle_dynamic_tick(a) do { } while (0) +#endif + #ifdef CONFIG_SMP static inline void set_native_irq_info(int irq, cpumask_t mask) { diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h new file mode 100644 index 00000000000..412e025bc5c --- /dev/null +++ b/include/linux/irqflags.h @@ -0,0 +1,96 @@ +/* + * include/linux/irqflags.h + * + * IRQ flags tracing: follow the state of the hardirq and softirq flags and + * provide callbacks for transitions between ON and OFF states. + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * raw_local_irq_*() macros from the lowlevel headers. + */ +#ifndef _LINUX_TRACE_IRQFLAGS_H +#define _LINUX_TRACE_IRQFLAGS_H + +#ifdef CONFIG_TRACE_IRQFLAGS + extern void trace_hardirqs_on(void); + extern void trace_hardirqs_off(void); + extern void trace_softirqs_on(unsigned long ip); + extern void trace_softirqs_off(unsigned long ip); +# define trace_hardirq_context(p) ((p)->hardirq_context) +# define trace_softirq_context(p) ((p)->softirq_context) +# define trace_hardirqs_enabled(p) ((p)->hardirqs_enabled) +# define trace_softirqs_enabled(p) ((p)->softirqs_enabled) +# define trace_hardirq_enter() do { current->hardirq_context++; } while (0) +# define trace_hardirq_exit() do { current->hardirq_context--; } while (0) +# define trace_softirq_enter() do { current->softirq_context++; } while (0) +# define trace_softirq_exit() do { current->softirq_context--; } while (0) +# define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1, +#else +# define trace_hardirqs_on() do { } while (0) +# define trace_hardirqs_off() do { } while (0) +# define trace_softirqs_on(ip) do { } while (0) +# define trace_softirqs_off(ip) do { } while (0) +# define trace_hardirq_context(p) 0 +# define trace_softirq_context(p) 0 +# define trace_hardirqs_enabled(p) 0 +# define trace_softirqs_enabled(p) 0 +# define trace_hardirq_enter() do { } while (0) +# define trace_hardirq_exit() do { } while (0) +# define trace_softirq_enter() do { } while (0) +# define trace_softirq_exit() do { } while (0) +# define INIT_TRACE_IRQFLAGS +#endif + +#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT + +#include <asm/irqflags.h> + +#define local_irq_enable() \ + do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0) +#define local_irq_disable() \ + do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0) +#define local_irq_save(flags) \ + do { raw_local_irq_save(flags); trace_hardirqs_off(); } while (0) + +#define local_irq_restore(flags) \ + do { \ + if (raw_irqs_disabled_flags(flags)) { \ + raw_local_irq_restore(flags); \ + trace_hardirqs_off(); \ + } else { \ + trace_hardirqs_on(); \ + raw_local_irq_restore(flags); \ + } \ + } while (0) +#else /* !CONFIG_TRACE_IRQFLAGS_SUPPORT */ +/* + * The local_irq_*() APIs are equal to the raw_local_irq*() + * if !TRACE_IRQFLAGS. + */ +# define raw_local_irq_disable() local_irq_disable() +# define raw_local_irq_enable() local_irq_enable() +# define raw_local_irq_save(flags) local_irq_save(flags) +# define raw_local_irq_restore(flags) local_irq_restore(flags) +#endif /* CONFIG_TRACE_IRQFLAGS_SUPPORT */ + +#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT +#define safe_halt() \ + do { \ + trace_hardirqs_on(); \ + raw_safe_halt(); \ + } while (0) + +#define local_save_flags(flags) raw_local_save_flags(flags) + +#define irqs_disabled() \ +({ \ + unsigned long flags; \ + \ + raw_local_save_flags(flags); \ + raw_irqs_disabled_flags(flags); \ +}) + +#define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) +#endif /* CONFIG_X86 */ + +#endif diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index 54e2549f96b..849043ce4ed 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -57,10 +57,25 @@ do { \ #define print_fn_descriptor_symbol(fmt, addr) print_symbol(fmt, addr) #endif -#define print_symbol(fmt, addr) \ -do { \ - __check_printsym_format(fmt, ""); \ - __print_symbol(fmt, addr); \ +static inline void print_symbol(const char *fmt, unsigned long addr) +{ + __check_printsym_format(fmt, ""); + __print_symbol(fmt, (unsigned long) + __builtin_extract_return_addr((void *)addr)); +} + +#ifndef CONFIG_64BIT +#define print_ip_sym(ip) \ +do { \ + printk("[<%08lx>]", ip); \ + print_symbol(" %s\n", ip); \ } while(0) +#else +#define print_ip_sym(ip) \ +do { \ + printk("[<%016lx>]", ip); \ + print_symbol(" %s\n", ip); \ +} while(0) +#endif #endif /*_LINUX_KALLSYMS_H*/ diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h new file mode 100644 index 00000000000..316e0fb8d7b --- /dev/null +++ b/include/linux/lockdep.h @@ -0,0 +1,353 @@ +/* + * Runtime locking correctness validator + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + * + * see Documentation/lockdep-design.txt for more details. + */ +#ifndef __LINUX_LOCKDEP_H +#define __LINUX_LOCKDEP_H + +#include <linux/linkage.h> +#include <linux/list.h> +#include <linux/debug_locks.h> +#include <linux/stacktrace.h> + +#ifdef CONFIG_LOCKDEP + +/* + * Lock-class usage-state bits: + */ +enum lock_usage_bit +{ + LOCK_USED = 0, + LOCK_USED_IN_HARDIRQ, + LOCK_USED_IN_SOFTIRQ, + LOCK_ENABLED_SOFTIRQS, + LOCK_ENABLED_HARDIRQS, + LOCK_USED_IN_HARDIRQ_READ, + LOCK_USED_IN_SOFTIRQ_READ, + LOCK_ENABLED_SOFTIRQS_READ, + LOCK_ENABLED_HARDIRQS_READ, + LOCK_USAGE_STATES +}; + +/* + * Usage-state bitmasks: + */ +#define LOCKF_USED (1 << LOCK_USED) +#define LOCKF_USED_IN_HARDIRQ (1 << LOCK_USED_IN_HARDIRQ) +#define LOCKF_USED_IN_SOFTIRQ (1 << LOCK_USED_IN_SOFTIRQ) +#define LOCKF_ENABLED_HARDIRQS (1 << LOCK_ENABLED_HARDIRQS) +#define LOCKF_ENABLED_SOFTIRQS (1 << LOCK_ENABLED_SOFTIRQS) + +#define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS | LOCKF_ENABLED_SOFTIRQS) +#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ) + +#define LOCKF_USED_IN_HARDIRQ_READ (1 << LOCK_USED_IN_HARDIRQ_READ) +#define LOCKF_USED_IN_SOFTIRQ_READ (1 << LOCK_USED_IN_SOFTIRQ_READ) +#define LOCKF_ENABLED_HARDIRQS_READ (1 << LOCK_ENABLED_HARDIRQS_READ) +#define LOCKF_ENABLED_SOFTIRQS_READ (1 << LOCK_ENABLED_SOFTIRQS_READ) + +#define LOCKF_ENABLED_IRQS_READ \ + (LOCKF_ENABLED_HARDIRQS_READ | LOCKF_ENABLED_SOFTIRQS_READ) +#define LOCKF_USED_IN_IRQ_READ \ + (LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ) + +#define MAX_LOCKDEP_SUBCLASSES 8UL + +/* + * Lock-classes are keyed via unique addresses, by embedding the + * lockclass-key into the kernel (or module) .data section. (For + * static locks we use the lock address itself as the key.) + */ +struct lockdep_subclass_key { + char __one_byte; +} __attribute__ ((__packed__)); + +struct lock_class_key { + struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES]; +}; + +/* + * The lock-class itself: + */ +struct lock_class { + /* + * class-hash: + */ + struct list_head hash_entry; + + /* + * global list of all lock-classes: + */ + struct list_head lock_entry; + + struct lockdep_subclass_key *key; + unsigned int subclass; + + /* + * IRQ/softirq usage tracking bits: + */ + unsigned long usage_mask; + struct stack_trace usage_traces[LOCK_USAGE_STATES]; + + /* + * These fields represent a directed graph of lock dependencies, + * to every node we attach a list of "forward" and a list of + * "backward" graph nodes. + */ + struct list_head locks_after, locks_before; + + /* + * Generation counter, when doing certain classes of graph walking, + * to ensure that we check one node only once: + */ + unsigned int version; + + /* + * Statistics counter: + */ + unsigned long ops; + + const char *name; + int name_version; +}; + +/* + * Map the lock object (the lock instance) to the lock-class object. + * This is embedded into specific lock instances: + */ +struct lockdep_map { + struct lock_class_key *key; + struct lock_class *class[MAX_LOCKDEP_SUBCLASSES]; + const char *name; +}; + +/* + * Every lock has a list of other locks that were taken after it. + * We only grow the list, never remove from it: + */ +struct lock_list { + struct list_head entry; + struct lock_class *class; + struct stack_trace trace; +}; + +/* + * We record lock dependency chains, so that we can cache them: + */ +struct lock_chain { + struct list_head entry; + u64 chain_key; +}; + +struct held_lock { + /* + * One-way hash of the dependency chain up to this point. We + * hash the hashes step by step as the dependency chain grows. + * + * We use it for dependency-caching and we skip detection + * passes and dependency-updates if there is a cache-hit, so + * it is absolutely critical for 100% coverage of the validator + * to have a unique key value for every unique dependency path + * that can occur in the system, to make a unique hash value + * as likely as possible - hence the 64-bit width. + * + * The task struct holds the current hash value (initialized + * with zero), here we store the previous hash value: + */ + u64 prev_chain_key; + struct lock_class *class; + unsigned long acquire_ip; + struct lockdep_map *instance; + + /* + * The lock-stack is unified in that the lock chains of interrupt + * contexts nest ontop of process context chains, but we 'separate' + * the hashes by starting with 0 if we cross into an interrupt + * context, and we also keep do not add cross-context lock + * dependencies - the lock usage graph walking covers that area + * anyway, and we'd just unnecessarily increase the number of + * dependencies otherwise. [Note: hardirq and softirq contexts + * are separated from each other too.] + * + * The following field is used to detect when we cross into an + * interrupt context: + */ + int irq_context; + int trylock; + int read; + int check; + int hardirqs_off; +}; + +/* + * Initialization, self-test and debugging-output methods: + */ +extern void lockdep_init(void); +extern void lockdep_info(void); +extern void lockdep_reset(void); +extern void lockdep_reset_lock(struct lockdep_map *lock); +extern void lockdep_free_key_range(void *start, unsigned long size); + +extern void lockdep_off(void); +extern void lockdep_on(void); +extern int lockdep_internal(void); + +/* + * These methods are used by specific locking variants (spinlocks, + * rwlocks, mutexes and rwsems) to pass init/acquire/release events + * to lockdep: + */ + +extern void lockdep_init_map(struct lockdep_map *lock, const char *name, + struct lock_class_key *key); + +/* + * Reinitialize a lock key - for cases where there is special locking or + * special initialization of locks so that the validator gets the scope + * of dependencies wrong: they are either too broad (they need a class-split) + * or they are too narrow (they suffer from a false class-split): + */ +#define lockdep_set_class(lock, key) \ + lockdep_init_map(&(lock)->dep_map, #key, key) +#define lockdep_set_class_and_name(lock, key, name) \ + lockdep_init_map(&(lock)->dep_map, name, key) + +/* + * Acquire a lock. + * + * Values for "read": + * + * 0: exclusive (write) acquire + * 1: read-acquire (no recursion allowed) + * 2: read-acquire with same-instance recursion allowed + * + * Values for check: + * + * 0: disabled + * 1: simple checks (freeing, held-at-exit-time, etc.) + * 2: full validation + */ +extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass, + int trylock, int read, int check, unsigned long ip); + +extern void lock_release(struct lockdep_map *lock, int nested, + unsigned long ip); + +# define INIT_LOCKDEP .lockdep_recursion = 0, + +#else /* !LOCKDEP */ + +static inline void lockdep_off(void) +{ +} + +static inline void lockdep_on(void) +{ +} + +static inline int lockdep_internal(void) +{ + return 0; +} + +# define lock_acquire(l, s, t, r, c, i) do { } while (0) +# define lock_release(l, n, i) do { } while (0) +# define lockdep_init() do { } while (0) +# define lockdep_info() do { } while (0) +# define lockdep_init_map(lock, name, key) do { (void)(key); } while (0) +# define lockdep_set_class(lock, key) do { (void)(key); } while (0) +# define lockdep_set_class_and_name(lock, key, name) \ + do { (void)(key); } while (0) +# define INIT_LOCKDEP +# define lockdep_reset() do { debug_locks = 1; } while (0) +# define lockdep_free_key_range(start, size) do { } while (0) +/* + * The class key takes no space if lockdep is disabled: + */ +struct lock_class_key { }; +#endif /* !LOCKDEP */ + +#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS) +extern void early_init_irq_lock_class(void); +#else +# define early_init_irq_lock_class() do { } while (0) +#endif + +#ifdef CONFIG_TRACE_IRQFLAGS +extern void early_boot_irqs_off(void); +extern void early_boot_irqs_on(void); +#else +# define early_boot_irqs_off() do { } while (0) +# define early_boot_irqs_on() do { } while (0) +#endif + +/* + * For trivial one-depth nesting of a lock-class, the following + * global define can be used. (Subsystems with multiple levels + * of nesting should define their own lock-nesting subclasses.) + */ +#define SINGLE_DEPTH_NESTING 1 + +/* + * Map the dependency ops to NOP or to real lockdep ops, depending + * on the per lock-class debug mode: + */ + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# ifdef CONFIG_PROVE_LOCKING +# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) +# else +# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) +# endif +# define spin_release(l, n, i) lock_release(l, n, i) +#else +# define spin_acquire(l, s, t, i) do { } while (0) +# define spin_release(l, n, i) do { } while (0) +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# ifdef CONFIG_PROVE_LOCKING +# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) +# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, i) +# else +# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) +# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, i) +# endif +# define rwlock_release(l, n, i) lock_release(l, n, i) +#else +# define rwlock_acquire(l, s, t, i) do { } while (0) +# define rwlock_acquire_read(l, s, t, i) do { } while (0) +# define rwlock_release(l, n, i) do { } while (0) +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# ifdef CONFIG_PROVE_LOCKING +# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) +# else +# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) +# endif +# define mutex_release(l, n, i) lock_release(l, n, i) +#else +# define mutex_acquire(l, s, t, i) do { } while (0) +# define mutex_release(l, n, i) do { } while (0) +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# ifdef CONFIG_PROVE_LOCKING +# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) +# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, i) +# else +# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) +# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, i) +# endif +# define rwsem_release(l, n, i) lock_release(l, n, i) +#else +# define rwsem_acquire(l, s, t, i) do { } while (0) +# define rwsem_acquire_read(l, s, t, i) do { } while (0) +# define rwsem_release(l, n, i) do { } while (0) +#endif + +#endif /* __LINUX_LOCKDEP_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 75179529e39..990957e0929 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -14,6 +14,7 @@ #include <linux/prio_tree.h> #include <linux/fs.h> #include <linux/mutex.h> +#include <linux/debug_locks.h> struct mempolicy; struct anon_vma; @@ -1034,13 +1035,6 @@ static inline void vm_stat_account(struct mm_struct *mm, } #endif /* CONFIG_PROC_FS */ -static inline void -debug_check_no_locks_freed(const void *from, unsigned long len) -{ - mutex_debug_check_no_locks_freed(from, len); - rt_mutex_debug_check_no_locks_freed(from, len); -} - #ifndef CONFIG_DEBUG_PAGEALLOC static inline void kernel_map_pages(struct page *page, int numpages, int enable) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 27e748eb72b..656b588a9f9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -150,6 +150,10 @@ struct zone { unsigned long lowmem_reserve[MAX_NR_ZONES]; #ifdef CONFIG_NUMA + /* + * zone reclaim becomes active if more unmapped pages exist. + */ + unsigned long min_unmapped_ratio; struct per_cpu_pageset *pageset[NR_CPUS]; #else struct per_cpu_pageset pageset[NR_CPUS]; @@ -414,6 +418,8 @@ int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, struct file *, void __user *, size_t *, loff_t *); int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, struct file *, void __user *, size_t *, loff_t *); +int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int, + struct file *, void __user *, size_t *, loff_t *); #include <linux/topology.h> /* Returns the number of the current Node. */ diff --git a/include/linux/module.h b/include/linux/module.h index 9e9dc7c24d9..d06c74fb8c2 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -358,6 +358,7 @@ static inline int module_is_live(struct module *mod) /* Is this address in a module? (second is with no locks, for oops) */ struct module *module_text_address(unsigned long addr); struct module *__module_text_address(unsigned long addr); +int is_module_address(unsigned long addr); /* Returns module and fills in value, defined and namebuf, or NULL if symnum out of range. */ @@ -496,6 +497,11 @@ static inline struct module *__module_text_address(unsigned long addr) return NULL; } +static inline int is_module_address(unsigned long addr) +{ + return 0; +} + /* Get/put a kernel symbol (calls should be symmetric) */ #define symbol_get(x) ({ extern typeof(x) x __attribute__((weak)); &(x); }) #define symbol_put(x) do { } while(0) diff --git a/include/linux/mutex-debug.h b/include/linux/mutex-debug.h index 8b5769f0046..2537285e106 100644 --- a/include/linux/mutex-debug.h +++ b/include/linux/mutex-debug.h @@ -2,22 +2,22 @@ #define __LINUX_MUTEX_DEBUG_H #include <linux/linkage.h> +#include <linux/lockdep.h> /* * Mutexes - debugging helpers: */ -#define __DEBUG_MUTEX_INITIALIZER(lockname) \ - , .held_list = LIST_HEAD_INIT(lockname.held_list), \ - .name = #lockname , .magic = &lockname +#define __DEBUG_MUTEX_INITIALIZER(lockname) \ + , .magic = &lockname -#define mutex_init(sem) __mutex_init(sem, __FUNCTION__) +#define mutex_init(mutex) \ +do { \ + static struct lock_class_key __key; \ + \ + __mutex_init((mutex), #mutex, &__key); \ +} while (0) extern void FASTCALL(mutex_destroy(struct mutex *lock)); -extern void mutex_debug_show_all_locks(void); -extern void mutex_debug_show_held_locks(struct task_struct *filter); -extern void mutex_debug_check_no_locks_held(struct task_struct *task); -extern void mutex_debug_check_no_locks_freed(const void *from, unsigned long len); - #endif diff --git a/include/linux/mutex.h b/include/linux/mutex.h index f1ac507fa20..27c48daa318 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -13,6 +13,7 @@ #include <linux/list.h> #include <linux/spinlock_types.h> #include <linux/linkage.h> +#include <linux/lockdep.h> #include <asm/atomic.h> @@ -50,11 +51,12 @@ struct mutex { struct list_head wait_list; #ifdef CONFIG_DEBUG_MUTEXES struct thread_info *owner; - struct list_head held_list; - unsigned long acquire_ip; const char *name; void *magic; #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif }; /* @@ -74,24 +76,34 @@ struct mutex_waiter { # include <linux/mutex-debug.h> #else # define __DEBUG_MUTEX_INITIALIZER(lockname) -# define mutex_init(mutex) __mutex_init(mutex, NULL) +# define mutex_init(mutex) \ +do { \ + static struct lock_class_key __key; \ + \ + __mutex_init((mutex), #mutex, &__key); \ +} while (0) # define mutex_destroy(mutex) do { } while (0) -# define mutex_debug_show_all_locks() do { } while (0) -# define mutex_debug_show_held_locks(p) do { } while (0) -# define mutex_debug_check_no_locks_held(task) do { } while (0) -# define mutex_debug_check_no_locks_freed(from, len) do { } while (0) +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ + , .dep_map = { .name = #lockname } +#else +# define __DEP_MAP_MUTEX_INITIALIZER(lockname) #endif #define __MUTEX_INITIALIZER(lockname) \ { .count = ATOMIC_INIT(1) \ , .wait_lock = SPIN_LOCK_UNLOCKED \ , .wait_list = LIST_HEAD_INIT(lockname.wait_list) \ - __DEBUG_MUTEX_INITIALIZER(lockname) } + __DEBUG_MUTEX_INITIALIZER(lockname) \ + __DEP_MAP_MUTEX_INITIALIZER(lockname) } #define DEFINE_MUTEX(mutexname) \ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) -extern void fastcall __mutex_init(struct mutex *lock, const char *name); +extern void __mutex_init(struct mutex *lock, const char *name, + struct lock_class_key *key); /*** * mutex_is_locked - is the mutex locked @@ -110,6 +122,13 @@ static inline int fastcall mutex_is_locked(struct mutex *lock) */ extern void fastcall mutex_lock(struct mutex *lock); extern int fastcall mutex_lock_interruptible(struct mutex *lock); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass); +#else +# define mutex_lock_nested(lock, subclass) mutex_lock(lock) +#endif + /* * NOTE: mutex_trylock() follows the spin_trylock() convention, * not the down_trylock() convention! diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 51dbab9710c..7ff386a6ae8 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -65,7 +65,7 @@ struct raw_notifier_head { } while (0) #define ATOMIC_NOTIFIER_INIT(name) { \ - .lock = SPIN_LOCK_UNLOCKED, \ + .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .head = NULL } #define BLOCKING_NOTIFIER_INIT(name) { \ .rwsem = __RWSEM_INITIALIZER((name).rwsem), \ diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index b093479a531..685081c0134 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -728,6 +728,7 @@ #define PCI_DEVICE_ID_TI_TVP4020 0x3d07 #define PCI_DEVICE_ID_TI_4450 0x8011 #define PCI_DEVICE_ID_TI_XX21_XX11 0x8031 +#define PCI_DEVICE_ID_TI_XX21_XX11_SD 0x8034 #define PCI_DEVICE_ID_TI_X515 0x8036 #define PCI_DEVICE_ID_TI_XX12 0x8039 #define PCI_DEVICE_ID_TI_1130 0xac12 @@ -1442,6 +1443,7 @@ #define PCI_DEVICE_ID_RICOH_RL5C475 0x0475 #define PCI_DEVICE_ID_RICOH_RL5C476 0x0476 #define PCI_DEVICE_ID_RICOH_RL5C478 0x0478 +#define PCI_DEVICE_ID_RICOH_R5C822 0x0822 #define PCI_VENDOR_ID_DLINK 0x1186 #define PCI_DEVICE_ID_DLINK_DGE510T 0x4c00 diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index fa4a3b82ba7..5d41dee82f8 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -29,8 +29,6 @@ struct rt_mutex { struct task_struct *owner; #ifdef CONFIG_DEBUG_RT_MUTEXES int save_state; - struct list_head held_list_entry; - unsigned long acquire_ip; const char *name, *file; int line; void *magic; @@ -98,14 +96,6 @@ extern int rt_mutex_trylock(struct rt_mutex *lock); extern void rt_mutex_unlock(struct rt_mutex *lock); -#ifdef CONFIG_DEBUG_RT_MUTEXES -# define INIT_RT_MUTEX_DEBUG(tsk) \ - .held_list_head = LIST_HEAD_INIT(tsk.held_list_head), \ - .held_list_lock = SPIN_LOCK_UNLOCKED -#else -# define INIT_RT_MUTEX_DEBUG(tsk) -#endif - #ifdef CONFIG_RT_MUTEXES # define INIT_RT_MUTEXES(tsk) \ .pi_waiters = PLIST_HEAD_INIT(tsk.pi_waiters, tsk.pi_lock), \ diff --git a/include/linux/rwsem-spinlock.h b/include/linux/rwsem-spinlock.h index f30f805080a..ae1fcadd598 100644 --- a/include/linux/rwsem-spinlock.h +++ b/include/linux/rwsem-spinlock.h @@ -32,30 +32,37 @@ struct rw_semaphore { __s32 activity; spinlock_t wait_lock; struct list_head wait_list; -#if RWSEM_DEBUG - int debug; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; #endif }; -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname } #else -#define __RWSEM_DEBUG_INIT /* */ +# define __RWSEM_DEP_MAP_INIT(lockname) #endif #define __RWSEM_INITIALIZER(name) \ -{ 0, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) __RWSEM_DEBUG_INIT } +{ 0, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) } #define DECLARE_RWSEM(name) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name) -extern void FASTCALL(init_rwsem(struct rw_semaphore *sem)); +extern void __init_rwsem(struct rw_semaphore *sem, const char *name, + struct lock_class_key *key); + +#define init_rwsem(sem) \ +do { \ + static struct lock_class_key __key; \ + \ + __init_rwsem((sem), #sem, &__key); \ +} while (0) + extern void FASTCALL(__down_read(struct rw_semaphore *sem)); extern int FASTCALL(__down_read_trylock(struct rw_semaphore *sem)); extern void FASTCALL(__down_write(struct rw_semaphore *sem)); +extern void FASTCALL(__down_write_nested(struct rw_semaphore *sem, int subclass)); extern int FASTCALL(__down_write_trylock(struct rw_semaphore *sem)); extern void FASTCALL(__up_read(struct rw_semaphore *sem)); extern void FASTCALL(__up_write(struct rw_semaphore *sem)); diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index f99fe90732a..658afb37c3f 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -9,8 +9,6 @@ #include <linux/linkage.h> -#define RWSEM_DEBUG 0 - #ifdef __KERNEL__ #include <linux/types.h> @@ -26,89 +24,58 @@ struct rw_semaphore; #include <asm/rwsem.h> /* use an arch-specific implementation */ #endif -#ifndef rwsemtrace -#if RWSEM_DEBUG -extern void FASTCALL(rwsemtrace(struct rw_semaphore *sem, const char *str)); -#else -#define rwsemtrace(SEM,FMT) -#endif -#endif - /* * lock for reading */ -static inline void down_read(struct rw_semaphore *sem) -{ - might_sleep(); - rwsemtrace(sem,"Entering down_read"); - __down_read(sem); - rwsemtrace(sem,"Leaving down_read"); -} +extern void down_read(struct rw_semaphore *sem); /* * trylock for reading -- returns 1 if successful, 0 if contention */ -static inline int down_read_trylock(struct rw_semaphore *sem) -{ - int ret; - rwsemtrace(sem,"Entering down_read_trylock"); - ret = __down_read_trylock(sem); - rwsemtrace(sem,"Leaving down_read_trylock"); - return ret; -} +extern int down_read_trylock(struct rw_semaphore *sem); /* * lock for writing */ -static inline void down_write(struct rw_semaphore *sem) -{ - might_sleep(); - rwsemtrace(sem,"Entering down_write"); - __down_write(sem); - rwsemtrace(sem,"Leaving down_write"); -} +extern void down_write(struct rw_semaphore *sem); /* * trylock for writing -- returns 1 if successful, 0 if contention */ -static inline int down_write_trylock(struct rw_semaphore *sem) -{ - int ret; - rwsemtrace(sem,"Entering down_write_trylock"); - ret = __down_write_trylock(sem); - rwsemtrace(sem,"Leaving down_write_trylock"); - return ret; -} +extern int down_write_trylock(struct rw_semaphore *sem); /* * release a read lock */ -static inline void up_read(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering up_read"); - __up_read(sem); - rwsemtrace(sem,"Leaving up_read"); -} +extern void up_read(struct rw_semaphore *sem); /* * release a write lock */ -static inline void up_write(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering up_write"); - __up_write(sem); - rwsemtrace(sem,"Leaving up_write"); -} +extern void up_write(struct rw_semaphore *sem); /* * downgrade write lock to read lock */ -static inline void downgrade_write(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering downgrade_write"); - __downgrade_write(sem); - rwsemtrace(sem,"Leaving downgrade_write"); -} +extern void downgrade_write(struct rw_semaphore *sem); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +/* + * nested locking: + */ +extern void down_read_nested(struct rw_semaphore *sem, int subclass); +extern void down_write_nested(struct rw_semaphore *sem, int subclass); +/* + * Take/release a lock when not the owner will release it: + */ +extern void down_read_non_owner(struct rw_semaphore *sem); +extern void up_read_non_owner(struct rw_semaphore *sem); +#else +# define down_read_nested(sem, subclass) down_read(sem) +# define down_write_nested(sem, subclass) down_write(sem) +# define down_read_non_owner(sem) down_read(sem) +# define up_read_non_owner(sem) up_read(sem) +#endif #endif /* __KERNEL__ */ #endif /* _LINUX_RWSEM_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index aaf723308ed..1c876e27ff9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -184,11 +184,11 @@ extern unsigned long weighted_cpuload(const int cpu); extern rwlock_t tasklist_lock; extern spinlock_t mmlist_lock; -typedef struct task_struct task_t; +struct task_struct; extern void sched_init(void); extern void sched_init_smp(void); -extern void init_idle(task_t *idle, int cpu); +extern void init_idle(struct task_struct *idle, int cpu); extern cpumask_t nohz_cpu_mask; @@ -383,7 +383,7 @@ struct signal_struct { wait_queue_head_t wait_chldexit; /* for wait4() */ /* current thread group signal load-balancing target: */ - task_t *curr_target; + struct task_struct *curr_target; /* shared signal handling: */ struct sigpending shared_pending; @@ -534,7 +534,6 @@ extern struct user_struct *find_user(uid_t); extern struct user_struct root_user; #define INIT_USER (&root_user) -typedef struct prio_array prio_array_t; struct backing_dev_info; struct reclaim_state; @@ -699,7 +698,7 @@ extern int groups_search(struct group_info *group_info, gid_t grp); ((gi)->blocks[(i)/NGROUPS_PER_BLOCK][(i)%NGROUPS_PER_BLOCK]) #ifdef ARCH_HAS_PREFETCH_SWITCH_STACK -extern void prefetch_stack(struct task_struct*); +extern void prefetch_stack(struct task_struct *t); #else static inline void prefetch_stack(struct task_struct *t) { } #endif @@ -715,6 +714,8 @@ enum sleep_type { SLEEP_INTERRUPTED, }; +struct prio_array; + struct task_struct { volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ struct thread_info *thread_info; @@ -732,7 +733,7 @@ struct task_struct { int load_weight; /* for niceness load balancing purposes */ int prio, static_prio, normal_prio; struct list_head run_list; - prio_array_t *array; + struct prio_array *array; unsigned short ioprio; unsigned int btrace_seq; @@ -865,16 +866,34 @@ struct task_struct { struct plist_head pi_waiters; /* Deadlock detection and priority inheritance handling */ struct rt_mutex_waiter *pi_blocked_on; -# ifdef CONFIG_DEBUG_RT_MUTEXES - spinlock_t held_list_lock; - struct list_head held_list_head; -# endif #endif #ifdef CONFIG_DEBUG_MUTEXES /* mutex deadlock detection */ struct mutex_waiter *blocked_on; #endif +#ifdef CONFIG_TRACE_IRQFLAGS + unsigned int irq_events; + int hardirqs_enabled; + unsigned long hardirq_enable_ip; + unsigned int hardirq_enable_event; + unsigned long hardirq_disable_ip; + unsigned int hardirq_disable_event; + int softirqs_enabled; + unsigned long softirq_disable_ip; + unsigned int softirq_disable_event; + unsigned long softirq_enable_ip; + unsigned int softirq_enable_event; + int hardirq_context; + int softirq_context; +#endif +#ifdef CONFIG_LOCKDEP +# define MAX_LOCK_DEPTH 30UL + u64 curr_chain_key; + int lockdep_depth; + struct held_lock held_locks[MAX_LOCK_DEPTH]; + unsigned int lockdep_recursion; +#endif /* journalling filesystem info */ void *journal_info; @@ -1013,9 +1032,9 @@ static inline void put_task_struct(struct task_struct *t) #define used_math() tsk_used_math(current) #ifdef CONFIG_SMP -extern int set_cpus_allowed(task_t *p, cpumask_t new_mask); +extern int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask); #else -static inline int set_cpus_allowed(task_t *p, cpumask_t new_mask) +static inline int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) { if (!cpu_isset(0, new_mask)) return -EINVAL; @@ -1024,7 +1043,8 @@ static inline int set_cpus_allowed(task_t *p, cpumask_t new_mask) #endif extern unsigned long long sched_clock(void); -extern unsigned long long current_sched_time(const task_t *current_task); +extern unsigned long long +current_sched_time(const struct task_struct *current_task); /* sched_exec is called by processes performing an exec */ #ifdef CONFIG_SMP @@ -1042,27 +1062,27 @@ static inline void idle_task_exit(void) {} extern void sched_idle_next(void); #ifdef CONFIG_RT_MUTEXES -extern int rt_mutex_getprio(task_t *p); -extern void rt_mutex_setprio(task_t *p, int prio); -extern void rt_mutex_adjust_pi(task_t *p); +extern int rt_mutex_getprio(struct task_struct *p); +extern void rt_mutex_setprio(struct task_struct *p, int prio); +extern void rt_mutex_adjust_pi(struct task_struct *p); #else -static inline int rt_mutex_getprio(task_t *p) +static inline int rt_mutex_getprio(struct task_struct *p) { return p->normal_prio; } # define rt_mutex_adjust_pi(p) do { } while (0) #endif -extern void set_user_nice(task_t *p, long nice); -extern int task_prio(const task_t *p); -extern int task_nice(const task_t *p); -extern int can_nice(const task_t *p, const int nice); -extern int task_curr(const task_t *p); +extern void set_user_nice(struct task_struct *p, long nice); +extern int task_prio(const struct task_struct *p); +extern int task_nice(const struct task_struct *p); +extern int can_nice(const struct task_struct *p, const int nice); +extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); extern int sched_setscheduler(struct task_struct *, int, struct sched_param *); -extern task_t *idle_task(int cpu); -extern task_t *curr_task(int cpu); -extern void set_curr_task(int cpu, task_t *p); +extern struct task_struct *idle_task(int cpu); +extern struct task_struct *curr_task(int cpu); +extern void set_curr_task(int cpu, struct task_struct *p); void yield(void); @@ -1119,8 +1139,8 @@ extern void FASTCALL(wake_up_new_task(struct task_struct * tsk, #else static inline void kick_process(struct task_struct *tsk) { } #endif -extern void FASTCALL(sched_fork(task_t * p, int clone_flags)); -extern void FASTCALL(sched_exit(task_t * p)); +extern void FASTCALL(sched_fork(struct task_struct * p, int clone_flags)); +extern void FASTCALL(sched_exit(struct task_struct * p)); extern int in_group_p(gid_t); extern int in_egroup_p(gid_t); @@ -1225,17 +1245,17 @@ extern NORET_TYPE void do_group_exit(int); extern void daemonize(const char *, ...); extern int allow_signal(int); extern int disallow_signal(int); -extern task_t *child_reaper; +extern struct task_struct *child_reaper; extern int do_execve(char *, char __user * __user *, char __user * __user *, struct pt_regs *); extern long do_fork(unsigned long, unsigned long, struct pt_regs *, unsigned long, int __user *, int __user *); -task_t *fork_idle(int); +struct task_struct *fork_idle(int); extern void set_task_comm(struct task_struct *tsk, char *from); extern void get_task_comm(char *to, struct task_struct *tsk); #ifdef CONFIG_SMP -extern void wait_task_inactive(task_t * p); +extern void wait_task_inactive(struct task_struct * p); #else #define wait_task_inactive(p) do { } while (0) #endif @@ -1261,13 +1281,13 @@ extern void wait_task_inactive(task_t * p); /* de_thread depends on thread_group_leader not being a pid based check */ #define thread_group_leader(p) (p == p->group_leader) -static inline task_t *next_thread(const task_t *p) +static inline struct task_struct *next_thread(const struct task_struct *p) { return list_entry(rcu_dereference(p->thread_group.next), - task_t, thread_group); + struct task_struct, thread_group); } -static inline int thread_group_empty(task_t *p) +static inline int thread_group_empty(struct task_struct *p) { return list_empty(&p->thread_group); } diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 7bc5c7c12b5..46000936f8f 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -38,9 +38,17 @@ typedef struct { * These macros triggered gcc-3.x compile-time problems. We think these are * OK now. Be cautious. */ -#define SEQLOCK_UNLOCKED { 0, SPIN_LOCK_UNLOCKED } -#define seqlock_init(x) do { *(x) = (seqlock_t) SEQLOCK_UNLOCKED; } while (0) +#define __SEQLOCK_UNLOCKED(lockname) \ + { 0, __SPIN_LOCK_UNLOCKED(lockname) } +#define SEQLOCK_UNLOCKED \ + __SEQLOCK_UNLOCKED(old_style_seqlock_init) + +#define seqlock_init(x) \ + do { *(x) = (seqlock_t) __SEQLOCK_UNLOCKED(x); } while (0) + +#define DEFINE_SEQLOCK(x) \ + seqlock_t x = __SEQLOCK_UNLOCKED(x) /* Lock out other writers and update the count. * Acts like a normal spin_lock/unlock. diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 57d7d4965f9..3597b4f1438 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -604,9 +604,12 @@ static inline __u32 skb_queue_len(const struct sk_buff_head *list_) return list_->qlen; } +extern struct lock_class_key skb_queue_lock_key; + static inline void skb_queue_head_init(struct sk_buff_head *list) { spin_lock_init(&list->lock); + lockdep_set_class(&list->lock, &skb_queue_lock_key); list->prev = list->next = (struct sk_buff *)list; list->qlen = 0; } diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index ae23beef9cc..31473db92d3 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -82,14 +82,40 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); /* * Pull the __raw*() functions/declarations (UP-nondebug doesnt need them): */ -#if defined(CONFIG_SMP) +#ifdef CONFIG_SMP # include <asm/spinlock.h> #else # include <linux/spinlock_up.h> #endif -#define spin_lock_init(lock) do { *(lock) = SPIN_LOCK_UNLOCKED; } while (0) -#define rwlock_init(lock) do { *(lock) = RW_LOCK_UNLOCKED; } while (0) +#ifdef CONFIG_DEBUG_SPINLOCK + extern void __spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key); +# define spin_lock_init(lock) \ +do { \ + static struct lock_class_key __key; \ + \ + __spin_lock_init((lock), #lock, &__key); \ +} while (0) + +#else +# define spin_lock_init(lock) \ + do { *(lock) = SPIN_LOCK_UNLOCKED; } while (0) +#endif + +#ifdef CONFIG_DEBUG_SPINLOCK + extern void __rwlock_init(rwlock_t *lock, const char *name, + struct lock_class_key *key); +# define rwlock_init(lock) \ +do { \ + static struct lock_class_key __key; \ + \ + __rwlock_init((lock), #lock, &__key); \ +} while (0) +#else +# define rwlock_init(lock) \ + do { *(lock) = RW_LOCK_UNLOCKED; } while (0) +#endif #define spin_is_locked(lock) __raw_spin_is_locked(&(lock)->raw_lock) @@ -113,7 +139,6 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) extern int _raw_spin_trylock(spinlock_t *lock); extern void _raw_spin_unlock(spinlock_t *lock); - extern void _raw_read_lock(rwlock_t *lock); extern int _raw_read_trylock(rwlock_t *lock); extern void _raw_read_unlock(rwlock_t *lock); @@ -121,17 +146,17 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); extern int _raw_write_trylock(rwlock_t *lock); extern void _raw_write_unlock(rwlock_t *lock); #else -# define _raw_spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) -# define _raw_spin_trylock(lock) __raw_spin_trylock(&(lock)->raw_lock) # define _raw_spin_lock(lock) __raw_spin_lock(&(lock)->raw_lock) # define _raw_spin_lock_flags(lock, flags) \ __raw_spin_lock_flags(&(lock)->raw_lock, *(flags)) +# define _raw_spin_trylock(lock) __raw_spin_trylock(&(lock)->raw_lock) +# define _raw_spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) # define _raw_read_lock(rwlock) __raw_read_lock(&(rwlock)->raw_lock) -# define _raw_write_lock(rwlock) __raw_write_lock(&(rwlock)->raw_lock) -# define _raw_read_unlock(rwlock) __raw_read_unlock(&(rwlock)->raw_lock) -# define _raw_write_unlock(rwlock) __raw_write_unlock(&(rwlock)->raw_lock) # define _raw_read_trylock(rwlock) __raw_read_trylock(&(rwlock)->raw_lock) +# define _raw_read_unlock(rwlock) __raw_read_unlock(&(rwlock)->raw_lock) +# define _raw_write_lock(rwlock) __raw_write_lock(&(rwlock)->raw_lock) # define _raw_write_trylock(rwlock) __raw_write_trylock(&(rwlock)->raw_lock) +# define _raw_write_unlock(rwlock) __raw_write_unlock(&(rwlock)->raw_lock) #endif #define read_can_lock(rwlock) __raw_read_can_lock(&(rwlock)->raw_lock) @@ -147,6 +172,13 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); #define write_trylock(lock) __cond_lock(_write_trylock(lock)) #define spin_lock(lock) _spin_lock(lock) + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass) +#else +# define spin_lock_nested(lock, subclass) _spin_lock(lock) +#endif + #define write_lock(lock) _write_lock(lock) #define read_lock(lock) _read_lock(lock) @@ -172,21 +204,18 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); /* * We inline the unlock functions in the nondebug case: */ -#if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) +#if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \ + !defined(CONFIG_SMP) # define spin_unlock(lock) _spin_unlock(lock) # define read_unlock(lock) _read_unlock(lock) # define write_unlock(lock) _write_unlock(lock) -#else -# define spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) -# define read_unlock(lock) __raw_read_unlock(&(lock)->raw_lock) -# define write_unlock(lock) __raw_write_unlock(&(lock)->raw_lock) -#endif - -#if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) # define spin_unlock_irq(lock) _spin_unlock_irq(lock) # define read_unlock_irq(lock) _read_unlock_irq(lock) # define write_unlock_irq(lock) _write_unlock_irq(lock) #else +# define spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) +# define read_unlock(lock) __raw_read_unlock(&(lock)->raw_lock) +# define write_unlock(lock) __raw_write_unlock(&(lock)->raw_lock) # define spin_unlock_irq(lock) \ do { __raw_spin_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) # define read_unlock_irq(lock) \ diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 78e6989ffb5..b2c4f829946 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -20,6 +20,8 @@ int in_lock_functions(unsigned long addr); #define assert_spin_locked(x) BUG_ON(!spin_is_locked(x)) void __lockfunc _spin_lock(spinlock_t *lock) __acquires(spinlock_t); +void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) + __acquires(spinlock_t); void __lockfunc _read_lock(rwlock_t *lock) __acquires(rwlock_t); void __lockfunc _write_lock(rwlock_t *lock) __acquires(rwlock_t); void __lockfunc _spin_lock_bh(spinlock_t *lock) __acquires(spinlock_t); diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index cd81cee566f..67faa044c5f 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -49,6 +49,7 @@ do { local_irq_restore(flags); __UNLOCK(lock); } while (0) #define _spin_lock(lock) __LOCK(lock) +#define _spin_lock_nested(lock, subclass) __LOCK(lock) #define _read_lock(lock) __LOCK(lock) #define _write_lock(lock) __LOCK(lock) #define _spin_lock_bh(lock) __LOCK_BH(lock) diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h index 9cb51e07039..dc5fb69e4de 100644 --- a/include/linux/spinlock_types.h +++ b/include/linux/spinlock_types.h @@ -9,6 +9,8 @@ * Released under the General Public License (GPL). */ +#include <linux/lockdep.h> + #if defined(CONFIG_SMP) # include <asm/spinlock_types.h> #else @@ -24,6 +26,9 @@ typedef struct { unsigned int magic, owner_cpu; void *owner; #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } spinlock_t; #define SPINLOCK_MAGIC 0xdead4ead @@ -37,31 +42,53 @@ typedef struct { unsigned int magic, owner_cpu; void *owner; #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } rwlock_t; #define RWLOCK_MAGIC 0xdeaf1eed #define SPINLOCK_OWNER_INIT ((void *)-1L) +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } +#else +# define SPIN_DEP_MAP_INIT(lockname) +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } +#else +# define RW_DEP_MAP_INIT(lockname) +#endif + #ifdef CONFIG_DEBUG_SPINLOCK -# define SPIN_LOCK_UNLOCKED \ +# define __SPIN_LOCK_UNLOCKED(lockname) \ (spinlock_t) { .raw_lock = __RAW_SPIN_LOCK_UNLOCKED, \ .magic = SPINLOCK_MAGIC, \ .owner = SPINLOCK_OWNER_INIT, \ - .owner_cpu = -1 } -#define RW_LOCK_UNLOCKED \ + .owner_cpu = -1, \ + SPIN_DEP_MAP_INIT(lockname) } +#define __RW_LOCK_UNLOCKED(lockname) \ (rwlock_t) { .raw_lock = __RAW_RW_LOCK_UNLOCKED, \ .magic = RWLOCK_MAGIC, \ .owner = SPINLOCK_OWNER_INIT, \ - .owner_cpu = -1 } + .owner_cpu = -1, \ + RW_DEP_MAP_INIT(lockname) } #else -# define SPIN_LOCK_UNLOCKED \ - (spinlock_t) { .raw_lock = __RAW_SPIN_LOCK_UNLOCKED } -#define RW_LOCK_UNLOCKED \ - (rwlock_t) { .raw_lock = __RAW_RW_LOCK_UNLOCKED } +# define __SPIN_LOCK_UNLOCKED(lockname) \ + (spinlock_t) { .raw_lock = __RAW_SPIN_LOCK_UNLOCKED, \ + SPIN_DEP_MAP_INIT(lockname) } +#define __RW_LOCK_UNLOCKED(lockname) \ + (rwlock_t) { .raw_lock = __RAW_RW_LOCK_UNLOCKED, \ + RW_DEP_MAP_INIT(lockname) } #endif -#define DEFINE_SPINLOCK(x) spinlock_t x = SPIN_LOCK_UNLOCKED -#define DEFINE_RWLOCK(x) rwlock_t x = RW_LOCK_UNLOCKED +#define SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(old_style_spin_init) +#define RW_LOCK_UNLOCKED __RW_LOCK_UNLOCKED(old_style_rw_init) + +#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) +#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) #endif /* __LINUX_SPINLOCK_TYPES_H */ diff --git a/include/linux/spinlock_types_up.h b/include/linux/spinlock_types_up.h index 04135b0e198..27644af20b7 100644 --- a/include/linux/spinlock_types_up.h +++ b/include/linux/spinlock_types_up.h @@ -12,10 +12,14 @@ * Released under the General Public License (GPL). */ -#ifdef CONFIG_DEBUG_SPINLOCK +#if defined(CONFIG_DEBUG_SPINLOCK) || \ + defined(CONFIG_DEBUG_LOCK_ALLOC) typedef struct { volatile unsigned int slock; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } raw_spinlock_t; #define __RAW_SPIN_LOCK_UNLOCKED { 1 } @@ -30,6 +34,9 @@ typedef struct { } raw_spinlock_t; typedef struct { /* no debug version on UP */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } raw_rwlock_t; #define __RAW_RW_LOCK_UNLOCKED { } diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h index 31accf2f0b1..ea54c4c9a4e 100644 --- a/include/linux/spinlock_up.h +++ b/include/linux/spinlock_up.h @@ -18,7 +18,6 @@ */ #ifdef CONFIG_DEBUG_SPINLOCK - #define __raw_spin_is_locked(x) ((x)->slock == 0) static inline void __raw_spin_lock(raw_spinlock_t *lock) diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h new file mode 100644 index 00000000000..9cc81e57222 --- /dev/null +++ b/include/linux/stacktrace.h @@ -0,0 +1,20 @@ +#ifndef __LINUX_STACKTRACE_H +#define __LINUX_STACKTRACE_H + +#ifdef CONFIG_STACKTRACE +struct stack_trace { + unsigned int nr_entries, max_entries; + unsigned long *entries; +}; + +extern void save_stack_trace(struct stack_trace *trace, + struct task_struct *task, int all_contexts, + unsigned int skip); + +extern void print_stack_trace(struct stack_trace *trace, int spaces); +#else +# define save_stack_trace(trace, task, all, skip) do { } while (0) +# define print_stack_trace(trace) do { } while (0) +#endif + +#endif diff --git a/include/linux/swap.h b/include/linux/swap.h index cf6ca6e377b..5e59184c909 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -189,6 +189,7 @@ extern long vm_total_pages; #ifdef CONFIG_NUMA extern int zone_reclaim_mode; +extern int sysctl_min_unmapped_ratio; extern int zone_reclaim(struct zone *, gfp_t, unsigned int); #else #define zone_reclaim_mode 0 diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 46e4d8f2771..e4b1a4d4dcf 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -188,7 +188,7 @@ enum VM_DROP_PAGECACHE=29, /* int: nuke lots of pagecache */ VM_PERCPU_PAGELIST_FRACTION=30,/* int: fraction of pages in each percpu_pagelist */ VM_ZONE_RECLAIM_MODE=31, /* reclaim local zone memory before going off node */ - VM_ZONE_RECLAIM_INTERVAL=32, /* time period to wait after reclaim failure */ + VM_MIN_UNMAPPED=32, /* Set min percent of unmapped pages */ VM_PANIC_ON_OOM=33, /* panic at out-of-memory */ VM_VDSO_ENABLED=34, /* map VDSO into new processes? */ }; diff --git a/include/linux/wait.h b/include/linux/wait.h index 544e855c7c0..794be7af58a 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -68,7 +68,7 @@ struct task_struct; wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk) #define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \ - .lock = SPIN_LOCK_UNLOCKED, \ + .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .task_list = { &(name).task_list, &(name).task_list } } #define DECLARE_WAIT_QUEUE_HEAD(name) \ @@ -77,9 +77,15 @@ struct task_struct; #define __WAIT_BIT_KEY_INITIALIZER(word, bit) \ { .flags = word, .bit_nr = bit, } +/* + * lockdep: we want one lock-class for all waitqueue locks. + */ +extern struct lock_class_key waitqueue_lock_key; + static inline void init_waitqueue_head(wait_queue_head_t *q) { spin_lock_init(&q->lock); + lockdep_set_class(&q->lock, &waitqueue_lock_key); INIT_LIST_HEAD(&q->task_list); } diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 5ba72d95280..2fec827c880 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -67,6 +67,9 @@ struct unix_skb_parms { #define unix_state_rlock(s) spin_lock(&unix_sk(s)->lock) #define unix_state_runlock(s) spin_unlock(&unix_sk(s)->lock) #define unix_state_wlock(s) spin_lock(&unix_sk(s)->lock) +#define unix_state_wlock_nested(s) \ + spin_lock_nested(&unix_sk(s)->lock, \ + SINGLE_DEPTH_NESTING) #define unix_state_wunlock(s) spin_unlock(&unix_sk(s)->lock) #ifdef __KERNEL__ diff --git a/include/net/sock.h b/include/net/sock.h index 7b3d6b85694..324b3ea233d 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -44,6 +44,7 @@ #include <linux/timer.h> #include <linux/cache.h> #include <linux/module.h> +#include <linux/lockdep.h> #include <linux/netdevice.h> #include <linux/skbuff.h> /* struct sk_buff */ #include <linux/security.h> @@ -78,14 +79,17 @@ typedef struct { spinlock_t slock; struct sock_iocb *owner; wait_queue_head_t wq; + /* + * We express the mutex-alike socket_lock semantics + * to the lock validator by explicitly managing + * the slock as a lock variant (in addition to + * the slock itself): + */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } socket_lock_t; -#define sock_lock_init(__sk) \ -do { spin_lock_init(&((__sk)->sk_lock.slock)); \ - (__sk)->sk_lock.owner = NULL; \ - init_waitqueue_head(&((__sk)->sk_lock.wq)); \ -} while(0) - struct sock; struct proto; @@ -747,6 +751,9 @@ extern void FASTCALL(release_sock(struct sock *sk)); /* BH context may only use the following locking interface. */ #define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock)) +#define bh_lock_sock_nested(__sk) \ + spin_lock_nested(&((__sk)->sk_lock.slock), \ + SINGLE_DEPTH_NESTING) #define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock)) extern struct sock *sk_alloc(int family, diff --git a/init/main.c b/init/main.c index b2f3b566790..628b8e9e841 100644 --- a/init/main.c +++ b/init/main.c @@ -47,6 +47,8 @@ #include <linux/key.h> #include <linux/unwind.h> #include <linux/buffer_head.h> +#include <linux/debug_locks.h> +#include <linux/lockdep.h> #include <asm/io.h> #include <asm/bugs.h> @@ -456,6 +458,16 @@ asmlinkage void __init start_kernel(void) smp_setup_processor_id(); + /* + * Need to run as early as possible, to initialize the + * lockdep hash: + */ + lockdep_init(); + + local_irq_disable(); + early_boot_irqs_off(); + early_init_irq_lock_class(); + /* * Interrupts are still disabled. Do necessary setups, then * enable them @@ -496,8 +508,13 @@ asmlinkage void __init start_kernel(void) init_timers(); hrtimers_init(); softirq_init(); - time_init(); timekeeping_init(); + time_init(); + profile_init(); + if (!irqs_disabled()) + printk("start_kernel(): bug: interrupts were enabled early\n"); + early_boot_irqs_on(); + local_irq_enable(); /* * HACK ALERT! This is early. We're enabling the console before @@ -507,8 +524,16 @@ asmlinkage void __init start_kernel(void) console_init(); if (panic_later) panic(panic_later, panic_param); - profile_init(); - local_irq_enable(); + + lockdep_info(); + + /* + * Need to run this when irqs are enabled, because it wants + * to self-test [hard/soft]-irqs on/off lock inversion bugs + * too: + */ + locking_selftest(); + #ifdef CONFIG_BLK_DEV_INITRD if (initrd_start && !initrd_below_start_ok && initrd_start < min_low_pfn << PAGE_SHIFT) { diff --git a/kernel/Makefile b/kernel/Makefile index 82fb182f6f6..47dbcd570cd 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -8,10 +8,15 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ signal.o sys.o kmod.o workqueue.o pid.o \ rcupdate.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ - hrtimer.o + hrtimer.o rwsem.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-y += time/ obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o +obj-$(CONFIG_LOCKDEP) += lockdep.o +ifeq ($(CONFIG_PROC_FS),y) +obj-$(CONFIG_LOCKDEP) += lockdep_proc.o +endif obj-$(CONFIG_FUTEX) += futex.o ifeq ($(CONFIG_COMPAT),y) obj-$(CONFIG_FUTEX) += futex_compat.o @@ -22,6 +27,7 @@ obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o obj-$(CONFIG_SMP) += cpu.o spinlock.o obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o +obj-$(CONFIG_PROVE_LOCKING) += spinlock.o obj-$(CONFIG_UID16) += uid16.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_KALLSYMS) += kallsyms.o diff --git a/kernel/capability.c b/kernel/capability.c index 1a4d8a40d3f..c7685ad00a9 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -46,7 +46,7 @@ asmlinkage long sys_capget(cap_user_header_t header, cap_user_data_t dataptr) int ret = 0; pid_t pid; __u32 version; - task_t *target; + struct task_struct *target; struct __user_cap_data_struct data; if (get_user(version, &header->version)) @@ -96,7 +96,7 @@ static inline int cap_set_pg(int pgrp, kernel_cap_t *effective, kernel_cap_t *inheritable, kernel_cap_t *permitted) { - task_t *g, *target; + struct task_struct *g, *target; int ret = -EPERM; int found = 0; @@ -128,7 +128,7 @@ static inline int cap_set_all(kernel_cap_t *effective, kernel_cap_t *inheritable, kernel_cap_t *permitted) { - task_t *g, *target; + struct task_struct *g, *target; int ret = -EPERM; int found = 0; @@ -172,7 +172,7 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data) { kernel_cap_t inheritable, permitted, effective; __u32 version; - task_t *target; + struct task_struct *target; int ret; pid_t pid; diff --git a/kernel/exit.c b/kernel/exit.c index 7f7ef225855..6664c084783 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -134,8 +134,8 @@ static void delayed_put_task_struct(struct rcu_head *rhp) void release_task(struct task_struct * p) { + struct task_struct *leader; int zap_leader; - task_t *leader; repeat: atomic_dec(&p->user->processes); write_lock_irq(&tasklist_lock); @@ -209,7 +209,7 @@ out: * * "I ask you, have you ever known what it is to be an orphan?" */ -static int will_become_orphaned_pgrp(int pgrp, task_t *ignored_task) +static int will_become_orphaned_pgrp(int pgrp, struct task_struct *ignored_task) { struct task_struct *p; int ret = 1; @@ -582,7 +582,8 @@ static void exit_mm(struct task_struct * tsk) mmput(mm); } -static inline void choose_new_parent(task_t *p, task_t *reaper) +static inline void +choose_new_parent(struct task_struct *p, struct task_struct *reaper) { /* * Make sure we're not reparenting to ourselves and that @@ -592,7 +593,8 @@ static inline void choose_new_parent(task_t *p, task_t *reaper) p->real_parent = reaper; } -static void reparent_thread(task_t *p, task_t *father, int traced) +static void +reparent_thread(struct task_struct *p, struct task_struct *father, int traced) { /* We don't want people slaying init. */ if (p->exit_signal != -1) @@ -656,8 +658,8 @@ static void reparent_thread(task_t *p, task_t *father, int traced) * group, and if no such member exists, give it to * the global child reaper process (ie "init") */ -static void forget_original_parent(struct task_struct * father, - struct list_head *to_release) +static void +forget_original_parent(struct task_struct *father, struct list_head *to_release) { struct task_struct *p, *reaper = father; struct list_head *_p, *_n; @@ -680,7 +682,7 @@ static void forget_original_parent(struct task_struct * father, */ list_for_each_safe(_p, _n, &father->children) { int ptrace; - p = list_entry(_p,struct task_struct,sibling); + p = list_entry(_p, struct task_struct, sibling); ptrace = p->ptrace; @@ -709,7 +711,7 @@ static void forget_original_parent(struct task_struct * father, list_add(&p->ptrace_list, to_release); } list_for_each_safe(_p, _n, &father->ptrace_children) { - p = list_entry(_p,struct task_struct,ptrace_list); + p = list_entry(_p, struct task_struct, ptrace_list); choose_new_parent(p, reaper); reparent_thread(p, father, 1); } @@ -829,7 +831,7 @@ static void exit_notify(struct task_struct *tsk) list_for_each_safe(_p, _n, &ptrace_dead) { list_del_init(_p); - t = list_entry(_p,struct task_struct,ptrace_list); + t = list_entry(_p, struct task_struct, ptrace_list); release_task(t); } @@ -933,10 +935,9 @@ fastcall NORET_TYPE void do_exit(long code) if (unlikely(current->pi_state_cache)) kfree(current->pi_state_cache); /* - * If DEBUG_MUTEXES is on, make sure we are holding no locks: + * Make sure we are holding no locks: */ - mutex_debug_check_no_locks_held(tsk); - rt_mutex_debug_check_no_locks_held(tsk); + debug_check_no_locks_held(tsk); if (tsk->io_context) exit_io_context(); @@ -1011,7 +1012,7 @@ asmlinkage void sys_exit_group(int error_code) do_group_exit((error_code & 0xff) << 8); } -static int eligible_child(pid_t pid, int options, task_t *p) +static int eligible_child(pid_t pid, int options, struct task_struct *p) { if (pid > 0) { if (p->pid != pid) @@ -1052,12 +1053,13 @@ static int eligible_child(pid_t pid, int options, task_t *p) return 1; } -static int wait_noreap_copyout(task_t *p, pid_t pid, uid_t uid, +static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid, int why, int status, struct siginfo __user *infop, struct rusage __user *rusagep) { int retval = rusagep ? getrusage(p, RUSAGE_BOTH, rusagep) : 0; + put_task_struct(p); if (!retval) retval = put_user(SIGCHLD, &infop->si_signo); @@ -1082,7 +1084,7 @@ static int wait_noreap_copyout(task_t *p, pid_t pid, uid_t uid, * the lock and this task is uninteresting. If we return nonzero, we have * released the lock and the system call should return. */ -static int wait_task_zombie(task_t *p, int noreap, +static int wait_task_zombie(struct task_struct *p, int noreap, struct siginfo __user *infop, int __user *stat_addr, struct rusage __user *ru) { @@ -1244,8 +1246,8 @@ static int wait_task_zombie(task_t *p, int noreap, * the lock and this task is uninteresting. If we return nonzero, we have * released the lock and the system call should return. */ -static int wait_task_stopped(task_t *p, int delayed_group_leader, int noreap, - struct siginfo __user *infop, +static int wait_task_stopped(struct task_struct *p, int delayed_group_leader, + int noreap, struct siginfo __user *infop, int __user *stat_addr, struct rusage __user *ru) { int retval, exit_code; @@ -1359,7 +1361,7 @@ bail_ref: * the lock and this task is uninteresting. If we return nonzero, we have * released the lock and the system call should return. */ -static int wait_task_continued(task_t *p, int noreap, +static int wait_task_continued(struct task_struct *p, int noreap, struct siginfo __user *infop, int __user *stat_addr, struct rusage __user *ru) { @@ -1445,7 +1447,7 @@ repeat: int ret; list_for_each(_p,&tsk->children) { - p = list_entry(_p,struct task_struct,sibling); + p = list_entry(_p, struct task_struct, sibling); ret = eligible_child(pid, options, p); if (!ret) diff --git a/kernel/fork.c b/kernel/fork.c index 9064bf9e131..56e4e07e45f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -193,7 +193,10 @@ static inline int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) down_write(&oldmm->mmap_sem); flush_cache_mm(oldmm); - down_write(&mm->mmap_sem); + /* + * Not linked in yet - no deadlock potential: + */ + down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); mm->locked_vm = 0; mm->mmap = NULL; @@ -919,10 +922,6 @@ static inline void rt_mutex_init_task(struct task_struct *p) spin_lock_init(&p->pi_lock); plist_head_init(&p->pi_waiters, &p->pi_lock); p->pi_blocked_on = NULL; -# ifdef CONFIG_DEBUG_RT_MUTEXES - spin_lock_init(&p->held_list_lock); - INIT_LIST_HEAD(&p->held_list_head); -# endif #endif } @@ -934,13 +933,13 @@ static inline void rt_mutex_init_task(struct task_struct *p) * parts of the process environment (as per the clone * flags). The actual kick-off is left to the caller. */ -static task_t *copy_process(unsigned long clone_flags, - unsigned long stack_start, - struct pt_regs *regs, - unsigned long stack_size, - int __user *parent_tidptr, - int __user *child_tidptr, - int pid) +static struct task_struct *copy_process(unsigned long clone_flags, + unsigned long stack_start, + struct pt_regs *regs, + unsigned long stack_size, + int __user *parent_tidptr, + int __user *child_tidptr, + int pid) { int retval; struct task_struct *p = NULL; @@ -972,6 +971,10 @@ static task_t *copy_process(unsigned long clone_flags, if (!p) goto fork_out; +#ifdef CONFIG_TRACE_IRQFLAGS + DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); + DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); +#endif retval = -EAGAIN; if (atomic_read(&p->user->processes) >= p->signal->rlim[RLIMIT_NPROC].rlim_cur) { @@ -1046,6 +1049,26 @@ static task_t *copy_process(unsigned long clone_flags, } mpol_fix_fork_child_flag(p); #endif +#ifdef CONFIG_TRACE_IRQFLAGS + p->irq_events = 0; + p->hardirqs_enabled = 0; + p->hardirq_enable_ip = 0; + p->hardirq_enable_event = 0; + p->hardirq_disable_ip = _THIS_IP_; + p->hardirq_disable_event = 0; + p->softirqs_enabled = 1; + p->softirq_enable_ip = _THIS_IP_; + p->softirq_enable_event = 0; + p->softirq_disable_ip = 0; + p->softirq_disable_event = 0; + p->hardirq_context = 0; + p->softirq_context = 0; +#endif +#ifdef CONFIG_LOCKDEP + p->lockdep_depth = 0; /* no locks held yet */ + p->curr_chain_key = 0; + p->lockdep_recursion = 0; +#endif rt_mutex_init_task(p); @@ -1271,9 +1294,9 @@ struct pt_regs * __devinit __attribute__((weak)) idle_regs(struct pt_regs *regs) return regs; } -task_t * __devinit fork_idle(int cpu) +struct task_struct * __devinit fork_idle(int cpu) { - task_t *task; + struct task_struct *task; struct pt_regs regs; task = copy_process(CLONE_VM, 0, idle_regs(®s), 0, NULL, NULL, 0); diff --git a/kernel/futex.c b/kernel/futex.c index 15caf93e4a4..1dc98e4dd28 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -607,6 +607,22 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval) } /* + * Express the locking dependencies for lockdep: + */ +static inline void +double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2) +{ + if (hb1 <= hb2) { + spin_lock(&hb1->lock); + if (hb1 < hb2) + spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING); + } else { /* hb1 > hb2 */ + spin_lock(&hb2->lock); + spin_lock_nested(&hb1->lock, SINGLE_DEPTH_NESTING); + } +} + +/* * Wake up all waiters hashed on the physical page that is mapped * to this virtual address: */ @@ -674,11 +690,7 @@ retryfull: hb2 = hash_futex(&key2); retry: - if (hb1 < hb2) - spin_lock(&hb1->lock); - spin_lock(&hb2->lock); - if (hb1 > hb2) - spin_lock(&hb1->lock); + double_lock_hb(hb1, hb2); op_ret = futex_atomic_op_inuser(op, uaddr2); if (unlikely(op_ret < 0)) { @@ -787,11 +799,7 @@ static int futex_requeue(u32 __user *uaddr1, u32 __user *uaddr2, hb1 = hash_futex(&key1); hb2 = hash_futex(&key2); - if (hb1 < hb2) - spin_lock(&hb1->lock); - spin_lock(&hb2->lock); - if (hb1 > hb2) - spin_lock(&hb1->lock); + double_lock_hb(hb1, hb2); if (likely(cmpval != NULL)) { u32 curval; diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 8d3dc29ef41..d17766d40da 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -669,7 +669,7 @@ static int hrtimer_wakeup(struct hrtimer *timer) return HRTIMER_NORESTART; } -void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, task_t *task) +void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) { sl->timer.function = hrtimer_wakeup; sl->task = task; @@ -782,8 +782,10 @@ static void __devinit init_hrtimers_cpu(int cpu) struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu); int i; - for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) + for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) { spin_lock_init(&base->lock); + lockdep_set_class(&base->lock, &base->lock_key); + } } #ifdef CONFIG_HOTPLUG_CPU diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 6d8b3011496..fc4e906aedb 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -16,10 +16,6 @@ #include <linux/interrupt.h> #include <linux/kernel_stat.h> -#if defined(CONFIG_NO_IDLE_HZ) && defined(CONFIG_ARM) -#include <asm/dyntick.h> -#endif - #include "internals.h" /** @@ -133,17 +129,10 @@ irqreturn_t handle_IRQ_event(unsigned int irq, struct pt_regs *regs, irqreturn_t ret, retval = IRQ_NONE; unsigned int status = 0; -#if defined(CONFIG_NO_IDLE_HZ) && defined(CONFIG_ARM) - if (!(action->flags & SA_TIMER) && system_timer->dyn_tick != NULL) { - write_seqlock(&xtime_lock); - if (system_timer->dyn_tick->state & DYN_TICK_ENABLED) - system_timer->dyn_tick->handler(irq, 0, regs); - write_sequnlock(&xtime_lock); - } -#endif + handle_dynamic_tick(action); if (!(action->flags & IRQF_DISABLED)) - local_irq_enable(); + local_irq_enable_in_hardirq(); do { ret = action->handler(irq, action->dev_id, regs); @@ -260,3 +249,19 @@ out: return 1; } +#ifdef CONFIG_TRACE_IRQFLAGS + +/* + * lockdep: we want to handle all irq_desc locks as a single lock-class: + */ +static struct lock_class_key irq_desc_lock_class; + +void early_init_irq_lock_class(void) +{ + int i; + + for (i = 0; i < NR_IRQS; i++) + lockdep_set_class(&irq_desc[i].lock, &irq_desc_lock_class); +} + +#endif diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index fede5fa351d..4e461438e48 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -234,7 +234,7 @@ int setup_irq(unsigned int irq, struct irqaction *new) ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK)) goto mismatch; -#if defined(CONFIG_IRQ_PER_CPU) && defined(IRQF_PERCPU) +#if defined(CONFIG_IRQ_PER_CPU) /* All handlers must agree on per-cpuness */ if ((old->flags & IRQF_PERCPU) != (new->flags & IRQF_PERCPU)) @@ -250,7 +250,7 @@ int setup_irq(unsigned int irq, struct irqaction *new) } *p = new; -#if defined(CONFIG_IRQ_PER_CPU) && defined(IRQF_PERCPU) +#if defined(CONFIG_IRQ_PER_CPU) if (new->flags & IRQF_PERCPU) desc->status |= IRQ_PER_CPU; #endif @@ -410,6 +410,12 @@ int request_irq(unsigned int irq, struct irqaction *action; int retval; +#ifdef CONFIG_LOCKDEP + /* + * Lockdep wants atomic interrupt handlers: + */ + irqflags |= SA_INTERRUPT; +#endif /* * Sanity-check: shared interrupts must pass in a real dev-ID, * otherwise we'll have trouble later trying to figure out diff --git a/kernel/kmod.c b/kernel/kmod.c index 1b7157af051..1d32defa38a 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c @@ -233,7 +233,7 @@ static void __call_usermodehelper(void *data) int call_usermodehelper_keys(char *path, char **argv, char **envp, struct key *session_keyring, int wait) { - DECLARE_COMPLETION(done); + DECLARE_COMPLETION_ONSTACK(done); struct subprocess_info sub_info = { .complete = &done, .path = path, diff --git a/kernel/lockdep.c b/kernel/lockdep.c new file mode 100644 index 00000000000..f32ca78c198 --- /dev/null +++ b/kernel/lockdep.c @@ -0,0 +1,2702 @@ +/* + * kernel/lockdep.c + * + * Runtime locking correctness validator + * + * Started by Ingo Molnar: + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + * + * this code maps all the lock dependencies as they occur in a live kernel + * and will warn about the following classes of locking bugs: + * + * - lock inversion scenarios + * - circular lock dependencies + * - hardirq/softirq safe/unsafe locking bugs + * + * Bugs are reported even if the current locking scenario does not cause + * any deadlock at this point. + * + * I.e. if anytime in the past two locks were taken in a different order, + * even if it happened for another task, even if those were different + * locks (but of the same class as this lock), this code will detect it. + * + * Thanks to Arjan van de Ven for coming up with the initial idea of + * mapping lock dependencies runtime. + */ +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/spinlock.h> +#include <linux/kallsyms.h> +#include <linux/interrupt.h> +#include <linux/stacktrace.h> +#include <linux/debug_locks.h> +#include <linux/irqflags.h> + +#include <asm/sections.h> + +#include "lockdep_internals.h" + +/* + * hash_lock: protects the lockdep hashes and class/list/hash allocators. + * + * This is one of the rare exceptions where it's justified + * to use a raw spinlock - we really dont want the spinlock + * code to recurse back into the lockdep code. + */ +static raw_spinlock_t hash_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; + +static int lockdep_initialized; + +unsigned long nr_list_entries; +static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; + +/* + * Allocate a lockdep entry. (assumes hash_lock held, returns + * with NULL on failure) + */ +static struct lock_list *alloc_list_entry(void) +{ + if (nr_list_entries >= MAX_LOCKDEP_ENTRIES) { + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + printk("BUG: MAX_LOCKDEP_ENTRIES too low!\n"); + printk("turning off the locking correctness validator.\n"); + return NULL; + } + return list_entries + nr_list_entries++; +} + +/* + * All data structures here are protected by the global debug_lock. + * + * Mutex key structs only get allocated, once during bootup, and never + * get freed - this significantly simplifies the debugging code. + */ +unsigned long nr_lock_classes; +static struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; + +/* + * We keep a global list of all lock classes. The list only grows, + * never shrinks. The list is only accessed with the lockdep + * spinlock lock held. + */ +LIST_HEAD(all_lock_classes); + +/* + * The lockdep classes are in a hash-table as well, for fast lookup: + */ +#define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1) +#define CLASSHASH_SIZE (1UL << CLASSHASH_BITS) +#define CLASSHASH_MASK (CLASSHASH_SIZE - 1) +#define __classhashfn(key) ((((unsigned long)key >> CLASSHASH_BITS) + (unsigned long)key) & CLASSHASH_MASK) +#define classhashentry(key) (classhash_table + __classhashfn((key))) + +static struct list_head classhash_table[CLASSHASH_SIZE]; + +unsigned long nr_lock_chains; +static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; + +/* + * We put the lock dependency chains into a hash-table as well, to cache + * their existence: + */ +#define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1) +#define CHAINHASH_SIZE (1UL << CHAINHASH_BITS) +#define CHAINHASH_MASK (CHAINHASH_SIZE - 1) +#define __chainhashfn(chain) \ + (((chain >> CHAINHASH_BITS) + chain) & CHAINHASH_MASK) +#define chainhashentry(chain) (chainhash_table + __chainhashfn((chain))) + +static struct list_head chainhash_table[CHAINHASH_SIZE]; + +/* + * The hash key of the lock dependency chains is a hash itself too: + * it's a hash of all locks taken up to that lock, including that lock. + * It's a 64-bit hash, because it's important for the keys to be + * unique. + */ +#define iterate_chain_key(key1, key2) \ + (((key1) << MAX_LOCKDEP_KEYS_BITS/2) ^ \ + ((key1) >> (64-MAX_LOCKDEP_KEYS_BITS/2)) ^ \ + (key2)) + +void lockdep_off(void) +{ + current->lockdep_recursion++; +} + +EXPORT_SYMBOL(lockdep_off); + +void lockdep_on(void) +{ + current->lockdep_recursion--; +} + +EXPORT_SYMBOL(lockdep_on); + +int lockdep_internal(void) +{ + return current->lockdep_recursion != 0; +} + +EXPORT_SYMBOL(lockdep_internal); + +/* + * Debugging switches: + */ + +#define VERBOSE 0 +#ifdef VERBOSE +# define VERY_VERBOSE 0 +#endif + +#if VERBOSE +# define HARDIRQ_VERBOSE 1 +# define SOFTIRQ_VERBOSE 1 +#else +# define HARDIRQ_VERBOSE 0 +# define SOFTIRQ_VERBOSE 0 +#endif + +#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE +/* + * Quick filtering for interesting events: + */ +static int class_filter(struct lock_class *class) +{ + if (class->name_version == 1 && + !strcmp(class->name, "&rl->lock")) + return 1; + if (class->name_version == 1 && + !strcmp(class->name, "&ni->mrec_lock")) + return 1; + if (class->name_version == 1 && + !strcmp(class->name, "mft_ni_runlist_lock")) + return 1; + if (class->name_version == 1 && + !strcmp(class->name, "mft_ni_mrec_lock")) + return 1; + if (class->name_version == 1 && + !strcmp(class->name, "&vol->lcnbmp_lock")) + return 1; + return 0; +} +#endif + +static int verbose(struct lock_class *class) +{ +#if VERBOSE + return class_filter(class); +#endif + return 0; +} + +#ifdef CONFIG_TRACE_IRQFLAGS + +static int hardirq_verbose(struct lock_class *class) +{ +#if HARDIRQ_VERBOSE + return class_filter(class); +#endif + return 0; +} + +static int softirq_verbose(struct lock_class *class) +{ +#if SOFTIRQ_VERBOSE + return class_filter(class); +#endif + return 0; +} + +#endif + +/* + * Stack-trace: tightly packed array of stack backtrace + * addresses. Protected by the hash_lock. + */ +unsigned long nr_stack_trace_entries; +static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES]; + +static int save_trace(struct stack_trace *trace) +{ + trace->nr_entries = 0; + trace->max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries; + trace->entries = stack_trace + nr_stack_trace_entries; + + save_stack_trace(trace, NULL, 0, 3); + + trace->max_entries = trace->nr_entries; + + nr_stack_trace_entries += trace->nr_entries; + if (DEBUG_LOCKS_WARN_ON(nr_stack_trace_entries > MAX_STACK_TRACE_ENTRIES)) + return 0; + + if (nr_stack_trace_entries == MAX_STACK_TRACE_ENTRIES) { + __raw_spin_unlock(&hash_lock); + if (debug_locks_off()) { + printk("BUG: MAX_STACK_TRACE_ENTRIES too low!\n"); + printk("turning off the locking correctness validator.\n"); + dump_stack(); + } + return 0; + } + + return 1; +} + +unsigned int nr_hardirq_chains; +unsigned int nr_softirq_chains; +unsigned int nr_process_chains; +unsigned int max_lockdep_depth; +unsigned int max_recursion_depth; + +#ifdef CONFIG_DEBUG_LOCKDEP +/* + * We cannot printk in early bootup code. Not even early_printk() + * might work. So we mark any initialization errors and printk + * about it later on, in lockdep_info(). + */ +static int lockdep_init_error; + +/* + * Various lockdep statistics: + */ +atomic_t chain_lookup_hits; +atomic_t chain_lookup_misses; +atomic_t hardirqs_on_events; +atomic_t hardirqs_off_events; +atomic_t redundant_hardirqs_on; +atomic_t redundant_hardirqs_off; +atomic_t softirqs_on_events; +atomic_t softirqs_off_events; +atomic_t redundant_softirqs_on; +atomic_t redundant_softirqs_off; +atomic_t nr_unused_locks; +atomic_t nr_cyclic_checks; +atomic_t nr_cyclic_check_recursions; +atomic_t nr_find_usage_forwards_checks; +atomic_t nr_find_usage_forwards_recursions; +atomic_t nr_find_usage_backwards_checks; +atomic_t nr_find_usage_backwards_recursions; +# define debug_atomic_inc(ptr) atomic_inc(ptr) +# define debug_atomic_dec(ptr) atomic_dec(ptr) +# define debug_atomic_read(ptr) atomic_read(ptr) +#else +# define debug_atomic_inc(ptr) do { } while (0) +# define debug_atomic_dec(ptr) do { } while (0) +# define debug_atomic_read(ptr) 0 +#endif + +/* + * Locking printouts: + */ + +static const char *usage_str[] = +{ + [LOCK_USED] = "initial-use ", + [LOCK_USED_IN_HARDIRQ] = "in-hardirq-W", + [LOCK_USED_IN_SOFTIRQ] = "in-softirq-W", + [LOCK_ENABLED_SOFTIRQS] = "softirq-on-W", + [LOCK_ENABLED_HARDIRQS] = "hardirq-on-W", + [LOCK_USED_IN_HARDIRQ_READ] = "in-hardirq-R", + [LOCK_USED_IN_SOFTIRQ_READ] = "in-softirq-R", + [LOCK_ENABLED_SOFTIRQS_READ] = "softirq-on-R", + [LOCK_ENABLED_HARDIRQS_READ] = "hardirq-on-R", +}; + +const char * __get_key_name(struct lockdep_subclass_key *key, char *str) +{ + unsigned long offs, size; + char *modname; + + return kallsyms_lookup((unsigned long)key, &size, &offs, &modname, str); +} + +void +get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4) +{ + *c1 = '.', *c2 = '.', *c3 = '.', *c4 = '.'; + + if (class->usage_mask & LOCKF_USED_IN_HARDIRQ) + *c1 = '+'; + else + if (class->usage_mask & LOCKF_ENABLED_HARDIRQS) + *c1 = '-'; + + if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ) + *c2 = '+'; + else + if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS) + *c2 = '-'; + + if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) + *c3 = '-'; + if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) { + *c3 = '+'; + if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) + *c3 = '?'; + } + + if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) + *c4 = '-'; + if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) { + *c4 = '+'; + if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) + *c4 = '?'; + } +} + +static void print_lock_name(struct lock_class *class) +{ + char str[128], c1, c2, c3, c4; + const char *name; + + get_usage_chars(class, &c1, &c2, &c3, &c4); + + name = class->name; + if (!name) { + name = __get_key_name(class->key, str); + printk(" (%s", name); + } else { + printk(" (%s", name); + if (class->name_version > 1) + printk("#%d", class->name_version); + if (class->subclass) + printk("/%d", class->subclass); + } + printk("){%c%c%c%c}", c1, c2, c3, c4); +} + +static void print_lockdep_cache(struct lockdep_map *lock) +{ + const char *name; + char str[128]; + + name = lock->name; + if (!name) + name = __get_key_name(lock->key->subkeys, str); + + printk("%s", name); +} + +static void print_lock(struct held_lock *hlock) +{ + print_lock_name(hlock->class); + printk(", at: "); + print_ip_sym(hlock->acquire_ip); +} + +static void lockdep_print_held_locks(struct task_struct *curr) +{ + int i, depth = curr->lockdep_depth; + + if (!depth) { + printk("no locks held by %s/%d.\n", curr->comm, curr->pid); + return; + } + printk("%d lock%s held by %s/%d:\n", + depth, depth > 1 ? "s" : "", curr->comm, curr->pid); + + for (i = 0; i < depth; i++) { + printk(" #%d: ", i); + print_lock(curr->held_locks + i); + } +} +/* + * Helper to print a nice hierarchy of lock dependencies: + */ +static void print_spaces(int nr) +{ + int i; + + for (i = 0; i < nr; i++) + printk(" "); +} + +static void print_lock_class_header(struct lock_class *class, int depth) +{ + int bit; + + print_spaces(depth); + printk("->"); + print_lock_name(class); + printk(" ops: %lu", class->ops); + printk(" {\n"); + + for (bit = 0; bit < LOCK_USAGE_STATES; bit++) { + if (class->usage_mask & (1 << bit)) { + int len = depth; + + print_spaces(depth); + len += printk(" %s", usage_str[bit]); + len += printk(" at:\n"); + print_stack_trace(class->usage_traces + bit, len); + } + } + print_spaces(depth); + printk(" }\n"); + + print_spaces(depth); + printk(" ... key at: "); + print_ip_sym((unsigned long)class->key); +} + +/* + * printk all lock dependencies starting at <entry>: + */ +static void print_lock_dependencies(struct lock_class *class, int depth) +{ + struct lock_list *entry; + + if (DEBUG_LOCKS_WARN_ON(depth >= 20)) + return; + + print_lock_class_header(class, depth); + + list_for_each_entry(entry, &class->locks_after, entry) { + DEBUG_LOCKS_WARN_ON(!entry->class); + print_lock_dependencies(entry->class, depth + 1); + + print_spaces(depth); + printk(" ... acquired at:\n"); + print_stack_trace(&entry->trace, 2); + printk("\n"); + } +} + +/* + * Add a new dependency to the head of the list: + */ +static int add_lock_to_list(struct lock_class *class, struct lock_class *this, + struct list_head *head, unsigned long ip) +{ + struct lock_list *entry; + /* + * Lock not present yet - get a new dependency struct and + * add it to the list: + */ + entry = alloc_list_entry(); + if (!entry) + return 0; + + entry->class = this; + save_trace(&entry->trace); + + /* + * Since we never remove from the dependency list, the list can + * be walked lockless by other CPUs, it's only allocation + * that must be protected by the spinlock. But this also means + * we must make new entries visible only once writes to the + * entry become visible - hence the RCU op: + */ + list_add_tail_rcu(&entry->entry, head); + + return 1; +} + +/* + * Recursive, forwards-direction lock-dependency checking, used for + * both noncyclic checking and for hardirq-unsafe/softirq-unsafe + * checking. + * + * (to keep the stackframe of the recursive functions small we + * use these global variables, and we also mark various helper + * functions as noinline.) + */ +static struct held_lock *check_source, *check_target; + +/* + * Print a dependency chain entry (this is only done when a deadlock + * has been detected): + */ +static noinline int +print_circular_bug_entry(struct lock_list *target, unsigned int depth) +{ + if (debug_locks_silent) + return 0; + printk("\n-> #%u", depth); + print_lock_name(target->class); + printk(":\n"); + print_stack_trace(&target->trace, 6); + + return 0; +} + +/* + * When a circular dependency is detected, print the + * header first: + */ +static noinline int +print_circular_bug_header(struct lock_list *entry, unsigned int depth) +{ + struct task_struct *curr = current; + + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + if (debug_locks_silent) + return 0; + + printk("\n=======================================================\n"); + printk( "[ INFO: possible circular locking dependency detected ]\n"); + printk( "-------------------------------------------------------\n"); + printk("%s/%d is trying to acquire lock:\n", + curr->comm, curr->pid); + print_lock(check_source); + printk("\nbut task is already holding lock:\n"); + print_lock(check_target); + printk("\nwhich lock already depends on the new lock.\n\n"); + printk("\nthe existing dependency chain (in reverse order) is:\n"); + + print_circular_bug_entry(entry, depth); + + return 0; +} + +static noinline int print_circular_bug_tail(void) +{ + struct task_struct *curr = current; + struct lock_list this; + + if (debug_locks_silent) + return 0; + + this.class = check_source->class; + save_trace(&this.trace); + print_circular_bug_entry(&this, 0); + + printk("\nother info that might help us debug this:\n\n"); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +static int noinline print_infinite_recursion_bug(void) +{ + __raw_spin_unlock(&hash_lock); + DEBUG_LOCKS_WARN_ON(1); + + return 0; +} + +/* + * Prove that the dependency graph starting at <entry> can not + * lead to <target>. Print an error and return 0 if it does. + */ +static noinline int +check_noncircular(struct lock_class *source, unsigned int depth) +{ + struct lock_list *entry; + + debug_atomic_inc(&nr_cyclic_check_recursions); + if (depth > max_recursion_depth) + max_recursion_depth = depth; + if (depth >= 20) + return print_infinite_recursion_bug(); + /* + * Check this lock's dependency list: + */ + list_for_each_entry(entry, &source->locks_after, entry) { + if (entry->class == check_target->class) + return print_circular_bug_header(entry, depth+1); + debug_atomic_inc(&nr_cyclic_checks); + if (!check_noncircular(entry->class, depth+1)) + return print_circular_bug_entry(entry, depth+1); + } + return 1; +} + +static int very_verbose(struct lock_class *class) +{ +#if VERY_VERBOSE + return class_filter(class); +#endif + return 0; +} +#ifdef CONFIG_TRACE_IRQFLAGS + +/* + * Forwards and backwards subgraph searching, for the purposes of + * proving that two subgraphs can be connected by a new dependency + * without creating any illegal irq-safe -> irq-unsafe lock dependency. + */ +static enum lock_usage_bit find_usage_bit; +static struct lock_class *forwards_match, *backwards_match; + +/* + * Find a node in the forwards-direction dependency sub-graph starting + * at <source> that matches <find_usage_bit>. + * + * Return 2 if such a node exists in the subgraph, and put that node + * into <forwards_match>. + * + * Return 1 otherwise and keep <forwards_match> unchanged. + * Return 0 on error. + */ +static noinline int +find_usage_forwards(struct lock_class *source, unsigned int depth) +{ + struct lock_list *entry; + int ret; + + if (depth > max_recursion_depth) + max_recursion_depth = depth; + if (depth >= 20) + return print_infinite_recursion_bug(); + + debug_atomic_inc(&nr_find_usage_forwards_checks); + if (source->usage_mask & (1 << find_usage_bit)) { + forwards_match = source; + return 2; + } + + /* + * Check this lock's dependency list: + */ + list_for_each_entry(entry, &source->locks_after, entry) { + debug_atomic_inc(&nr_find_usage_forwards_recursions); + ret = find_usage_forwards(entry->class, depth+1); + if (ret == 2 || ret == 0) + return ret; + } + return 1; +} + +/* + * Find a node in the backwards-direction dependency sub-graph starting + * at <source> that matches <find_usage_bit>. + * + * Return 2 if such a node exists in the subgraph, and put that node + * into <backwards_match>. + * + * Return 1 otherwise and keep <backwards_match> unchanged. + * Return 0 on error. + */ +static noinline int +find_usage_backwards(struct lock_class *source, unsigned int depth) +{ + struct lock_list *entry; + int ret; + + if (depth > max_recursion_depth) + max_recursion_depth = depth; + if (depth >= 20) + return print_infinite_recursion_bug(); + + debug_atomic_inc(&nr_find_usage_backwards_checks); + if (source->usage_mask & (1 << find_usage_bit)) { + backwards_match = source; + return 2; + } + + /* + * Check this lock's dependency list: + */ + list_for_each_entry(entry, &source->locks_before, entry) { + debug_atomic_inc(&nr_find_usage_backwards_recursions); + ret = find_usage_backwards(entry->class, depth+1); + if (ret == 2 || ret == 0) + return ret; + } + return 1; +} + +static int +print_bad_irq_dependency(struct task_struct *curr, + struct held_lock *prev, + struct held_lock *next, + enum lock_usage_bit bit1, + enum lock_usage_bit bit2, + const char *irqclass) +{ + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + if (debug_locks_silent) + return 0; + + printk("\n======================================================\n"); + printk( "[ INFO: %s-safe -> %s-unsafe lock order detected ]\n", + irqclass, irqclass); + printk( "------------------------------------------------------\n"); + printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n", + curr->comm, curr->pid, + curr->hardirq_context, hardirq_count() >> HARDIRQ_SHIFT, + curr->softirq_context, softirq_count() >> SOFTIRQ_SHIFT, + curr->hardirqs_enabled, + curr->softirqs_enabled); + print_lock(next); + + printk("\nand this task is already holding:\n"); + print_lock(prev); + printk("which would create a new lock dependency:\n"); + print_lock_name(prev->class); + printk(" ->"); + print_lock_name(next->class); + printk("\n"); + + printk("\nbut this new dependency connects a %s-irq-safe lock:\n", + irqclass); + print_lock_name(backwards_match); + printk("\n... which became %s-irq-safe at:\n", irqclass); + + print_stack_trace(backwards_match->usage_traces + bit1, 1); + + printk("\nto a %s-irq-unsafe lock:\n", irqclass); + print_lock_name(forwards_match); + printk("\n... which became %s-irq-unsafe at:\n", irqclass); + printk("..."); + + print_stack_trace(forwards_match->usage_traces + bit2, 1); + + printk("\nother info that might help us debug this:\n\n"); + lockdep_print_held_locks(curr); + + printk("\nthe %s-irq-safe lock's dependencies:\n", irqclass); + print_lock_dependencies(backwards_match, 0); + + printk("\nthe %s-irq-unsafe lock's dependencies:\n", irqclass); + print_lock_dependencies(forwards_match, 0); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +static int +check_usage(struct task_struct *curr, struct held_lock *prev, + struct held_lock *next, enum lock_usage_bit bit_backwards, + enum lock_usage_bit bit_forwards, const char *irqclass) +{ + int ret; + + find_usage_bit = bit_backwards; + /* fills in <backwards_match> */ + ret = find_usage_backwards(prev->class, 0); + if (!ret || ret == 1) + return ret; + + find_usage_bit = bit_forwards; + ret = find_usage_forwards(next->class, 0); + if (!ret || ret == 1) + return ret; + /* ret == 2 */ + return print_bad_irq_dependency(curr, prev, next, + bit_backwards, bit_forwards, irqclass); +} + +#endif + +static int +print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, + struct held_lock *next) +{ + debug_locks_off(); + __raw_spin_unlock(&hash_lock); + if (debug_locks_silent) + return 0; + + printk("\n=============================================\n"); + printk( "[ INFO: possible recursive locking detected ]\n"); + printk( "---------------------------------------------\n"); + printk("%s/%d is trying to acquire lock:\n", + curr->comm, curr->pid); + print_lock(next); + printk("\nbut task is already holding lock:\n"); + print_lock(prev); + + printk("\nother info that might help us debug this:\n"); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +/* + * Check whether we are holding such a class already. + * + * (Note that this has to be done separately, because the graph cannot + * detect such classes of deadlocks.) + * + * Returns: 0 on deadlock detected, 1 on OK, 2 on recursive read + */ +static int +check_deadlock(struct task_struct *curr, struct held_lock *next, + struct lockdep_map *next_instance, int read) +{ + struct held_lock *prev; + int i; + + for (i = 0; i < curr->lockdep_depth; i++) { + prev = curr->held_locks + i; + if (prev->class != next->class) + continue; + /* + * Allow read-after-read recursion of the same + * lock class (i.e. read_lock(lock)+read_lock(lock)): + */ + if ((read == 2) && prev->read) + return 2; + return print_deadlock_bug(curr, prev, next); + } + return 1; +} + +/* + * There was a chain-cache miss, and we are about to add a new dependency + * to a previous lock. We recursively validate the following rules: + * + * - would the adding of the <prev> -> <next> dependency create a + * circular dependency in the graph? [== circular deadlock] + * + * - does the new prev->next dependency connect any hardirq-safe lock + * (in the full backwards-subgraph starting at <prev>) with any + * hardirq-unsafe lock (in the full forwards-subgraph starting at + * <next>)? [== illegal lock inversion with hardirq contexts] + * + * - does the new prev->next dependency connect any softirq-safe lock + * (in the full backwards-subgraph starting at <prev>) with any + * softirq-unsafe lock (in the full forwards-subgraph starting at + * <next>)? [== illegal lock inversion with softirq contexts] + * + * any of these scenarios could lead to a deadlock. + * + * Then if all the validations pass, we add the forwards and backwards + * dependency. + */ +static int +check_prev_add(struct task_struct *curr, struct held_lock *prev, + struct held_lock *next) +{ + struct lock_list *entry; + int ret; + + /* + * Prove that the new <prev> -> <next> dependency would not + * create a circular dependency in the graph. (We do this by + * forward-recursing into the graph starting at <next>, and + * checking whether we can reach <prev>.) + * + * We are using global variables to control the recursion, to + * keep the stackframe size of the recursive functions low: + */ + check_source = next; + check_target = prev; + if (!(check_noncircular(next->class, 0))) + return print_circular_bug_tail(); + +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * Prove that the new dependency does not connect a hardirq-safe + * lock with a hardirq-unsafe lock - to achieve this we search + * the backwards-subgraph starting at <prev>, and the + * forwards-subgraph starting at <next>: + */ + if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ, + LOCK_ENABLED_HARDIRQS, "hard")) + return 0; + + /* + * Prove that the new dependency does not connect a hardirq-safe-read + * lock with a hardirq-unsafe lock - to achieve this we search + * the backwards-subgraph starting at <prev>, and the + * forwards-subgraph starting at <next>: + */ + if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ_READ, + LOCK_ENABLED_HARDIRQS, "hard-read")) + return 0; + + /* + * Prove that the new dependency does not connect a softirq-safe + * lock with a softirq-unsafe lock - to achieve this we search + * the backwards-subgraph starting at <prev>, and the + * forwards-subgraph starting at <next>: + */ + if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ, + LOCK_ENABLED_SOFTIRQS, "soft")) + return 0; + /* + * Prove that the new dependency does not connect a softirq-safe-read + * lock with a softirq-unsafe lock - to achieve this we search + * the backwards-subgraph starting at <prev>, and the + * forwards-subgraph starting at <next>: + */ + if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ_READ, + LOCK_ENABLED_SOFTIRQS, "soft")) + return 0; +#endif + /* + * For recursive read-locks we do all the dependency checks, + * but we dont store read-triggered dependencies (only + * write-triggered dependencies). This ensures that only the + * write-side dependencies matter, and that if for example a + * write-lock never takes any other locks, then the reads are + * equivalent to a NOP. + */ + if (next->read == 2 || prev->read == 2) + return 1; + /* + * Is the <prev> -> <next> dependency already present? + * + * (this may occur even though this is a new chain: consider + * e.g. the L1 -> L2 -> L3 -> L4 and the L5 -> L1 -> L2 -> L3 + * chains - the second one will be new, but L1 already has + * L2 added to its dependency list, due to the first chain.) + */ + list_for_each_entry(entry, &prev->class->locks_after, entry) { + if (entry->class == next->class) + return 2; + } + + /* + * Ok, all validations passed, add the new lock + * to the previous lock's dependency list: + */ + ret = add_lock_to_list(prev->class, next->class, + &prev->class->locks_after, next->acquire_ip); + if (!ret) + return 0; + /* + * Return value of 2 signals 'dependency already added', + * in that case we dont have to add the backlink either. + */ + if (ret == 2) + return 2; + ret = add_lock_to_list(next->class, prev->class, + &next->class->locks_before, next->acquire_ip); + + /* + * Debugging printouts: + */ + if (verbose(prev->class) || verbose(next->class)) { + __raw_spin_unlock(&hash_lock); + printk("\n new dependency: "); + print_lock_name(prev->class); + printk(" => "); + print_lock_name(next->class); + printk("\n"); + dump_stack(); + __raw_spin_lock(&hash_lock); + } + return 1; +} + +/* + * Add the dependency to all directly-previous locks that are 'relevant'. + * The ones that are relevant are (in increasing distance from curr): + * all consecutive trylock entries and the final non-trylock entry - or + * the end of this context's lock-chain - whichever comes first. + */ +static int +check_prevs_add(struct task_struct *curr, struct held_lock *next) +{ + int depth = curr->lockdep_depth; + struct held_lock *hlock; + + /* + * Debugging checks. + * + * Depth must not be zero for a non-head lock: + */ + if (!depth) + goto out_bug; + /* + * At least two relevant locks must exist for this + * to be a head: + */ + if (curr->held_locks[depth].irq_context != + curr->held_locks[depth-1].irq_context) + goto out_bug; + + for (;;) { + hlock = curr->held_locks + depth-1; + /* + * Only non-recursive-read entries get new dependencies + * added: + */ + if (hlock->read != 2) { + check_prev_add(curr, hlock, next); + /* + * Stop after the first non-trylock entry, + * as non-trylock entries have added their + * own direct dependencies already, so this + * lock is connected to them indirectly: + */ + if (!hlock->trylock) + break; + } + depth--; + /* + * End of lock-stack? + */ + if (!depth) + break; + /* + * Stop the search if we cross into another context: + */ + if (curr->held_locks[depth].irq_context != + curr->held_locks[depth-1].irq_context) + break; + } + return 1; +out_bug: + __raw_spin_unlock(&hash_lock); + DEBUG_LOCKS_WARN_ON(1); + + return 0; +} + + +/* + * Is this the address of a static object: + */ +static int static_obj(void *obj) +{ + unsigned long start = (unsigned long) &_stext, + end = (unsigned long) &_end, + addr = (unsigned long) obj; +#ifdef CONFIG_SMP + int i; +#endif + + /* + * static variable? + */ + if ((addr >= start) && (addr < end)) + return 1; + +#ifdef CONFIG_SMP + /* + * percpu var? + */ + for_each_possible_cpu(i) { + start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); + end = (unsigned long) &__per_cpu_end + per_cpu_offset(i); + + if ((addr >= start) && (addr < end)) + return 1; + } +#endif + + /* + * module var? + */ + return is_module_address(addr); +} + +/* + * To make lock name printouts unique, we calculate a unique + * class->name_version generation counter: + */ +static int count_matching_names(struct lock_class *new_class) +{ + struct lock_class *class; + int count = 0; + + if (!new_class->name) + return 0; + + list_for_each_entry(class, &all_lock_classes, lock_entry) { + if (new_class->key - new_class->subclass == class->key) + return class->name_version; + if (class->name && !strcmp(class->name, new_class->name)) + count = max(count, class->name_version); + } + + return count + 1; +} + +extern void __error_too_big_MAX_LOCKDEP_SUBCLASSES(void); + +/* + * Register a lock's class in the hash-table, if the class is not present + * yet. Otherwise we look it up. We cache the result in the lock object + * itself, so actual lookup of the hash should be once per lock object. + */ +static inline struct lock_class * +register_lock_class(struct lockdep_map *lock, unsigned int subclass) +{ + struct lockdep_subclass_key *key; + struct list_head *hash_head; + struct lock_class *class; + +#ifdef CONFIG_DEBUG_LOCKDEP + /* + * If the architecture calls into lockdep before initializing + * the hashes then we'll warn about it later. (we cannot printk + * right now) + */ + if (unlikely(!lockdep_initialized)) { + lockdep_init(); + lockdep_init_error = 1; + } +#endif + + /* + * Static locks do not have their class-keys yet - for them the key + * is the lock object itself: + */ + if (unlikely(!lock->key)) + lock->key = (void *)lock; + + /* + * NOTE: the class-key must be unique. For dynamic locks, a static + * lock_class_key variable is passed in through the mutex_init() + * (or spin_lock_init()) call - which acts as the key. For static + * locks we use the lock object itself as the key. + */ + if (sizeof(struct lock_class_key) > sizeof(struct lock_class)) + __error_too_big_MAX_LOCKDEP_SUBCLASSES(); + + key = lock->key->subkeys + subclass; + + hash_head = classhashentry(key); + + /* + * We can walk the hash lockfree, because the hash only + * grows, and we are careful when adding entries to the end: + */ + list_for_each_entry(class, hash_head, hash_entry) + if (class->key == key) + goto out_set; + + /* + * Debug-check: all keys must be persistent! + */ + if (!static_obj(lock->key)) { + debug_locks_off(); + printk("INFO: trying to register non-static key.\n"); + printk("the code is fine but needs lockdep annotation.\n"); + printk("turning off the locking correctness validator.\n"); + dump_stack(); + + return NULL; + } + + __raw_spin_lock(&hash_lock); + /* + * We have to do the hash-walk again, to avoid races + * with another CPU: + */ + list_for_each_entry(class, hash_head, hash_entry) + if (class->key == key) + goto out_unlock_set; + /* + * Allocate a new key from the static array, and add it to + * the hash: + */ + if (nr_lock_classes >= MAX_LOCKDEP_KEYS) { + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + printk("BUG: MAX_LOCKDEP_KEYS too low!\n"); + printk("turning off the locking correctness validator.\n"); + return NULL; + } + class = lock_classes + nr_lock_classes++; + debug_atomic_inc(&nr_unused_locks); + class->key = key; + class->name = lock->name; + class->subclass = subclass; + INIT_LIST_HEAD(&class->lock_entry); + INIT_LIST_HEAD(&class->locks_before); + INIT_LIST_HEAD(&class->locks_after); + class->name_version = count_matching_names(class); + /* + * We use RCU's safe list-add method to make + * parallel walking of the hash-list safe: + */ + list_add_tail_rcu(&class->hash_entry, hash_head); + + if (verbose(class)) { + __raw_spin_unlock(&hash_lock); + printk("\nnew class %p: %s", class->key, class->name); + if (class->name_version > 1) + printk("#%d", class->name_version); + printk("\n"); + dump_stack(); + __raw_spin_lock(&hash_lock); + } +out_unlock_set: + __raw_spin_unlock(&hash_lock); + +out_set: + lock->class[subclass] = class; + + DEBUG_LOCKS_WARN_ON(class->subclass != subclass); + + return class; +} + +/* + * Look up a dependency chain. If the key is not present yet then + * add it and return 0 - in this case the new dependency chain is + * validated. If the key is already hashed, return 1. + */ +static inline int lookup_chain_cache(u64 chain_key) +{ + struct list_head *hash_head = chainhashentry(chain_key); + struct lock_chain *chain; + + DEBUG_LOCKS_WARN_ON(!irqs_disabled()); + /* + * We can walk it lock-free, because entries only get added + * to the hash: + */ + list_for_each_entry(chain, hash_head, entry) { + if (chain->chain_key == chain_key) { +cache_hit: + debug_atomic_inc(&chain_lookup_hits); + /* + * In the debugging case, force redundant checking + * by returning 1: + */ +#ifdef CONFIG_DEBUG_LOCKDEP + __raw_spin_lock(&hash_lock); + return 1; +#endif + return 0; + } + } + /* + * Allocate a new chain entry from the static array, and add + * it to the hash: + */ + __raw_spin_lock(&hash_lock); + /* + * We have to walk the chain again locked - to avoid duplicates: + */ + list_for_each_entry(chain, hash_head, entry) { + if (chain->chain_key == chain_key) { + __raw_spin_unlock(&hash_lock); + goto cache_hit; + } + } + if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) { + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + printk("BUG: MAX_LOCKDEP_CHAINS too low!\n"); + printk("turning off the locking correctness validator.\n"); + return 0; + } + chain = lock_chains + nr_lock_chains++; + chain->chain_key = chain_key; + list_add_tail_rcu(&chain->entry, hash_head); + debug_atomic_inc(&chain_lookup_misses); +#ifdef CONFIG_TRACE_IRQFLAGS + if (current->hardirq_context) + nr_hardirq_chains++; + else { + if (current->softirq_context) + nr_softirq_chains++; + else + nr_process_chains++; + } +#else + nr_process_chains++; +#endif + + return 1; +} + +/* + * We are building curr_chain_key incrementally, so double-check + * it from scratch, to make sure that it's done correctly: + */ +static void check_chain_key(struct task_struct *curr) +{ +#ifdef CONFIG_DEBUG_LOCKDEP + struct held_lock *hlock, *prev_hlock = NULL; + unsigned int i, id; + u64 chain_key = 0; + + for (i = 0; i < curr->lockdep_depth; i++) { + hlock = curr->held_locks + i; + if (chain_key != hlock->prev_chain_key) { + debug_locks_off(); + printk("hm#1, depth: %u [%u], %016Lx != %016Lx\n", + curr->lockdep_depth, i, + (unsigned long long)chain_key, + (unsigned long long)hlock->prev_chain_key); + WARN_ON(1); + return; + } + id = hlock->class - lock_classes; + DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS); + if (prev_hlock && (prev_hlock->irq_context != + hlock->irq_context)) + chain_key = 0; + chain_key = iterate_chain_key(chain_key, id); + prev_hlock = hlock; + } + if (chain_key != curr->curr_chain_key) { + debug_locks_off(); + printk("hm#2, depth: %u [%u], %016Lx != %016Lx\n", + curr->lockdep_depth, i, + (unsigned long long)chain_key, + (unsigned long long)curr->curr_chain_key); + WARN_ON(1); + } +#endif +} + +#ifdef CONFIG_TRACE_IRQFLAGS + +/* + * print irq inversion bug: + */ +static int +print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other, + struct held_lock *this, int forwards, + const char *irqclass) +{ + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + if (debug_locks_silent) + return 0; + + printk("\n=========================================================\n"); + printk( "[ INFO: possible irq lock inversion dependency detected ]\n"); + printk( "---------------------------------------------------------\n"); + printk("%s/%d just changed the state of lock:\n", + curr->comm, curr->pid); + print_lock(this); + if (forwards) + printk("but this lock took another, %s-irq-unsafe lock in the past:\n", irqclass); + else + printk("but this lock was taken by another, %s-irq-safe lock in the past:\n", irqclass); + print_lock_name(other); + printk("\n\nand interrupts could create inverse lock ordering between them.\n\n"); + + printk("\nother info that might help us debug this:\n"); + lockdep_print_held_locks(curr); + + printk("\nthe first lock's dependencies:\n"); + print_lock_dependencies(this->class, 0); + + printk("\nthe second lock's dependencies:\n"); + print_lock_dependencies(other, 0); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +/* + * Prove that in the forwards-direction subgraph starting at <this> + * there is no lock matching <mask>: + */ +static int +check_usage_forwards(struct task_struct *curr, struct held_lock *this, + enum lock_usage_bit bit, const char *irqclass) +{ + int ret; + + find_usage_bit = bit; + /* fills in <forwards_match> */ + ret = find_usage_forwards(this->class, 0); + if (!ret || ret == 1) + return ret; + + return print_irq_inversion_bug(curr, forwards_match, this, 1, irqclass); +} + +/* + * Prove that in the backwards-direction subgraph starting at <this> + * there is no lock matching <mask>: + */ +static int +check_usage_backwards(struct task_struct *curr, struct held_lock *this, + enum lock_usage_bit bit, const char *irqclass) +{ + int ret; + + find_usage_bit = bit; + /* fills in <backwards_match> */ + ret = find_usage_backwards(this->class, 0); + if (!ret || ret == 1) + return ret; + + return print_irq_inversion_bug(curr, backwards_match, this, 0, irqclass); +} + +static inline void print_irqtrace_events(struct task_struct *curr) +{ + printk("irq event stamp: %u\n", curr->irq_events); + printk("hardirqs last enabled at (%u): ", curr->hardirq_enable_event); + print_ip_sym(curr->hardirq_enable_ip); + printk("hardirqs last disabled at (%u): ", curr->hardirq_disable_event); + print_ip_sym(curr->hardirq_disable_ip); + printk("softirqs last enabled at (%u): ", curr->softirq_enable_event); + print_ip_sym(curr->softirq_enable_ip); + printk("softirqs last disabled at (%u): ", curr->softirq_disable_event); + print_ip_sym(curr->softirq_disable_ip); +} + +#else +static inline void print_irqtrace_events(struct task_struct *curr) +{ +} +#endif + +static int +print_usage_bug(struct task_struct *curr, struct held_lock *this, + enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit) +{ + __raw_spin_unlock(&hash_lock); + debug_locks_off(); + if (debug_locks_silent) + return 0; + + printk("\n=================================\n"); + printk( "[ INFO: inconsistent lock state ]\n"); + printk( "---------------------------------\n"); + + printk("inconsistent {%s} -> {%s} usage.\n", + usage_str[prev_bit], usage_str[new_bit]); + + printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] takes:\n", + curr->comm, curr->pid, + trace_hardirq_context(curr), hardirq_count() >> HARDIRQ_SHIFT, + trace_softirq_context(curr), softirq_count() >> SOFTIRQ_SHIFT, + trace_hardirqs_enabled(curr), + trace_softirqs_enabled(curr)); + print_lock(this); + + printk("{%s} state was registered at:\n", usage_str[prev_bit]); + print_stack_trace(this->class->usage_traces + prev_bit, 1); + + print_irqtrace_events(curr); + printk("\nother info that might help us debug this:\n"); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +/* + * Print out an error if an invalid bit is set: + */ +static inline int +valid_state(struct task_struct *curr, struct held_lock *this, + enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit) +{ + if (unlikely(this->class->usage_mask & (1 << bad_bit))) + return print_usage_bug(curr, this, bad_bit, new_bit); + return 1; +} + +#define STRICT_READ_CHECKS 1 + +/* + * Mark a lock with a usage bit, and validate the state transition: + */ +static int mark_lock(struct task_struct *curr, struct held_lock *this, + enum lock_usage_bit new_bit, unsigned long ip) +{ + unsigned int new_mask = 1 << new_bit, ret = 1; + + /* + * If already set then do not dirty the cacheline, + * nor do any checks: + */ + if (likely(this->class->usage_mask & new_mask)) + return 1; + + __raw_spin_lock(&hash_lock); + /* + * Make sure we didnt race: + */ + if (unlikely(this->class->usage_mask & new_mask)) { + __raw_spin_unlock(&hash_lock); + return 1; + } + + this->class->usage_mask |= new_mask; + +#ifdef CONFIG_TRACE_IRQFLAGS + if (new_bit == LOCK_ENABLED_HARDIRQS || + new_bit == LOCK_ENABLED_HARDIRQS_READ) + ip = curr->hardirq_enable_ip; + else if (new_bit == LOCK_ENABLED_SOFTIRQS || + new_bit == LOCK_ENABLED_SOFTIRQS_READ) + ip = curr->softirq_enable_ip; +#endif + if (!save_trace(this->class->usage_traces + new_bit)) + return 0; + + switch (new_bit) { +#ifdef CONFIG_TRACE_IRQFLAGS + case LOCK_USED_IN_HARDIRQ: + if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS)) + return 0; + if (!valid_state(curr, this, new_bit, + LOCK_ENABLED_HARDIRQS_READ)) + return 0; + /* + * just marked it hardirq-safe, check that this lock + * took no hardirq-unsafe lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_HARDIRQS, "hard")) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it hardirq-safe, check that this lock + * took no hardirq-unsafe-read lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_HARDIRQS_READ, "hard-read")) + return 0; +#endif + if (hardirq_verbose(this->class)) + ret = 2; + break; + case LOCK_USED_IN_SOFTIRQ: + if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS)) + return 0; + if (!valid_state(curr, this, new_bit, + LOCK_ENABLED_SOFTIRQS_READ)) + return 0; + /* + * just marked it softirq-safe, check that this lock + * took no softirq-unsafe lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_SOFTIRQS, "soft")) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it softirq-safe, check that this lock + * took no softirq-unsafe-read lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_SOFTIRQS_READ, "soft-read")) + return 0; +#endif + if (softirq_verbose(this->class)) + ret = 2; + break; + case LOCK_USED_IN_HARDIRQ_READ: + if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS)) + return 0; + /* + * just marked it hardirq-read-safe, check that this lock + * took no hardirq-unsafe lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_HARDIRQS, "hard")) + return 0; + if (hardirq_verbose(this->class)) + ret = 2; + break; + case LOCK_USED_IN_SOFTIRQ_READ: + if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS)) + return 0; + /* + * just marked it softirq-read-safe, check that this lock + * took no softirq-unsafe lock in the past: + */ + if (!check_usage_forwards(curr, this, + LOCK_ENABLED_SOFTIRQS, "soft")) + return 0; + if (softirq_verbose(this->class)) + ret = 2; + break; + case LOCK_ENABLED_HARDIRQS: + if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ)) + return 0; + if (!valid_state(curr, this, new_bit, + LOCK_USED_IN_HARDIRQ_READ)) + return 0; + /* + * just marked it hardirq-unsafe, check that no hardirq-safe + * lock in the system ever took it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_HARDIRQ, "hard")) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it hardirq-unsafe, check that no + * hardirq-safe-read lock in the system ever took + * it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_HARDIRQ_READ, "hard-read")) + return 0; +#endif + if (hardirq_verbose(this->class)) + ret = 2; + break; + case LOCK_ENABLED_SOFTIRQS: + if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ)) + return 0; + if (!valid_state(curr, this, new_bit, + LOCK_USED_IN_SOFTIRQ_READ)) + return 0; + /* + * just marked it softirq-unsafe, check that no softirq-safe + * lock in the system ever took it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_SOFTIRQ, "soft")) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it softirq-unsafe, check that no + * softirq-safe-read lock in the system ever took + * it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_SOFTIRQ_READ, "soft-read")) + return 0; +#endif + if (softirq_verbose(this->class)) + ret = 2; + break; + case LOCK_ENABLED_HARDIRQS_READ: + if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ)) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it hardirq-read-unsafe, check that no + * hardirq-safe lock in the system ever took it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_HARDIRQ, "hard")) + return 0; +#endif + if (hardirq_verbose(this->class)) + ret = 2; + break; + case LOCK_ENABLED_SOFTIRQS_READ: + if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ)) + return 0; +#if STRICT_READ_CHECKS + /* + * just marked it softirq-read-unsafe, check that no + * softirq-safe lock in the system ever took it in the past: + */ + if (!check_usage_backwards(curr, this, + LOCK_USED_IN_SOFTIRQ, "soft")) + return 0; +#endif + if (softirq_verbose(this->class)) + ret = 2; + break; +#endif + case LOCK_USED: + /* + * Add it to the global list of classes: + */ + list_add_tail_rcu(&this->class->lock_entry, &all_lock_classes); + debug_atomic_dec(&nr_unused_locks); + break; + default: + debug_locks_off(); + WARN_ON(1); + return 0; + } + + __raw_spin_unlock(&hash_lock); + + /* + * We must printk outside of the hash_lock: + */ + if (ret == 2) { + printk("\nmarked lock as {%s}:\n", usage_str[new_bit]); + print_lock(this); + print_irqtrace_events(curr); + dump_stack(); + } + + return ret; +} + +#ifdef CONFIG_TRACE_IRQFLAGS +/* + * Mark all held locks with a usage bit: + */ +static int +mark_held_locks(struct task_struct *curr, int hardirq, unsigned long ip) +{ + enum lock_usage_bit usage_bit; + struct held_lock *hlock; + int i; + + for (i = 0; i < curr->lockdep_depth; i++) { + hlock = curr->held_locks + i; + + if (hardirq) { + if (hlock->read) + usage_bit = LOCK_ENABLED_HARDIRQS_READ; + else + usage_bit = LOCK_ENABLED_HARDIRQS; + } else { + if (hlock->read) + usage_bit = LOCK_ENABLED_SOFTIRQS_READ; + else + usage_bit = LOCK_ENABLED_SOFTIRQS; + } + if (!mark_lock(curr, hlock, usage_bit, ip)) + return 0; + } + + return 1; +} + +/* + * Debugging helper: via this flag we know that we are in + * 'early bootup code', and will warn about any invalid irqs-on event: + */ +static int early_boot_irqs_enabled; + +void early_boot_irqs_off(void) +{ + early_boot_irqs_enabled = 0; +} + +void early_boot_irqs_on(void) +{ + early_boot_irqs_enabled = 1; +} + +/* + * Hardirqs will be enabled: + */ +void trace_hardirqs_on(void) +{ + struct task_struct *curr = current; + unsigned long ip; + + if (unlikely(!debug_locks || current->lockdep_recursion)) + return; + + if (DEBUG_LOCKS_WARN_ON(unlikely(!early_boot_irqs_enabled))) + return; + + if (unlikely(curr->hardirqs_enabled)) { + debug_atomic_inc(&redundant_hardirqs_on); + return; + } + /* we'll do an OFF -> ON transition: */ + curr->hardirqs_enabled = 1; + ip = (unsigned long) __builtin_return_address(0); + + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + if (DEBUG_LOCKS_WARN_ON(current->hardirq_context)) + return; + /* + * We are going to turn hardirqs on, so set the + * usage bit for all held locks: + */ + if (!mark_held_locks(curr, 1, ip)) + return; + /* + * If we have softirqs enabled, then set the usage + * bit for all held locks. (disabled hardirqs prevented + * this bit from being set before) + */ + if (curr->softirqs_enabled) + if (!mark_held_locks(curr, 0, ip)) + return; + + curr->hardirq_enable_ip = ip; + curr->hardirq_enable_event = ++curr->irq_events; + debug_atomic_inc(&hardirqs_on_events); +} + +EXPORT_SYMBOL(trace_hardirqs_on); + +/* + * Hardirqs were disabled: + */ +void trace_hardirqs_off(void) +{ + struct task_struct *curr = current; + + if (unlikely(!debug_locks || current->lockdep_recursion)) + return; + + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + + if (curr->hardirqs_enabled) { + /* + * We have done an ON -> OFF transition: + */ + curr->hardirqs_enabled = 0; + curr->hardirq_disable_ip = _RET_IP_; + curr->hardirq_disable_event = ++curr->irq_events; + debug_atomic_inc(&hardirqs_off_events); + } else + debug_atomic_inc(&redundant_hardirqs_off); +} + +EXPORT_SYMBOL(trace_hardirqs_off); + +/* + * Softirqs will be enabled: + */ +void trace_softirqs_on(unsigned long ip) +{ + struct task_struct *curr = current; + + if (unlikely(!debug_locks)) + return; + + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + + if (curr->softirqs_enabled) { + debug_atomic_inc(&redundant_softirqs_on); + return; + } + + /* + * We'll do an OFF -> ON transition: + */ + curr->softirqs_enabled = 1; + curr->softirq_enable_ip = ip; + curr->softirq_enable_event = ++curr->irq_events; + debug_atomic_inc(&softirqs_on_events); + /* + * We are going to turn softirqs on, so set the + * usage bit for all held locks, if hardirqs are + * enabled too: + */ + if (curr->hardirqs_enabled) + mark_held_locks(curr, 0, ip); +} + +/* + * Softirqs were disabled: + */ +void trace_softirqs_off(unsigned long ip) +{ + struct task_struct *curr = current; + + if (unlikely(!debug_locks)) + return; + + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + + if (curr->softirqs_enabled) { + /* + * We have done an ON -> OFF transition: + */ + curr->softirqs_enabled = 0; + curr->softirq_disable_ip = ip; + curr->softirq_disable_event = ++curr->irq_events; + debug_atomic_inc(&softirqs_off_events); + DEBUG_LOCKS_WARN_ON(!softirq_count()); + } else + debug_atomic_inc(&redundant_softirqs_off); +} + +#endif + +/* + * Initialize a lock instance's lock-class mapping info: + */ +void lockdep_init_map(struct lockdep_map *lock, const char *name, + struct lock_class_key *key) +{ + if (unlikely(!debug_locks)) + return; + + if (DEBUG_LOCKS_WARN_ON(!key)) + return; + if (DEBUG_LOCKS_WARN_ON(!name)) + return; + /* + * Sanity check, the lock-class key must be persistent: + */ + if (!static_obj(key)) { + printk("BUG: key %p not in .data!\n", key); + DEBUG_LOCKS_WARN_ON(1); + return; + } + lock->name = name; + lock->key = key; + memset(lock->class, 0, sizeof(lock->class[0])*MAX_LOCKDEP_SUBCLASSES); +} + +EXPORT_SYMBOL_GPL(lockdep_init_map); + +/* + * This gets called for every mutex_lock*()/spin_lock*() operation. + * We maintain the dependency maps and validate the locking attempt: + */ +static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, + int trylock, int read, int check, int hardirqs_off, + unsigned long ip) +{ + struct task_struct *curr = current; + struct held_lock *hlock; + struct lock_class *class; + unsigned int depth, id; + int chain_head = 0; + u64 chain_key; + + if (unlikely(!debug_locks)) + return 0; + + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return 0; + + if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) { + debug_locks_off(); + printk("BUG: MAX_LOCKDEP_SUBCLASSES too low!\n"); + printk("turning off the locking correctness validator.\n"); + return 0; + } + + class = lock->class[subclass]; + /* not cached yet? */ + if (unlikely(!class)) { + class = register_lock_class(lock, subclass); + if (!class) + return 0; + } + debug_atomic_inc((atomic_t *)&class->ops); + if (very_verbose(class)) { + printk("\nacquire class [%p] %s", class->key, class->name); + if (class->name_version > 1) + printk("#%d", class->name_version); + printk("\n"); + dump_stack(); + } + + /* + * Add the lock to the list of currently held locks. + * (we dont increase the depth just yet, up until the + * dependency checks are done) + */ + depth = curr->lockdep_depth; + if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH)) + return 0; + + hlock = curr->held_locks + depth; + + hlock->class = class; + hlock->acquire_ip = ip; + hlock->instance = lock; + hlock->trylock = trylock; + hlock->read = read; + hlock->check = check; + hlock->hardirqs_off = hardirqs_off; + + if (check != 2) + goto out_calc_hash; +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * If non-trylock use in a hardirq or softirq context, then + * mark the lock as used in these contexts: + */ + if (!trylock) { + if (read) { + if (curr->hardirq_context) + if (!mark_lock(curr, hlock, + LOCK_USED_IN_HARDIRQ_READ, ip)) + return 0; + if (curr->softirq_context) + if (!mark_lock(curr, hlock, + LOCK_USED_IN_SOFTIRQ_READ, ip)) + return 0; + } else { + if (curr->hardirq_context) + if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ, ip)) + return 0; + if (curr->softirq_context) + if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ, ip)) + return 0; + } + } + if (!hardirqs_off) { + if (read) { + if (!mark_lock(curr, hlock, + LOCK_ENABLED_HARDIRQS_READ, ip)) + return 0; + if (curr->softirqs_enabled) + if (!mark_lock(curr, hlock, + LOCK_ENABLED_SOFTIRQS_READ, ip)) + return 0; + } else { + if (!mark_lock(curr, hlock, + LOCK_ENABLED_HARDIRQS, ip)) + return 0; + if (curr->softirqs_enabled) + if (!mark_lock(curr, hlock, + LOCK_ENABLED_SOFTIRQS, ip)) + return 0; + } + } +#endif + /* mark it as used: */ + if (!mark_lock(curr, hlock, LOCK_USED, ip)) + return 0; +out_calc_hash: + /* + * Calculate the chain hash: it's the combined has of all the + * lock keys along the dependency chain. We save the hash value + * at every step so that we can get the current hash easily + * after unlock. The chain hash is then used to cache dependency + * results. + * + * The 'key ID' is what is the most compact key value to drive + * the hash, not class->key. + */ + id = class - lock_classes; + if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS)) + return 0; + + chain_key = curr->curr_chain_key; + if (!depth) { + if (DEBUG_LOCKS_WARN_ON(chain_key != 0)) + return 0; + chain_head = 1; + } + + hlock->prev_chain_key = chain_key; + +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * Keep track of points where we cross into an interrupt context: + */ + hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) + + curr->softirq_context; + if (depth) { + struct held_lock *prev_hlock; + + prev_hlock = curr->held_locks + depth-1; + /* + * If we cross into another context, reset the + * hash key (this also prevents the checking and the + * adding of the dependency to 'prev'): + */ + if (prev_hlock->irq_context != hlock->irq_context) { + chain_key = 0; + chain_head = 1; + } + } +#endif + chain_key = iterate_chain_key(chain_key, id); + curr->curr_chain_key = chain_key; + + /* + * Trylock needs to maintain the stack of held locks, but it + * does not add new dependencies, because trylock can be done + * in any order. + * + * We look up the chain_key and do the O(N^2) check and update of + * the dependencies only if this is a new dependency chain. + * (If lookup_chain_cache() returns with 1 it acquires + * hash_lock for us) + */ + if (!trylock && (check == 2) && lookup_chain_cache(chain_key)) { + /* + * Check whether last held lock: + * + * - is irq-safe, if this lock is irq-unsafe + * - is softirq-safe, if this lock is hardirq-unsafe + * + * And check whether the new lock's dependency graph + * could lead back to the previous lock. + * + * any of these scenarios could lead to a deadlock. If + * All validations + */ + int ret = check_deadlock(curr, hlock, lock, read); + + if (!ret) + return 0; + /* + * Mark recursive read, as we jump over it when + * building dependencies (just like we jump over + * trylock entries): + */ + if (ret == 2) + hlock->read = 2; + /* + * Add dependency only if this lock is not the head + * of the chain, and if it's not a secondary read-lock: + */ + if (!chain_head && ret != 2) + if (!check_prevs_add(curr, hlock)) + return 0; + __raw_spin_unlock(&hash_lock); + } + curr->lockdep_depth++; + check_chain_key(curr); + if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) { + debug_locks_off(); + printk("BUG: MAX_LOCK_DEPTH too low!\n"); + printk("turning off the locking correctness validator.\n"); + return 0; + } + if (unlikely(curr->lockdep_depth > max_lockdep_depth)) + max_lockdep_depth = curr->lockdep_depth; + + return 1; +} + +static int +print_unlock_inbalance_bug(struct task_struct *curr, struct lockdep_map *lock, + unsigned long ip) +{ + if (!debug_locks_off()) + return 0; + if (debug_locks_silent) + return 0; + + printk("\n=====================================\n"); + printk( "[ BUG: bad unlock balance detected! ]\n"); + printk( "-------------------------------------\n"); + printk("%s/%d is trying to release lock (", + curr->comm, curr->pid); + print_lockdep_cache(lock); + printk(") at:\n"); + print_ip_sym(ip); + printk("but there are no more locks to release!\n"); + printk("\nother info that might help us debug this:\n"); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); + + return 0; +} + +/* + * Common debugging checks for both nested and non-nested unlock: + */ +static int check_unlock(struct task_struct *curr, struct lockdep_map *lock, + unsigned long ip) +{ + if (unlikely(!debug_locks)) + return 0; + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return 0; + + if (curr->lockdep_depth <= 0) + return print_unlock_inbalance_bug(curr, lock, ip); + + return 1; +} + +/* + * Remove the lock to the list of currently held locks in a + * potentially non-nested (out of order) manner. This is a + * relatively rare operation, as all the unlock APIs default + * to nested mode (which uses lock_release()): + */ +static int +lock_release_non_nested(struct task_struct *curr, + struct lockdep_map *lock, unsigned long ip) +{ + struct held_lock *hlock, *prev_hlock; + unsigned int depth; + int i; + + /* + * Check whether the lock exists in the current stack + * of held locks: + */ + depth = curr->lockdep_depth; + if (DEBUG_LOCKS_WARN_ON(!depth)) + return 0; + + prev_hlock = NULL; + for (i = depth-1; i >= 0; i--) { + hlock = curr->held_locks + i; + /* + * We must not cross into another context: + */ + if (prev_hlock && prev_hlock->irq_context != hlock->irq_context) + break; + if (hlock->instance == lock) + goto found_it; + prev_hlock = hlock; + } + return print_unlock_inbalance_bug(curr, lock, ip); + +found_it: + /* + * We have the right lock to unlock, 'hlock' points to it. + * Now we remove it from the stack, and add back the other + * entries (if any), recalculating the hash along the way: + */ + curr->lockdep_depth = i; + curr->curr_chain_key = hlock->prev_chain_key; + + for (i++; i < depth; i++) { + hlock = curr->held_locks + i; + if (!__lock_acquire(hlock->instance, + hlock->class->subclass, hlock->trylock, + hlock->read, hlock->check, hlock->hardirqs_off, + hlock->acquire_ip)) + return 0; + } + + if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1)) + return 0; + return 1; +} + +/* + * Remove the lock to the list of currently held locks - this gets + * called on mutex_unlock()/spin_unlock*() (or on a failed + * mutex_lock_interruptible()). This is done for unlocks that nest + * perfectly. (i.e. the current top of the lock-stack is unlocked) + */ +static int lock_release_nested(struct task_struct *curr, + struct lockdep_map *lock, unsigned long ip) +{ + struct held_lock *hlock; + unsigned int depth; + + /* + * Pop off the top of the lock stack: + */ + depth = curr->lockdep_depth - 1; + hlock = curr->held_locks + depth; + + /* + * Is the unlock non-nested: + */ + if (hlock->instance != lock) + return lock_release_non_nested(curr, lock, ip); + curr->lockdep_depth--; + + if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0))) + return 0; + + curr->curr_chain_key = hlock->prev_chain_key; + +#ifdef CONFIG_DEBUG_LOCKDEP + hlock->prev_chain_key = 0; + hlock->class = NULL; + hlock->acquire_ip = 0; + hlock->irq_context = 0; +#endif + return 1; +} + +/* + * Remove the lock to the list of currently held locks - this gets + * called on mutex_unlock()/spin_unlock*() (or on a failed + * mutex_lock_interruptible()). This is done for unlocks that nest + * perfectly. (i.e. the current top of the lock-stack is unlocked) + */ +static void +__lock_release(struct lockdep_map *lock, int nested, unsigned long ip) +{ + struct task_struct *curr = current; + + if (!check_unlock(curr, lock, ip)) + return; + + if (nested) { + if (!lock_release_nested(curr, lock, ip)) + return; + } else { + if (!lock_release_non_nested(curr, lock, ip)) + return; + } + + check_chain_key(curr); +} + +/* + * Check whether we follow the irq-flags state precisely: + */ +static void check_flags(unsigned long flags) +{ +#if defined(CONFIG_DEBUG_LOCKDEP) && defined(CONFIG_TRACE_IRQFLAGS) + if (!debug_locks) + return; + + if (irqs_disabled_flags(flags)) + DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled); + else + DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled); + + /* + * We dont accurately track softirq state in e.g. + * hardirq contexts (such as on 4KSTACKS), so only + * check if not in hardirq contexts: + */ + if (!hardirq_count()) { + if (softirq_count()) + DEBUG_LOCKS_WARN_ON(current->softirqs_enabled); + else + DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); + } + + if (!debug_locks) + print_irqtrace_events(current); +#endif +} + +/* + * We are not always called with irqs disabled - do that here, + * and also avoid lockdep recursion: + */ +void lock_acquire(struct lockdep_map *lock, unsigned int subclass, + int trylock, int read, int check, unsigned long ip) +{ + unsigned long flags; + + if (unlikely(current->lockdep_recursion)) + return; + + raw_local_irq_save(flags); + check_flags(flags); + + current->lockdep_recursion = 1; + __lock_acquire(lock, subclass, trylock, read, check, + irqs_disabled_flags(flags), ip); + current->lockdep_recursion = 0; + raw_local_irq_restore(flags); +} + +EXPORT_SYMBOL_GPL(lock_acquire); + +void lock_release(struct lockdep_map *lock, int nested, unsigned long ip) +{ + unsigned long flags; + + if (unlikely(current->lockdep_recursion)) + return; + + raw_local_irq_save(flags); + check_flags(flags); + current->lockdep_recursion = 1; + __lock_release(lock, nested, ip); + current->lockdep_recursion = 0; + raw_local_irq_restore(flags); +} + +EXPORT_SYMBOL_GPL(lock_release); + +/* + * Used by the testsuite, sanitize the validator state + * after a simulated failure: + */ + +void lockdep_reset(void) +{ + unsigned long flags; + + raw_local_irq_save(flags); + current->curr_chain_key = 0; + current->lockdep_depth = 0; + current->lockdep_recursion = 0; + memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock)); + nr_hardirq_chains = 0; + nr_softirq_chains = 0; + nr_process_chains = 0; + debug_locks = 1; + raw_local_irq_restore(flags); +} + +static void zap_class(struct lock_class *class) +{ + int i; + + /* + * Remove all dependencies this lock is + * involved in: + */ + for (i = 0; i < nr_list_entries; i++) { + if (list_entries[i].class == class) + list_del_rcu(&list_entries[i].entry); + } + /* + * Unhash the class and remove it from the all_lock_classes list: + */ + list_del_rcu(&class->hash_entry); + list_del_rcu(&class->lock_entry); + +} + +static inline int within(void *addr, void *start, unsigned long size) +{ + return addr >= start && addr < start + size; +} + +void lockdep_free_key_range(void *start, unsigned long size) +{ + struct lock_class *class, *next; + struct list_head *head; + unsigned long flags; + int i; + + raw_local_irq_save(flags); + __raw_spin_lock(&hash_lock); + + /* + * Unhash all classes that were created by this module: + */ + for (i = 0; i < CLASSHASH_SIZE; i++) { + head = classhash_table + i; + if (list_empty(head)) + continue; + list_for_each_entry_safe(class, next, head, hash_entry) + if (within(class->key, start, size)) + zap_class(class); + } + + __raw_spin_unlock(&hash_lock); + raw_local_irq_restore(flags); +} + +void lockdep_reset_lock(struct lockdep_map *lock) +{ + struct lock_class *class, *next, *entry; + struct list_head *head; + unsigned long flags; + int i, j; + + raw_local_irq_save(flags); + __raw_spin_lock(&hash_lock); + + /* + * Remove all classes this lock has: + */ + for (i = 0; i < CLASSHASH_SIZE; i++) { + head = classhash_table + i; + if (list_empty(head)) + continue; + list_for_each_entry_safe(class, next, head, hash_entry) { + for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) { + entry = lock->class[j]; + if (class == entry) { + zap_class(class); + lock->class[j] = NULL; + break; + } + } + } + } + + /* + * Debug check: in the end all mapped classes should + * be gone. + */ + for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) { + entry = lock->class[j]; + if (!entry) + continue; + __raw_spin_unlock(&hash_lock); + DEBUG_LOCKS_WARN_ON(1); + raw_local_irq_restore(flags); + return; + } + + __raw_spin_unlock(&hash_lock); + raw_local_irq_restore(flags); +} + +void __init lockdep_init(void) +{ + int i; + + /* + * Some architectures have their own start_kernel() + * code which calls lockdep_init(), while we also + * call lockdep_init() from the start_kernel() itself, + * and we want to initialize the hashes only once: + */ + if (lockdep_initialized) + return; + + for (i = 0; i < CLASSHASH_SIZE; i++) + INIT_LIST_HEAD(classhash_table + i); + + for (i = 0; i < CHAINHASH_SIZE; i++) + INIT_LIST_HEAD(chainhash_table + i); + + lockdep_initialized = 1; +} + +void __init lockdep_info(void) +{ + printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n"); + + printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES); + printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH); + printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS); + printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE); + printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES); + printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS); + printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE); + + printk(" memory used by lock dependency info: %lu kB\n", + (sizeof(struct lock_class) * MAX_LOCKDEP_KEYS + + sizeof(struct list_head) * CLASSHASH_SIZE + + sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES + + sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS + + sizeof(struct list_head) * CHAINHASH_SIZE) / 1024); + + printk(" per task-struct memory footprint: %lu bytes\n", + sizeof(struct held_lock) * MAX_LOCK_DEPTH); + +#ifdef CONFIG_DEBUG_LOCKDEP + if (lockdep_init_error) + printk("WARNING: lockdep init error! Arch code didnt call lockdep_init() early enough?\n"); +#endif +} + +static inline int in_range(const void *start, const void *addr, const void *end) +{ + return addr >= start && addr <= end; +} + +static void +print_freed_lock_bug(struct task_struct *curr, const void *mem_from, + const void *mem_to) +{ + if (!debug_locks_off()) + return; + if (debug_locks_silent) + return; + + printk("\n=========================\n"); + printk( "[ BUG: held lock freed! ]\n"); + printk( "-------------------------\n"); + printk("%s/%d is freeing memory %p-%p, with a lock still held there!\n", + curr->comm, curr->pid, mem_from, mem_to-1); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); +} + +/* + * Called when kernel memory is freed (or unmapped), or if a lock + * is destroyed or reinitialized - this code checks whether there is + * any held lock in the memory range of <from> to <to>: + */ +void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len) +{ + const void *mem_to = mem_from + mem_len, *lock_from, *lock_to; + struct task_struct *curr = current; + struct held_lock *hlock; + unsigned long flags; + int i; + + if (unlikely(!debug_locks)) + return; + + local_irq_save(flags); + for (i = 0; i < curr->lockdep_depth; i++) { + hlock = curr->held_locks + i; + + lock_from = (void *)hlock->instance; + lock_to = (void *)(hlock->instance + 1); + + if (!in_range(mem_from, lock_from, mem_to) && + !in_range(mem_from, lock_to, mem_to)) + continue; + + print_freed_lock_bug(curr, mem_from, mem_to); + break; + } + local_irq_restore(flags); +} + +static void print_held_locks_bug(struct task_struct *curr) +{ + if (!debug_locks_off()) + return; + if (debug_locks_silent) + return; + + printk("\n=====================================\n"); + printk( "[ BUG: lock held at task exit time! ]\n"); + printk( "-------------------------------------\n"); + printk("%s/%d is exiting with locks still held!\n", + curr->comm, curr->pid); + lockdep_print_held_locks(curr); + + printk("\nstack backtrace:\n"); + dump_stack(); +} + +void debug_check_no_locks_held(struct task_struct *task) +{ + if (unlikely(task->lockdep_depth > 0)) + print_held_locks_bug(task); +} + +void debug_show_all_locks(void) +{ + struct task_struct *g, *p; + int count = 10; + int unlock = 1; + + printk("\nShowing all locks held in the system:\n"); + + /* + * Here we try to get the tasklist_lock as hard as possible, + * if not successful after 2 seconds we ignore it (but keep + * trying). This is to enable a debug printout even if a + * tasklist_lock-holding task deadlocks or crashes. + */ +retry: + if (!read_trylock(&tasklist_lock)) { + if (count == 10) + printk("hm, tasklist_lock locked, retrying... "); + if (count) { + count--; + printk(" #%d", 10-count); + mdelay(200); + goto retry; + } + printk(" ignoring it.\n"); + unlock = 0; + } + if (count != 10) + printk(" locked it.\n"); + + do_each_thread(g, p) { + if (p->lockdep_depth) + lockdep_print_held_locks(p); + if (!unlock) + if (read_trylock(&tasklist_lock)) + unlock = 1; + } while_each_thread(g, p); + + printk("\n"); + printk("=============================================\n\n"); + + if (unlock) + read_unlock(&tasklist_lock); +} + +EXPORT_SYMBOL_GPL(debug_show_all_locks); + +void debug_show_held_locks(struct task_struct *task) +{ + lockdep_print_held_locks(task); +} + +EXPORT_SYMBOL_GPL(debug_show_held_locks); + diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h new file mode 100644 index 00000000000..0d355f24fe0 --- /dev/null +++ b/kernel/lockdep_internals.h @@ -0,0 +1,78 @@ +/* + * kernel/lockdep_internals.h + * + * Runtime locking correctness validator + * + * lockdep subsystem internal functions and variables. + */ + +/* + * MAX_LOCKDEP_ENTRIES is the maximum number of lock dependencies + * we track. + * + * We use the per-lock dependency maps in two ways: we grow it by adding + * every to-be-taken lock to all currently held lock's own dependency + * table (if it's not there yet), and we check it for lock order + * conflicts and deadlocks. + */ +#define MAX_LOCKDEP_ENTRIES 8192UL + +#define MAX_LOCKDEP_KEYS_BITS 11 +#define MAX_LOCKDEP_KEYS (1UL << MAX_LOCKDEP_KEYS_BITS) + +#define MAX_LOCKDEP_CHAINS_BITS 13 +#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) + +/* + * Stack-trace: tightly packed array of stack backtrace + * addresses. Protected by the hash_lock. + */ +#define MAX_STACK_TRACE_ENTRIES 131072UL + +extern struct list_head all_lock_classes; + +extern void +get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4); + +extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str); + +extern unsigned long nr_lock_classes; +extern unsigned long nr_list_entries; +extern unsigned long nr_lock_chains; +extern unsigned long nr_stack_trace_entries; + +extern unsigned int nr_hardirq_chains; +extern unsigned int nr_softirq_chains; +extern unsigned int nr_process_chains; +extern unsigned int max_lockdep_depth; +extern unsigned int max_recursion_depth; + +#ifdef CONFIG_DEBUG_LOCKDEP +/* + * Various lockdep statistics: + */ +extern atomic_t chain_lookup_hits; +extern atomic_t chain_lookup_misses; +extern atomic_t hardirqs_on_events; +extern atomic_t hardirqs_off_events; +extern atomic_t redundant_hardirqs_on; +extern atomic_t redundant_hardirqs_off; +extern atomic_t softirqs_on_events; +extern atomic_t softirqs_off_events; +extern atomic_t redundant_softirqs_on; +extern atomic_t redundant_softirqs_off; +extern atomic_t nr_unused_locks; +extern atomic_t nr_cyclic_checks; +extern atomic_t nr_cyclic_check_recursions; +extern atomic_t nr_find_usage_forwards_checks; +extern atomic_t nr_find_usage_forwards_recursions; +extern atomic_t nr_find_usage_backwards_checks; +extern atomic_t nr_find_usage_backwards_recursions; +# define debug_atomic_inc(ptr) atomic_inc(ptr) +# define debug_atomic_dec(ptr) atomic_dec(ptr) +# define debug_atomic_read(ptr) atomic_read(ptr) +#else +# define debug_atomic_inc(ptr) do { } while (0) +# define debug_atomic_dec(ptr) do { } while (0) +# define debug_atomic_read(ptr) 0 +#endif diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c new file mode 100644 index 00000000000..f6e72eaab3f --- /dev/null +++ b/kernel/lockdep_proc.c @@ -0,0 +1,345 @@ +/* + * kernel/lockdep_proc.c + * + * Runtime locking correctness validator + * + * Started by Ingo Molnar: + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + * + * Code for /proc/lockdep and /proc/lockdep_stats: + * + */ +#include <linux/sched.h> +#include <linux/module.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/kallsyms.h> +#include <linux/debug_locks.h> + +#include "lockdep_internals.h" + +static void *l_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct lock_class *class = v; + + (*pos)++; + + if (class->lock_entry.next != &all_lock_classes) + class = list_entry(class->lock_entry.next, struct lock_class, + lock_entry); + else + class = NULL; + m->private = class; + + return class; +} + +static void *l_start(struct seq_file *m, loff_t *pos) +{ + struct lock_class *class = m->private; + + if (&class->lock_entry == all_lock_classes.next) + seq_printf(m, "all lock classes:\n"); + + return class; +} + +static void l_stop(struct seq_file *m, void *v) +{ +} + +static unsigned long count_forward_deps(struct lock_class *class) +{ + struct lock_list *entry; + unsigned long ret = 1; + + /* + * Recurse this class's dependency list: + */ + list_for_each_entry(entry, &class->locks_after, entry) + ret += count_forward_deps(entry->class); + + return ret; +} + +static unsigned long count_backward_deps(struct lock_class *class) +{ + struct lock_list *entry; + unsigned long ret = 1; + + /* + * Recurse this class's dependency list: + */ + list_for_each_entry(entry, &class->locks_before, entry) + ret += count_backward_deps(entry->class); + + return ret; +} + +static int l_show(struct seq_file *m, void *v) +{ + unsigned long nr_forward_deps, nr_backward_deps; + struct lock_class *class = m->private; + char str[128], c1, c2, c3, c4; + const char *name; + + seq_printf(m, "%p", class->key); +#ifdef CONFIG_DEBUG_LOCKDEP + seq_printf(m, " OPS:%8ld", class->ops); +#endif + nr_forward_deps = count_forward_deps(class); + seq_printf(m, " FD:%5ld", nr_forward_deps); + + nr_backward_deps = count_backward_deps(class); + seq_printf(m, " BD:%5ld", nr_backward_deps); + + get_usage_chars(class, &c1, &c2, &c3, &c4); + seq_printf(m, " %c%c%c%c", c1, c2, c3, c4); + + name = class->name; + if (!name) { + name = __get_key_name(class->key, str); + seq_printf(m, ": %s", name); + } else{ + seq_printf(m, ": %s", name); + if (class->name_version > 1) + seq_printf(m, "#%d", class->name_version); + if (class->subclass) + seq_printf(m, "/%d", class->subclass); + } + seq_puts(m, "\n"); + + return 0; +} + +static struct seq_operations lockdep_ops = { + .start = l_start, + .next = l_next, + .stop = l_stop, + .show = l_show, +}; + +static int lockdep_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &lockdep_ops); + if (!res) { + struct seq_file *m = file->private_data; + + if (!list_empty(&all_lock_classes)) + m->private = list_entry(all_lock_classes.next, + struct lock_class, lock_entry); + else + m->private = NULL; + } + return res; +} + +static struct file_operations proc_lockdep_operations = { + .open = lockdep_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static void lockdep_stats_debug_show(struct seq_file *m) +{ +#ifdef CONFIG_DEBUG_LOCKDEP + unsigned int hi1 = debug_atomic_read(&hardirqs_on_events), + hi2 = debug_atomic_read(&hardirqs_off_events), + hr1 = debug_atomic_read(&redundant_hardirqs_on), + hr2 = debug_atomic_read(&redundant_hardirqs_off), + si1 = debug_atomic_read(&softirqs_on_events), + si2 = debug_atomic_read(&softirqs_off_events), + sr1 = debug_atomic_read(&redundant_softirqs_on), + sr2 = debug_atomic_read(&redundant_softirqs_off); + + seq_printf(m, " chain lookup misses: %11u\n", + debug_atomic_read(&chain_lookup_misses)); + seq_printf(m, " chain lookup hits: %11u\n", + debug_atomic_read(&chain_lookup_hits)); + seq_printf(m, " cyclic checks: %11u\n", + debug_atomic_read(&nr_cyclic_checks)); + seq_printf(m, " cyclic-check recursions: %11u\n", + debug_atomic_read(&nr_cyclic_check_recursions)); + seq_printf(m, " find-mask forwards checks: %11u\n", + debug_atomic_read(&nr_find_usage_forwards_checks)); + seq_printf(m, " find-mask forwards recursions: %11u\n", + debug_atomic_read(&nr_find_usage_forwards_recursions)); + seq_printf(m, " find-mask backwards checks: %11u\n", + debug_atomic_read(&nr_find_usage_backwards_checks)); + seq_printf(m, " find-mask backwards recursions:%11u\n", + debug_atomic_read(&nr_find_usage_backwards_recursions)); + + seq_printf(m, " hardirq on events: %11u\n", hi1); + seq_printf(m, " hardirq off events: %11u\n", hi2); + seq_printf(m, " redundant hardirq ons: %11u\n", hr1); + seq_printf(m, " redundant hardirq offs: %11u\n", hr2); + seq_printf(m, " softirq on events: %11u\n", si1); + seq_printf(m, " softirq off events: %11u\n", si2); + seq_printf(m, " redundant softirq ons: %11u\n", sr1); + seq_printf(m, " redundant softirq offs: %11u\n", sr2); +#endif +} + +static int lockdep_stats_show(struct seq_file *m, void *v) +{ + struct lock_class *class; + unsigned long nr_unused = 0, nr_uncategorized = 0, + nr_irq_safe = 0, nr_irq_unsafe = 0, + nr_softirq_safe = 0, nr_softirq_unsafe = 0, + nr_hardirq_safe = 0, nr_hardirq_unsafe = 0, + nr_irq_read_safe = 0, nr_irq_read_unsafe = 0, + nr_softirq_read_safe = 0, nr_softirq_read_unsafe = 0, + nr_hardirq_read_safe = 0, nr_hardirq_read_unsafe = 0, + sum_forward_deps = 0, factor = 0; + + list_for_each_entry(class, &all_lock_classes, lock_entry) { + + if (class->usage_mask == 0) + nr_unused++; + if (class->usage_mask == LOCKF_USED) + nr_uncategorized++; + if (class->usage_mask & LOCKF_USED_IN_IRQ) + nr_irq_safe++; + if (class->usage_mask & LOCKF_ENABLED_IRQS) + nr_irq_unsafe++; + if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ) + nr_softirq_safe++; + if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS) + nr_softirq_unsafe++; + if (class->usage_mask & LOCKF_USED_IN_HARDIRQ) + nr_hardirq_safe++; + if (class->usage_mask & LOCKF_ENABLED_HARDIRQS) + nr_hardirq_unsafe++; + if (class->usage_mask & LOCKF_USED_IN_IRQ_READ) + nr_irq_read_safe++; + if (class->usage_mask & LOCKF_ENABLED_IRQS_READ) + nr_irq_read_unsafe++; + if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) + nr_softirq_read_safe++; + if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) + nr_softirq_read_unsafe++; + if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) + nr_hardirq_read_safe++; + if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) + nr_hardirq_read_unsafe++; + + sum_forward_deps += count_forward_deps(class); + } +#ifdef CONFIG_LOCKDEP_DEBUG + DEBUG_LOCKS_WARN_ON(debug_atomic_read(&nr_unused_locks) != nr_unused); +#endif + seq_printf(m, " lock-classes: %11lu [max: %lu]\n", + nr_lock_classes, MAX_LOCKDEP_KEYS); + seq_printf(m, " direct dependencies: %11lu [max: %lu]\n", + nr_list_entries, MAX_LOCKDEP_ENTRIES); + seq_printf(m, " indirect dependencies: %11lu\n", + sum_forward_deps); + + /* + * Total number of dependencies: + * + * All irq-safe locks may nest inside irq-unsafe locks, + * plus all the other known dependencies: + */ + seq_printf(m, " all direct dependencies: %11lu\n", + nr_irq_unsafe * nr_irq_safe + + nr_hardirq_unsafe * nr_hardirq_safe + + nr_list_entries); + + /* + * Estimated factor between direct and indirect + * dependencies: + */ + if (nr_list_entries) + factor = sum_forward_deps / nr_list_entries; + + seq_printf(m, " dependency chains: %11lu [max: %lu]\n", + nr_lock_chains, MAX_LOCKDEP_CHAINS); + +#ifdef CONFIG_TRACE_IRQFLAGS + seq_printf(m, " in-hardirq chains: %11u\n", + nr_hardirq_chains); + seq_printf(m, " in-softirq chains: %11u\n", + nr_softirq_chains); +#endif + seq_printf(m, " in-process chains: %11u\n", + nr_process_chains); + seq_printf(m, " stack-trace entries: %11lu [max: %lu]\n", + nr_stack_trace_entries, MAX_STACK_TRACE_ENTRIES); + seq_printf(m, " combined max dependencies: %11u\n", + (nr_hardirq_chains + 1) * + (nr_softirq_chains + 1) * + (nr_process_chains + 1) + ); + seq_printf(m, " hardirq-safe locks: %11lu\n", + nr_hardirq_safe); + seq_printf(m, " hardirq-unsafe locks: %11lu\n", + nr_hardirq_unsafe); + seq_printf(m, " softirq-safe locks: %11lu\n", + nr_softirq_safe); + seq_printf(m, " softirq-unsafe locks: %11lu\n", + nr_softirq_unsafe); + seq_printf(m, " irq-safe locks: %11lu\n", + nr_irq_safe); + seq_printf(m, " irq-unsafe locks: %11lu\n", + nr_irq_unsafe); + + seq_printf(m, " hardirq-read-safe locks: %11lu\n", + nr_hardirq_read_safe); + seq_printf(m, " hardirq-read-unsafe locks: %11lu\n", + nr_hardirq_read_unsafe); + seq_printf(m, " softirq-read-safe locks: %11lu\n", + nr_softirq_read_safe); + seq_printf(m, " softirq-read-unsafe locks: %11lu\n", + nr_softirq_read_unsafe); + seq_printf(m, " irq-read-safe locks: %11lu\n", + nr_irq_read_safe); + seq_printf(m, " irq-read-unsafe locks: %11lu\n", + nr_irq_read_unsafe); + + seq_printf(m, " uncategorized locks: %11lu\n", + nr_uncategorized); + seq_printf(m, " unused locks: %11lu\n", + nr_unused); + seq_printf(m, " max locking depth: %11u\n", + max_lockdep_depth); + seq_printf(m, " max recursion depth: %11u\n", + max_recursion_depth); + lockdep_stats_debug_show(m); + seq_printf(m, " debug_locks: %11u\n", + debug_locks); + + return 0; +} + +static int lockdep_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, lockdep_stats_show, NULL); +} + +static struct file_operations proc_lockdep_stats_operations = { + .open = lockdep_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __init lockdep_proc_init(void) +{ + struct proc_dir_entry *entry; + + entry = create_proc_entry("lockdep", S_IRUSR, NULL); + if (entry) + entry->proc_fops = &proc_lockdep_operations; + + entry = create_proc_entry("lockdep_stats", S_IRUSR, NULL); + if (entry) + entry->proc_fops = &proc_lockdep_stats_operations; + + return 0; +} + +__initcall(lockdep_proc_init); + diff --git a/kernel/module.c b/kernel/module.c index 281172f01e9..35e1b1f859d 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1121,6 +1121,9 @@ static void free_module(struct module *mod) if (mod->percpu) percpu_modfree(mod->percpu); + /* Free lock-classes: */ + lockdep_free_key_range(mod->module_core, mod->core_size); + /* Finally, free the core (containing the module structure) */ module_free(mod, mod->module_core); } @@ -2159,6 +2162,29 @@ const struct exception_table_entry *search_module_extables(unsigned long addr) return e; } +/* + * Is this a valid module address? + */ +int is_module_address(unsigned long addr) +{ + unsigned long flags; + struct module *mod; + + spin_lock_irqsave(&modlist_lock, flags); + + list_for_each_entry(mod, &modules, list) { + if (within(addr, mod->module_core, mod->core_size)) { + spin_unlock_irqrestore(&modlist_lock, flags); + return 1; + } + } + + spin_unlock_irqrestore(&modlist_lock, flags); + + return 0; +} + + /* Is this a valid kernel address? We don't grab the lock: we are oopsing. */ struct module *__module_text_address(unsigned long addr) { diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c index e38e4bac97c..e3203c654dd 100644 --- a/kernel/mutex-debug.c +++ b/kernel/mutex-debug.c @@ -20,367 +20,19 @@ #include <linux/spinlock.h> #include <linux/kallsyms.h> #include <linux/interrupt.h> +#include <linux/debug_locks.h> #include "mutex-debug.h" /* - * We need a global lock when we walk through the multi-process - * lock tree. Only used in the deadlock-debugging case. - */ -DEFINE_SPINLOCK(debug_mutex_lock); - -/* - * All locks held by all tasks, in a single global list: - */ -LIST_HEAD(debug_mutex_held_locks); - -/* - * In the debug case we carry the caller's instruction pointer into - * other functions, but we dont want the function argument overhead - * in the nondebug case - hence these macros: - */ -#define __IP_DECL__ , unsigned long ip -#define __IP__ , ip -#define __RET_IP__ , (unsigned long)__builtin_return_address(0) - -/* - * "mutex debugging enabled" flag. We turn it off when we detect - * the first problem because we dont want to recurse back - * into the tracing code when doing error printk or - * executing a BUG(): - */ -int debug_mutex_on = 1; - -static void printk_task(struct task_struct *p) -{ - if (p) - printk("%16s:%5d [%p, %3d]", p->comm, p->pid, p, p->prio); - else - printk("<none>"); -} - -static void printk_ti(struct thread_info *ti) -{ - if (ti) - printk_task(ti->task); - else - printk("<none>"); -} - -static void printk_task_short(struct task_struct *p) -{ - if (p) - printk("%s/%d [%p, %3d]", p->comm, p->pid, p, p->prio); - else - printk("<none>"); -} - -static void printk_lock(struct mutex *lock, int print_owner) -{ - printk(" [%p] {%s}\n", lock, lock->name); - - if (print_owner && lock->owner) { - printk(".. held by: "); - printk_ti(lock->owner); - printk("\n"); - } - if (lock->owner) { - printk("... acquired at: "); - print_symbol("%s\n", lock->acquire_ip); - } -} - -/* - * printk locks held by a task: - */ -static void show_task_locks(struct task_struct *p) -{ - switch (p->state) { - case TASK_RUNNING: printk("R"); break; - case TASK_INTERRUPTIBLE: printk("S"); break; - case TASK_UNINTERRUPTIBLE: printk("D"); break; - case TASK_STOPPED: printk("T"); break; - case EXIT_ZOMBIE: printk("Z"); break; - case EXIT_DEAD: printk("X"); break; - default: printk("?"); break; - } - printk_task(p); - if (p->blocked_on) { - struct mutex *lock = p->blocked_on->lock; - - printk(" blocked on mutex:"); - printk_lock(lock, 1); - } else - printk(" (not blocked on mutex)\n"); -} - -/* - * printk all locks held in the system (if filter == NULL), - * or all locks belonging to a single task (if filter != NULL): - */ -void show_held_locks(struct task_struct *filter) -{ - struct list_head *curr, *cursor = NULL; - struct mutex *lock; - struct thread_info *t; - unsigned long flags; - int count = 0; - - if (filter) { - printk("------------------------------\n"); - printk("| showing all locks held by: | ("); - printk_task_short(filter); - printk("):\n"); - printk("------------------------------\n"); - } else { - printk("---------------------------\n"); - printk("| showing all locks held: |\n"); - printk("---------------------------\n"); - } - - /* - * Play safe and acquire the global trace lock. We - * cannot printk with that lock held so we iterate - * very carefully: - */ -next: - debug_spin_lock_save(&debug_mutex_lock, flags); - list_for_each(curr, &debug_mutex_held_locks) { - if (cursor && curr != cursor) - continue; - lock = list_entry(curr, struct mutex, held_list); - t = lock->owner; - if (filter && (t != filter->thread_info)) - continue; - count++; - cursor = curr->next; - debug_spin_unlock_restore(&debug_mutex_lock, flags); - - printk("\n#%03d: ", count); - printk_lock(lock, filter ? 0 : 1); - goto next; - } - debug_spin_unlock_restore(&debug_mutex_lock, flags); - printk("\n"); -} - -void mutex_debug_show_all_locks(void) -{ - struct task_struct *g, *p; - int count = 10; - int unlock = 1; - - printk("\nShowing all blocking locks in the system:\n"); - - /* - * Here we try to get the tasklist_lock as hard as possible, - * if not successful after 2 seconds we ignore it (but keep - * trying). This is to enable a debug printout even if a - * tasklist_lock-holding task deadlocks or crashes. - */ -retry: - if (!read_trylock(&tasklist_lock)) { - if (count == 10) - printk("hm, tasklist_lock locked, retrying... "); - if (count) { - count--; - printk(" #%d", 10-count); - mdelay(200); - goto retry; - } - printk(" ignoring it.\n"); - unlock = 0; - } - if (count != 10) - printk(" locked it.\n"); - - do_each_thread(g, p) { - show_task_locks(p); - if (!unlock) - if (read_trylock(&tasklist_lock)) - unlock = 1; - } while_each_thread(g, p); - - printk("\n"); - show_held_locks(NULL); - printk("=============================================\n\n"); - - if (unlock) - read_unlock(&tasklist_lock); -} - -static void report_deadlock(struct task_struct *task, struct mutex *lock, - struct mutex *lockblk, unsigned long ip) -{ - printk("\n%s/%d is trying to acquire this lock:\n", - current->comm, current->pid); - printk_lock(lock, 1); - printk("... trying at: "); - print_symbol("%s\n", ip); - show_held_locks(current); - - if (lockblk) { - printk("but %s/%d is deadlocking current task %s/%d!\n\n", - task->comm, task->pid, current->comm, current->pid); - printk("\n%s/%d is blocked on this lock:\n", - task->comm, task->pid); - printk_lock(lockblk, 1); - - show_held_locks(task); - - printk("\n%s/%d's [blocked] stackdump:\n\n", - task->comm, task->pid); - show_stack(task, NULL); - } - - printk("\n%s/%d's [current] stackdump:\n\n", - current->comm, current->pid); - dump_stack(); - mutex_debug_show_all_locks(); - printk("[ turning off deadlock detection. Please report this. ]\n\n"); - local_irq_disable(); -} - -/* - * Recursively check for mutex deadlocks: - */ -static int check_deadlock(struct mutex *lock, int depth, - struct thread_info *ti, unsigned long ip) -{ - struct mutex *lockblk; - struct task_struct *task; - - if (!debug_mutex_on) - return 0; - - ti = lock->owner; - if (!ti) - return 0; - - task = ti->task; - lockblk = NULL; - if (task->blocked_on) - lockblk = task->blocked_on->lock; - - /* Self-deadlock: */ - if (current == task) { - DEBUG_OFF(); - if (depth) - return 1; - printk("\n==========================================\n"); - printk( "[ BUG: lock recursion deadlock detected! |\n"); - printk( "------------------------------------------\n"); - report_deadlock(task, lock, NULL, ip); - return 0; - } - - /* Ugh, something corrupted the lock data structure? */ - if (depth > 20) { - DEBUG_OFF(); - printk("\n===========================================\n"); - printk( "[ BUG: infinite lock dependency detected!? |\n"); - printk( "-------------------------------------------\n"); - report_deadlock(task, lock, lockblk, ip); - return 0; - } - - /* Recursively check for dependencies: */ - if (lockblk && check_deadlock(lockblk, depth+1, ti, ip)) { - printk("\n============================================\n"); - printk( "[ BUG: circular locking deadlock detected! ]\n"); - printk( "--------------------------------------------\n"); - report_deadlock(task, lock, lockblk, ip); - return 0; - } - return 0; -} - -/* - * Called when a task exits, this function checks whether the - * task is holding any locks, and reports the first one if so: - */ -void mutex_debug_check_no_locks_held(struct task_struct *task) -{ - struct list_head *curr, *next; - struct thread_info *t; - unsigned long flags; - struct mutex *lock; - - if (!debug_mutex_on) - return; - - debug_spin_lock_save(&debug_mutex_lock, flags); - list_for_each_safe(curr, next, &debug_mutex_held_locks) { - lock = list_entry(curr, struct mutex, held_list); - t = lock->owner; - if (t != task->thread_info) - continue; - list_del_init(curr); - DEBUG_OFF(); - debug_spin_unlock_restore(&debug_mutex_lock, flags); - - printk("BUG: %s/%d, lock held at task exit time!\n", - task->comm, task->pid); - printk_lock(lock, 1); - if (lock->owner != task->thread_info) - printk("exiting task is not even the owner??\n"); - return; - } - debug_spin_unlock_restore(&debug_mutex_lock, flags); -} - -/* - * Called when kernel memory is freed (or unmapped), or if a mutex - * is destroyed or reinitialized - this code checks whether there is - * any held lock in the memory range of <from> to <to>: - */ -void mutex_debug_check_no_locks_freed(const void *from, unsigned long len) -{ - struct list_head *curr, *next; - const void *to = from + len; - unsigned long flags; - struct mutex *lock; - void *lock_addr; - - if (!debug_mutex_on) - return; - - debug_spin_lock_save(&debug_mutex_lock, flags); - list_for_each_safe(curr, next, &debug_mutex_held_locks) { - lock = list_entry(curr, struct mutex, held_list); - lock_addr = lock; - if (lock_addr < from || lock_addr >= to) - continue; - list_del_init(curr); - DEBUG_OFF(); - debug_spin_unlock_restore(&debug_mutex_lock, flags); - - printk("BUG: %s/%d, active lock [%p(%p-%p)] freed!\n", - current->comm, current->pid, lock, from, to); - dump_stack(); - printk_lock(lock, 1); - if (lock->owner != current_thread_info()) - printk("freeing task is not even the owner??\n"); - return; - } - debug_spin_unlock_restore(&debug_mutex_lock, flags); -} - -/* * Must be called with lock->wait_lock held. */ -void debug_mutex_set_owner(struct mutex *lock, - struct thread_info *new_owner __IP_DECL__) +void debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner) { lock->owner = new_owner; - DEBUG_WARN_ON(!list_empty(&lock->held_list)); - if (debug_mutex_on) { - list_add_tail(&lock->held_list, &debug_mutex_held_locks); - lock->acquire_ip = ip; - } } -void debug_mutex_init_waiter(struct mutex_waiter *waiter) +void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) { memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); waiter->magic = waiter; @@ -389,23 +41,23 @@ void debug_mutex_init_waiter(struct mutex_waiter *waiter) void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter) { - SMP_DEBUG_WARN_ON(!spin_is_locked(&lock->wait_lock)); - DEBUG_WARN_ON(list_empty(&lock->wait_list)); - DEBUG_WARN_ON(waiter->magic != waiter); - DEBUG_WARN_ON(list_empty(&waiter->list)); + SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock)); + DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list)); + DEBUG_LOCKS_WARN_ON(waiter->magic != waiter); + DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list)); } void debug_mutex_free_waiter(struct mutex_waiter *waiter) { - DEBUG_WARN_ON(!list_empty(&waiter->list)); + DEBUG_LOCKS_WARN_ON(!list_empty(&waiter->list)); memset(waiter, MUTEX_DEBUG_FREE, sizeof(*waiter)); } void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, - struct thread_info *ti __IP_DECL__) + struct thread_info *ti) { - SMP_DEBUG_WARN_ON(!spin_is_locked(&lock->wait_lock)); - check_deadlock(lock, 0, ti, ip); + SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock)); + /* Mark the current thread as blocked on the lock: */ ti->task->blocked_on = waiter; waiter->lock = lock; @@ -414,9 +66,9 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter, struct thread_info *ti) { - DEBUG_WARN_ON(list_empty(&waiter->list)); - DEBUG_WARN_ON(waiter->task != ti->task); - DEBUG_WARN_ON(ti->task->blocked_on != waiter); + DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list)); + DEBUG_LOCKS_WARN_ON(waiter->task != ti->task); + DEBUG_LOCKS_WARN_ON(ti->task->blocked_on != waiter); ti->task->blocked_on = NULL; list_del_init(&waiter->list); @@ -425,24 +77,23 @@ void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter, void debug_mutex_unlock(struct mutex *lock) { - DEBUG_WARN_ON(lock->magic != lock); - DEBUG_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); - DEBUG_WARN_ON(lock->owner != current_thread_info()); - if (debug_mutex_on) { - DEBUG_WARN_ON(list_empty(&lock->held_list)); - list_del_init(&lock->held_list); - } + DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); + DEBUG_LOCKS_WARN_ON(lock->magic != lock); + DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); + DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); } -void debug_mutex_init(struct mutex *lock, const char *name) +void debug_mutex_init(struct mutex *lock, const char *name, + struct lock_class_key *key) { +#ifdef CONFIG_DEBUG_LOCK_ALLOC /* * Make sure we are not reinitializing a held lock: */ - mutex_debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + lockdep_init_map(&lock->dep_map, name, key); +#endif lock->owner = NULL; - INIT_LIST_HEAD(&lock->held_list); - lock->name = name; lock->magic = lock; } @@ -456,7 +107,7 @@ void debug_mutex_init(struct mutex *lock, const char *name) */ void fastcall mutex_destroy(struct mutex *lock) { - DEBUG_WARN_ON(mutex_is_locked(lock)); + DEBUG_LOCKS_WARN_ON(mutex_is_locked(lock)); lock->magic = NULL; } diff --git a/kernel/mutex-debug.h b/kernel/mutex-debug.h index a5196c36a5f..babfbdfc534 100644 --- a/kernel/mutex-debug.h +++ b/kernel/mutex-debug.h @@ -10,110 +10,44 @@ * More details are in kernel/mutex-debug.c. */ -extern spinlock_t debug_mutex_lock; -extern struct list_head debug_mutex_held_locks; -extern int debug_mutex_on; - -/* - * In the debug case we carry the caller's instruction pointer into - * other functions, but we dont want the function argument overhead - * in the nondebug case - hence these macros: - */ -#define __IP_DECL__ , unsigned long ip -#define __IP__ , ip -#define __RET_IP__ , (unsigned long)__builtin_return_address(0) - /* * This must be called with lock->wait_lock held. */ -extern void debug_mutex_set_owner(struct mutex *lock, - struct thread_info *new_owner __IP_DECL__); +extern void +debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner); static inline void debug_mutex_clear_owner(struct mutex *lock) { lock->owner = NULL; } -extern void debug_mutex_init_waiter(struct mutex_waiter *waiter); +extern void debug_mutex_lock_common(struct mutex *lock, + struct mutex_waiter *waiter); extern void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter); extern void debug_mutex_free_waiter(struct mutex_waiter *waiter); extern void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, - struct thread_info *ti __IP_DECL__); + struct thread_info *ti); extern void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter, struct thread_info *ti); extern void debug_mutex_unlock(struct mutex *lock); -extern void debug_mutex_init(struct mutex *lock, const char *name); - -#define debug_spin_lock_save(lock, flags) \ - do { \ - local_irq_save(flags); \ - if (debug_mutex_on) \ - spin_lock(lock); \ - } while (0) - -#define debug_spin_unlock_restore(lock, flags) \ - do { \ - if (debug_mutex_on) \ - spin_unlock(lock); \ - local_irq_restore(flags); \ - preempt_check_resched(); \ - } while (0) +extern void debug_mutex_init(struct mutex *lock, const char *name, + struct lock_class_key *key); #define spin_lock_mutex(lock, flags) \ do { \ struct mutex *l = container_of(lock, struct mutex, wait_lock); \ \ - DEBUG_WARN_ON(in_interrupt()); \ - debug_spin_lock_save(&debug_mutex_lock, flags); \ - spin_lock(lock); \ - DEBUG_WARN_ON(l->magic != l); \ + DEBUG_LOCKS_WARN_ON(in_interrupt()); \ + local_irq_save(flags); \ + __raw_spin_lock(&(lock)->raw_lock); \ + DEBUG_LOCKS_WARN_ON(l->magic != l); \ } while (0) #define spin_unlock_mutex(lock, flags) \ do { \ - spin_unlock(lock); \ - debug_spin_unlock_restore(&debug_mutex_lock, flags); \ + __raw_spin_unlock(&(lock)->raw_lock); \ + local_irq_restore(flags); \ + preempt_check_resched(); \ } while (0) - -#define DEBUG_OFF() \ -do { \ - if (debug_mutex_on) { \ - debug_mutex_on = 0; \ - console_verbose(); \ - if (spin_is_locked(&debug_mutex_lock)) \ - spin_unlock(&debug_mutex_lock); \ - } \ -} while (0) - -#define DEBUG_BUG() \ -do { \ - if (debug_mutex_on) { \ - DEBUG_OFF(); \ - BUG(); \ - } \ -} while (0) - -#define DEBUG_WARN_ON(c) \ -do { \ - if (unlikely(c && debug_mutex_on)) { \ - DEBUG_OFF(); \ - WARN_ON(1); \ - } \ -} while (0) - -# define DEBUG_BUG_ON(c) \ -do { \ - if (unlikely(c)) \ - DEBUG_BUG(); \ -} while (0) - -#ifdef CONFIG_SMP -# define SMP_DEBUG_WARN_ON(c) DEBUG_WARN_ON(c) -# define SMP_DEBUG_BUG_ON(c) DEBUG_BUG_ON(c) -#else -# define SMP_DEBUG_WARN_ON(c) do { } while (0) -# define SMP_DEBUG_BUG_ON(c) do { } while (0) -#endif - diff --git a/kernel/mutex.c b/kernel/mutex.c index 7043db21bbc..8c71cf72a49 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -17,6 +17,7 @@ #include <linux/module.h> #include <linux/spinlock.h> #include <linux/interrupt.h> +#include <linux/debug_locks.h> /* * In the DEBUG case we are using the "NULL fastpath" for mutexes, @@ -38,13 +39,14 @@ * * It is not allowed to initialize an already locked mutex. */ -void fastcall __mutex_init(struct mutex *lock, const char *name) +void +__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) { atomic_set(&lock->count, 1); spin_lock_init(&lock->wait_lock); INIT_LIST_HEAD(&lock->wait_list); - debug_mutex_init(lock, name); + debug_mutex_init(lock, name, key); } EXPORT_SYMBOL(__mutex_init); @@ -56,7 +58,7 @@ EXPORT_SYMBOL(__mutex_init); * branch is predicted by the CPU as default-untaken. */ static void fastcall noinline __sched -__mutex_lock_slowpath(atomic_t *lock_count __IP_DECL__); +__mutex_lock_slowpath(atomic_t *lock_count); /*** * mutex_lock - acquire the mutex @@ -79,7 +81,7 @@ __mutex_lock_slowpath(atomic_t *lock_count __IP_DECL__); * * This function is similar to (but not equivalent to) down(). */ -void fastcall __sched mutex_lock(struct mutex *lock) +void inline fastcall __sched mutex_lock(struct mutex *lock) { might_sleep(); /* @@ -92,7 +94,7 @@ void fastcall __sched mutex_lock(struct mutex *lock) EXPORT_SYMBOL(mutex_lock); static void fastcall noinline __sched -__mutex_unlock_slowpath(atomic_t *lock_count __IP_DECL__); +__mutex_unlock_slowpath(atomic_t *lock_count); /*** * mutex_unlock - release the mutex @@ -120,18 +122,18 @@ EXPORT_SYMBOL(mutex_unlock); * Lock a mutex (possibly interruptible), slowpath: */ static inline int __sched -__mutex_lock_common(struct mutex *lock, long state __IP_DECL__) +__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass) { struct task_struct *task = current; struct mutex_waiter waiter; unsigned int old_val; unsigned long flags; - debug_mutex_init_waiter(&waiter); - spin_lock_mutex(&lock->wait_lock, flags); - debug_mutex_add_waiter(lock, &waiter, task->thread_info, ip); + debug_mutex_lock_common(lock, &waiter); + mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); + debug_mutex_add_waiter(lock, &waiter, task->thread_info); /* add waiting tasks to the end of the waitqueue (FIFO): */ list_add_tail(&waiter.list, &lock->wait_list); @@ -158,6 +160,7 @@ __mutex_lock_common(struct mutex *lock, long state __IP_DECL__) if (unlikely(state == TASK_INTERRUPTIBLE && signal_pending(task))) { mutex_remove_waiter(lock, &waiter, task->thread_info); + mutex_release(&lock->dep_map, 1, _RET_IP_); spin_unlock_mutex(&lock->wait_lock, flags); debug_mutex_free_waiter(&waiter); @@ -173,7 +176,7 @@ __mutex_lock_common(struct mutex *lock, long state __IP_DECL__) /* got the lock - rejoice! */ mutex_remove_waiter(lock, &waiter, task->thread_info); - debug_mutex_set_owner(lock, task->thread_info __IP__); + debug_mutex_set_owner(lock, task->thread_info); /* set it to 0 if there are no waiters left: */ if (likely(list_empty(&lock->wait_list))) @@ -183,32 +186,40 @@ __mutex_lock_common(struct mutex *lock, long state __IP_DECL__) debug_mutex_free_waiter(&waiter); - DEBUG_WARN_ON(list_empty(&lock->held_list)); - DEBUG_WARN_ON(lock->owner != task->thread_info); - return 0; } static void fastcall noinline __sched -__mutex_lock_slowpath(atomic_t *lock_count __IP_DECL__) +__mutex_lock_slowpath(atomic_t *lock_count) { struct mutex *lock = container_of(lock_count, struct mutex, count); - __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE __IP__); + __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0); +} + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +void __sched +mutex_lock_nested(struct mutex *lock, unsigned int subclass) +{ + might_sleep(); + __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass); } +EXPORT_SYMBOL_GPL(mutex_lock_nested); +#endif + /* * Release the lock, slowpath: */ -static fastcall noinline void -__mutex_unlock_slowpath(atomic_t *lock_count __IP_DECL__) +static fastcall inline void +__mutex_unlock_common_slowpath(atomic_t *lock_count, int nested) { struct mutex *lock = container_of(lock_count, struct mutex, count); unsigned long flags; - DEBUG_WARN_ON(lock->owner != current_thread_info()); - spin_lock_mutex(&lock->wait_lock, flags); + mutex_release(&lock->dep_map, nested, _RET_IP_); + debug_mutex_unlock(lock); /* * some architectures leave the lock unlocked in the fastpath failure @@ -218,8 +229,6 @@ __mutex_unlock_slowpath(atomic_t *lock_count __IP_DECL__) if (__mutex_slowpath_needs_to_unlock()) atomic_set(&lock->count, 1); - debug_mutex_unlock(lock); - if (!list_empty(&lock->wait_list)) { /* get the first entry from the wait-list: */ struct mutex_waiter *waiter = @@ -237,11 +246,20 @@ __mutex_unlock_slowpath(atomic_t *lock_count __IP_DECL__) } /* + * Release the lock, slowpath: + */ +static fastcall noinline void +__mutex_unlock_slowpath(atomic_t *lock_count) +{ + __mutex_unlock_common_slowpath(lock_count, 1); +} + +/* * Here come the less common (and hence less performance-critical) APIs: * mutex_lock_interruptible() and mutex_trylock(). */ static int fastcall noinline __sched -__mutex_lock_interruptible_slowpath(atomic_t *lock_count __IP_DECL__); +__mutex_lock_interruptible_slowpath(atomic_t *lock_count); /*** * mutex_lock_interruptible - acquire the mutex, interruptable @@ -264,11 +282,11 @@ int fastcall __sched mutex_lock_interruptible(struct mutex *lock) EXPORT_SYMBOL(mutex_lock_interruptible); static int fastcall noinline __sched -__mutex_lock_interruptible_slowpath(atomic_t *lock_count __IP_DECL__) +__mutex_lock_interruptible_slowpath(atomic_t *lock_count) { struct mutex *lock = container_of(lock_count, struct mutex, count); - return __mutex_lock_common(lock, TASK_INTERRUPTIBLE __IP__); + return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0); } /* @@ -284,8 +302,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count) spin_lock_mutex(&lock->wait_lock, flags); prev = atomic_xchg(&lock->count, -1); - if (likely(prev == 1)) - debug_mutex_set_owner(lock, current_thread_info() __RET_IP__); + if (likely(prev == 1)) { + debug_mutex_set_owner(lock, current_thread_info()); + mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); + } /* Set it back to 0 if there are no waiters: */ if (likely(list_empty(&lock->wait_list))) atomic_set(&lock->count, 0); @@ -309,7 +329,7 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count) * This function must not be used in interrupt context. The * mutex must be released by the same task that acquired it. */ -int fastcall mutex_trylock(struct mutex *lock) +int fastcall __sched mutex_trylock(struct mutex *lock) { return __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath); diff --git a/kernel/mutex.h b/kernel/mutex.h index 06918994725..a075dafbb29 100644 --- a/kernel/mutex.h +++ b/kernel/mutex.h @@ -16,22 +16,15 @@ #define mutex_remove_waiter(lock, waiter, ti) \ __list_del((waiter)->list.prev, (waiter)->list.next) -#define DEBUG_WARN_ON(c) do { } while (0) #define debug_mutex_set_owner(lock, new_owner) do { } while (0) #define debug_mutex_clear_owner(lock) do { } while (0) -#define debug_mutex_init_waiter(waiter) do { } while (0) #define debug_mutex_wake_waiter(lock, waiter) do { } while (0) #define debug_mutex_free_waiter(waiter) do { } while (0) -#define debug_mutex_add_waiter(lock, waiter, ti, ip) do { } while (0) +#define debug_mutex_add_waiter(lock, waiter, ti) do { } while (0) #define debug_mutex_unlock(lock) do { } while (0) -#define debug_mutex_init(lock, name) do { } while (0) - -/* - * Return-address parameters/declarations. They are very useful for - * debugging, but add overhead in the !DEBUG case - so we go the - * trouble of using this not too elegant but zero-cost solution: - */ -#define __IP_DECL__ -#define __IP__ -#define __RET_IP__ +#define debug_mutex_init(lock, name, key) do { } while (0) +static inline void +debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) +{ +} diff --git a/kernel/pid.c b/kernel/pid.c index eeb836b65ca..93e212f2067 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -218,7 +218,7 @@ struct pid * fastcall find_pid(int nr) return NULL; } -int fastcall attach_pid(task_t *task, enum pid_type type, int nr) +int fastcall attach_pid(struct task_struct *task, enum pid_type type, int nr) { struct pid_link *link; struct pid *pid; @@ -233,7 +233,7 @@ int fastcall attach_pid(task_t *task, enum pid_type type, int nr) return 0; } -void fastcall detach_pid(task_t *task, enum pid_type type) +void fastcall detach_pid(struct task_struct *task, enum pid_type type) { struct pid_link *link; struct pid *pid; @@ -267,7 +267,7 @@ struct task_struct * fastcall pid_task(struct pid *pid, enum pid_type type) /* * Must be called under rcu_read_lock() or with tasklist_lock read-held. */ -task_t *find_task_by_pid_type(int type, int nr) +struct task_struct *find_task_by_pid_type(int type, int nr) { return pid_task(find_pid(nr), type); } diff --git a/kernel/printk.c b/kernel/printk.c index 39ae24d2a41..bdba5d80496 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -518,7 +518,9 @@ asmlinkage int vprintk(const char *fmt, va_list args) zap_locks(); /* This stops the holder of console_sem just where we want him */ - spin_lock_irqsave(&logbuf_lock, flags); + local_irq_save(flags); + lockdep_off(); + spin_lock(&logbuf_lock); printk_cpu = smp_processor_id(); /* Emit the output into the temporary buffer */ @@ -588,7 +590,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) */ console_locked = 1; printk_cpu = UINT_MAX; - spin_unlock_irqrestore(&logbuf_lock, flags); + spin_unlock(&logbuf_lock); /* * Console drivers may assume that per-cpu resources have @@ -604,6 +606,8 @@ asmlinkage int vprintk(const char *fmt, va_list args) console_locked = 0; up(&console_sem); } + lockdep_on(); + local_irq_restore(flags); } else { /* * Someone else owns the drivers. We drop the spinlock, which @@ -611,7 +615,9 @@ asmlinkage int vprintk(const char *fmt, va_list args) * console drivers with the output which we just produced. */ printk_cpu = UINT_MAX; - spin_unlock_irqrestore(&logbuf_lock, flags); + spin_unlock(&logbuf_lock); + lockdep_on(); + local_irq_restore(flags); } preempt_enable(); @@ -809,8 +815,15 @@ void release_console_sem(void) console_may_schedule = 0; up(&console_sem); spin_unlock_irqrestore(&logbuf_lock, flags); - if (wake_klogd && !oops_in_progress && waitqueue_active(&log_wait)) - wake_up_interruptible(&log_wait); + if (wake_klogd && !oops_in_progress && waitqueue_active(&log_wait)) { + /* + * If we printk from within the lock dependency code, + * from within the scheduler code, then do not lock + * up due to self-recursion: + */ + if (!lockdep_internal()) + wake_up_interruptible(&log_wait); + } } EXPORT_SYMBOL(release_console_sem); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 335c5b932e1..9a111f70145 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -28,7 +28,7 @@ * * Must be called with the tasklist lock write-held. */ -void __ptrace_link(task_t *child, task_t *new_parent) +void __ptrace_link(struct task_struct *child, struct task_struct *new_parent) { BUG_ON(!list_empty(&child->ptrace_list)); if (child->parent == new_parent) @@ -46,7 +46,7 @@ void __ptrace_link(task_t *child, task_t *new_parent) * TASK_TRACED, resume it now. * Requires that irqs be disabled. */ -void ptrace_untrace(task_t *child) +void ptrace_untrace(struct task_struct *child) { spin_lock(&child->sighand->siglock); if (child->state == TASK_TRACED) { @@ -65,7 +65,7 @@ void ptrace_untrace(task_t *child) * * Must be called with the tasklist lock write-held. */ -void __ptrace_unlink(task_t *child) +void __ptrace_unlink(struct task_struct *child) { BUG_ON(!child->ptrace); diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index f464f5ae3f1..759805c9859 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -53,13 +53,13 @@ static struct rcu_ctrlblk rcu_ctrlblk = { .cur = -300, .completed = -300, - .lock = SPIN_LOCK_UNLOCKED, + .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), .cpumask = CPU_MASK_NONE, }; static struct rcu_ctrlblk rcu_bh_ctrlblk = { .cur = -300, .completed = -300, - .lock = SPIN_LOCK_UNLOCKED, + .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), .cpumask = CPU_MASK_NONE, }; diff --git a/kernel/rtmutex-debug.c b/kernel/rtmutex-debug.c index 4aa8a2c9f45..0c1faa950af 100644 --- a/kernel/rtmutex-debug.c +++ b/kernel/rtmutex-debug.c @@ -26,6 +26,7 @@ #include <linux/interrupt.h> #include <linux/plist.h> #include <linux/fs.h> +#include <linux/debug_locks.h> #include "rtmutex_common.h" @@ -45,8 +46,6 @@ do { \ console_verbose(); \ if (spin_is_locked(¤t->pi_lock)) \ spin_unlock(¤t->pi_lock); \ - if (spin_is_locked(¤t->held_list_lock)) \ - spin_unlock(¤t->held_list_lock); \ } \ } while (0) @@ -97,7 +96,7 @@ void deadlock_trace_off(void) rt_trace_on = 0; } -static void printk_task(task_t *p) +static void printk_task(struct task_struct *p) { if (p) printk("%16s:%5d [%p, %3d]", p->comm, p->pid, p, p->prio); @@ -105,14 +104,6 @@ static void printk_task(task_t *p) printk("<none>"); } -static void printk_task_short(task_t *p) -{ - if (p) - printk("%s/%d [%p, %3d]", p->comm, p->pid, p, p->prio); - else - printk("<none>"); -} - static void printk_lock(struct rt_mutex *lock, int print_owner) { if (lock->name) @@ -128,222 +119,6 @@ static void printk_lock(struct rt_mutex *lock, int print_owner) printk_task(rt_mutex_owner(lock)); printk("\n"); } - if (rt_mutex_owner(lock)) { - printk("... acquired at: "); - print_symbol("%s\n", lock->acquire_ip); - } -} - -static void printk_waiter(struct rt_mutex_waiter *w) -{ - printk("-------------------------\n"); - printk("| waiter struct %p:\n", w); - printk("| w->list_entry: [DP:%p/%p|SP:%p/%p|PRI:%d]\n", - w->list_entry.plist.prio_list.prev, w->list_entry.plist.prio_list.next, - w->list_entry.plist.node_list.prev, w->list_entry.plist.node_list.next, - w->list_entry.prio); - printk("| w->pi_list_entry: [DP:%p/%p|SP:%p/%p|PRI:%d]\n", - w->pi_list_entry.plist.prio_list.prev, w->pi_list_entry.plist.prio_list.next, - w->pi_list_entry.plist.node_list.prev, w->pi_list_entry.plist.node_list.next, - w->pi_list_entry.prio); - printk("\n| lock:\n"); - printk_lock(w->lock, 1); - printk("| w->ti->task:\n"); - printk_task(w->task); - printk("| blocked at: "); - print_symbol("%s\n", w->ip); - printk("-------------------------\n"); -} - -static void show_task_locks(task_t *p) -{ - switch (p->state) { - case TASK_RUNNING: printk("R"); break; - case TASK_INTERRUPTIBLE: printk("S"); break; - case TASK_UNINTERRUPTIBLE: printk("D"); break; - case TASK_STOPPED: printk("T"); break; - case EXIT_ZOMBIE: printk("Z"); break; - case EXIT_DEAD: printk("X"); break; - default: printk("?"); break; - } - printk_task(p); - if (p->pi_blocked_on) { - struct rt_mutex *lock = p->pi_blocked_on->lock; - - printk(" blocked on:"); - printk_lock(lock, 1); - } else - printk(" (not blocked)\n"); -} - -void rt_mutex_show_held_locks(task_t *task, int verbose) -{ - struct list_head *curr, *cursor = NULL; - struct rt_mutex *lock; - task_t *t; - unsigned long flags; - int count = 0; - - if (!rt_trace_on) - return; - - if (verbose) { - printk("------------------------------\n"); - printk("| showing all locks held by: | ("); - printk_task_short(task); - printk("):\n"); - printk("------------------------------\n"); - } - -next: - spin_lock_irqsave(&task->held_list_lock, flags); - list_for_each(curr, &task->held_list_head) { - if (cursor && curr != cursor) - continue; - lock = list_entry(curr, struct rt_mutex, held_list_entry); - t = rt_mutex_owner(lock); - WARN_ON(t != task); - count++; - cursor = curr->next; - spin_unlock_irqrestore(&task->held_list_lock, flags); - - printk("\n#%03d: ", count); - printk_lock(lock, 0); - goto next; - } - spin_unlock_irqrestore(&task->held_list_lock, flags); - - printk("\n"); -} - -void rt_mutex_show_all_locks(void) -{ - task_t *g, *p; - int count = 10; - int unlock = 1; - - printk("\n"); - printk("----------------------\n"); - printk("| showing all tasks: |\n"); - printk("----------------------\n"); - - /* - * Here we try to get the tasklist_lock as hard as possible, - * if not successful after 2 seconds we ignore it (but keep - * trying). This is to enable a debug printout even if a - * tasklist_lock-holding task deadlocks or crashes. - */ -retry: - if (!read_trylock(&tasklist_lock)) { - if (count == 10) - printk("hm, tasklist_lock locked, retrying... "); - if (count) { - count--; - printk(" #%d", 10-count); - mdelay(200); - goto retry; - } - printk(" ignoring it.\n"); - unlock = 0; - } - if (count != 10) - printk(" locked it.\n"); - - do_each_thread(g, p) { - show_task_locks(p); - if (!unlock) - if (read_trylock(&tasklist_lock)) - unlock = 1; - } while_each_thread(g, p); - - printk("\n"); - - printk("-----------------------------------------\n"); - printk("| showing all locks held in the system: |\n"); - printk("-----------------------------------------\n"); - - do_each_thread(g, p) { - rt_mutex_show_held_locks(p, 0); - if (!unlock) - if (read_trylock(&tasklist_lock)) - unlock = 1; - } while_each_thread(g, p); - - - printk("=============================================\n\n"); - - if (unlock) - read_unlock(&tasklist_lock); -} - -void rt_mutex_debug_check_no_locks_held(task_t *task) -{ - struct rt_mutex_waiter *w; - struct list_head *curr; - struct rt_mutex *lock; - - if (!rt_trace_on) - return; - if (!rt_prio(task->normal_prio) && rt_prio(task->prio)) { - printk("BUG: PI priority boost leaked!\n"); - printk_task(task); - printk("\n"); - } - if (list_empty(&task->held_list_head)) - return; - - spin_lock(&task->pi_lock); - plist_for_each_entry(w, &task->pi_waiters, pi_list_entry) { - TRACE_OFF(); - - printk("hm, PI interest held at exit time? Task:\n"); - printk_task(task); - printk_waiter(w); - return; - } - spin_unlock(&task->pi_lock); - - list_for_each(curr, &task->held_list_head) { - lock = list_entry(curr, struct rt_mutex, held_list_entry); - - printk("BUG: %s/%d, lock held at task exit time!\n", - task->comm, task->pid); - printk_lock(lock, 1); - if (rt_mutex_owner(lock) != task) - printk("exiting task is not even the owner??\n"); - } -} - -int rt_mutex_debug_check_no_locks_freed(const void *from, unsigned long len) -{ - const void *to = from + len; - struct list_head *curr; - struct rt_mutex *lock; - unsigned long flags; - void *lock_addr; - - if (!rt_trace_on) - return 0; - - spin_lock_irqsave(¤t->held_list_lock, flags); - list_for_each(curr, ¤t->held_list_head) { - lock = list_entry(curr, struct rt_mutex, held_list_entry); - lock_addr = lock; - if (lock_addr < from || lock_addr >= to) - continue; - TRACE_OFF(); - - printk("BUG: %s/%d, active lock [%p(%p-%p)] freed!\n", - current->comm, current->pid, lock, from, to); - dump_stack(); - printk_lock(lock, 1); - if (rt_mutex_owner(lock) != current) - printk("freeing task is not even the owner??\n"); - return 1; - } - spin_unlock_irqrestore(¤t->held_list_lock, flags); - - return 0; } void rt_mutex_debug_task_free(struct task_struct *task) @@ -395,85 +170,41 @@ void debug_rt_mutex_print_deadlock(struct rt_mutex_waiter *waiter) current->comm, current->pid); printk_lock(waiter->lock, 1); - printk("... trying at: "); - print_symbol("%s\n", waiter->ip); - printk("\n2) %s/%d is blocked on this lock:\n", task->comm, task->pid); printk_lock(waiter->deadlock_lock, 1); - rt_mutex_show_held_locks(current, 1); - rt_mutex_show_held_locks(task, 1); + debug_show_held_locks(current); + debug_show_held_locks(task); printk("\n%s/%d's [blocked] stackdump:\n\n", task->comm, task->pid); show_stack(task, NULL); printk("\n%s/%d's [current] stackdump:\n\n", current->comm, current->pid); dump_stack(); - rt_mutex_show_all_locks(); + debug_show_all_locks(); + printk("[ turning off deadlock detection." "Please report this trace. ]\n\n"); local_irq_disable(); } -void debug_rt_mutex_lock(struct rt_mutex *lock __IP_DECL__) +void debug_rt_mutex_lock(struct rt_mutex *lock) { - unsigned long flags; - - if (rt_trace_on) { - TRACE_WARN_ON_LOCKED(!list_empty(&lock->held_list_entry)); - - spin_lock_irqsave(¤t->held_list_lock, flags); - list_add_tail(&lock->held_list_entry, ¤t->held_list_head); - spin_unlock_irqrestore(¤t->held_list_lock, flags); - - lock->acquire_ip = ip; - } } void debug_rt_mutex_unlock(struct rt_mutex *lock) { - unsigned long flags; - - if (rt_trace_on) { - TRACE_WARN_ON_LOCKED(rt_mutex_owner(lock) != current); - TRACE_WARN_ON_LOCKED(list_empty(&lock->held_list_entry)); - - spin_lock_irqsave(¤t->held_list_lock, flags); - list_del_init(&lock->held_list_entry); - spin_unlock_irqrestore(¤t->held_list_lock, flags); - } + TRACE_WARN_ON_LOCKED(rt_mutex_owner(lock) != current); } -void debug_rt_mutex_proxy_lock(struct rt_mutex *lock, - struct task_struct *powner __IP_DECL__) +void +debug_rt_mutex_proxy_lock(struct rt_mutex *lock, struct task_struct *powner) { - unsigned long flags; - - if (rt_trace_on) { - TRACE_WARN_ON_LOCKED(!list_empty(&lock->held_list_entry)); - - spin_lock_irqsave(&powner->held_list_lock, flags); - list_add_tail(&lock->held_list_entry, &powner->held_list_head); - spin_unlock_irqrestore(&powner->held_list_lock, flags); - - lock->acquire_ip = ip; - } } void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock) { - unsigned long flags; - - if (rt_trace_on) { - struct task_struct *owner = rt_mutex_owner(lock); - - TRACE_WARN_ON_LOCKED(!owner); - TRACE_WARN_ON_LOCKED(list_empty(&lock->held_list_entry)); - - spin_lock_irqsave(&owner->held_list_lock, flags); - list_del_init(&lock->held_list_entry); - spin_unlock_irqrestore(&owner->held_list_lock, flags); - } + TRACE_WARN_ON_LOCKED(!rt_mutex_owner(lock)); } void debug_rt_mutex_init_waiter(struct rt_mutex_waiter *waiter) @@ -493,17 +224,15 @@ void debug_rt_mutex_free_waiter(struct rt_mutex_waiter *waiter) void debug_rt_mutex_init(struct rt_mutex *lock, const char *name) { - void *addr = lock; - - if (rt_trace_on) { - rt_mutex_debug_check_no_locks_freed(addr, - sizeof(struct rt_mutex)); - INIT_LIST_HEAD(&lock->held_list_entry); - lock->name = name; - } + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + lock->name = name; } -void rt_mutex_deadlock_account_lock(struct rt_mutex *lock, task_t *task) +void +rt_mutex_deadlock_account_lock(struct rt_mutex *lock, struct task_struct *task) { } diff --git a/kernel/rtmutex-debug.h b/kernel/rtmutex-debug.h index 7612fbc62d7..14193d596d7 100644 --- a/kernel/rtmutex-debug.h +++ b/kernel/rtmutex-debug.h @@ -9,20 +9,16 @@ * This file contains macros used solely by rtmutex.c. Debug version. */ -#define __IP_DECL__ , unsigned long ip -#define __IP__ , ip -#define __RET_IP__ , (unsigned long)__builtin_return_address(0) - extern void rt_mutex_deadlock_account_lock(struct rt_mutex *lock, struct task_struct *task); extern void rt_mutex_deadlock_account_unlock(struct task_struct *task); extern void debug_rt_mutex_init_waiter(struct rt_mutex_waiter *waiter); extern void debug_rt_mutex_free_waiter(struct rt_mutex_waiter *waiter); extern void debug_rt_mutex_init(struct rt_mutex *lock, const char *name); -extern void debug_rt_mutex_lock(struct rt_mutex *lock __IP_DECL__); +extern void debug_rt_mutex_lock(struct rt_mutex *lock); extern void debug_rt_mutex_unlock(struct rt_mutex *lock); extern void debug_rt_mutex_proxy_lock(struct rt_mutex *lock, - struct task_struct *powner __IP_DECL__); + struct task_struct *powner); extern void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock); extern void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *waiter, struct rt_mutex *lock); diff --git a/kernel/rtmutex-tester.c b/kernel/rtmutex-tester.c index e82c2f84824..494dac872a1 100644 --- a/kernel/rtmutex-tester.c +++ b/kernel/rtmutex-tester.c @@ -33,7 +33,7 @@ struct test_thread_data { }; static struct test_thread_data thread_data[MAX_RT_TEST_THREADS]; -static task_t *threads[MAX_RT_TEST_THREADS]; +static struct task_struct *threads[MAX_RT_TEST_THREADS]; static struct rt_mutex mutexes[MAX_RT_TEST_MUTEXES]; enum test_opcodes { @@ -361,8 +361,8 @@ static ssize_t sysfs_test_command(struct sys_device *dev, const char *buf, static ssize_t sysfs_test_status(struct sys_device *dev, char *buf) { struct test_thread_data *td; + struct task_struct *tsk; char *curr = buf; - task_t *tsk; int i; td = container_of(dev, struct test_thread_data, sysdev); diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c index 45d61016da5..d2ef13b485e 100644 --- a/kernel/rtmutex.c +++ b/kernel/rtmutex.c @@ -157,12 +157,11 @@ int max_lock_depth = 1024; * Decreases task's usage by one - may thus free the task. * Returns 0 or -EDEADLK. */ -static int rt_mutex_adjust_prio_chain(task_t *task, +static int rt_mutex_adjust_prio_chain(struct task_struct *task, int deadlock_detect, struct rt_mutex *orig_lock, struct rt_mutex_waiter *orig_waiter, - struct task_struct *top_task - __IP_DECL__) + struct task_struct *top_task) { struct rt_mutex *lock; struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter; @@ -283,6 +282,7 @@ static int rt_mutex_adjust_prio_chain(task_t *task, spin_unlock_irqrestore(&task->pi_lock, flags); out_put_task: put_task_struct(task); + return ret; } @@ -357,7 +357,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock) * * Must be called with lock->wait_lock held. */ -static int try_to_take_rt_mutex(struct rt_mutex *lock __IP_DECL__) +static int try_to_take_rt_mutex(struct rt_mutex *lock) { /* * We have to be careful here if the atomic speedups are @@ -384,7 +384,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock __IP_DECL__) return 0; /* We got the lock. */ - debug_rt_mutex_lock(lock __IP__); + debug_rt_mutex_lock(lock); rt_mutex_set_owner(lock, current, 0); @@ -402,13 +402,12 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock __IP_DECL__) */ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, struct rt_mutex_waiter *waiter, - int detect_deadlock - __IP_DECL__) + int detect_deadlock) { + struct task_struct *owner = rt_mutex_owner(lock); struct rt_mutex_waiter *top_waiter = waiter; - task_t *owner = rt_mutex_owner(lock); - int boost = 0, res; unsigned long flags; + int boost = 0, res; spin_lock_irqsave(¤t->pi_lock, flags); __rt_mutex_adjust_prio(current); @@ -454,7 +453,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, spin_unlock(&lock->wait_lock); res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, - current __IP__); + current); spin_lock(&lock->wait_lock); @@ -526,12 +525,12 @@ static void wakeup_next_waiter(struct rt_mutex *lock) * Must be called with lock->wait_lock held */ static void remove_waiter(struct rt_mutex *lock, - struct rt_mutex_waiter *waiter __IP_DECL__) + struct rt_mutex_waiter *waiter) { int first = (waiter == rt_mutex_top_waiter(lock)); - int boost = 0; - task_t *owner = rt_mutex_owner(lock); + struct task_struct *owner = rt_mutex_owner(lock); unsigned long flags; + int boost = 0; spin_lock_irqsave(¤t->pi_lock, flags); plist_del(&waiter->list_entry, &lock->wait_list); @@ -568,7 +567,7 @@ static void remove_waiter(struct rt_mutex *lock, spin_unlock(&lock->wait_lock); - rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current __IP__); + rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); spin_lock(&lock->wait_lock); } @@ -595,7 +594,7 @@ void rt_mutex_adjust_pi(struct task_struct *task) get_task_struct(task); spin_unlock_irqrestore(&task->pi_lock, flags); - rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task __RET_IP__); + rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task); } /* @@ -604,7 +603,7 @@ void rt_mutex_adjust_pi(struct task_struct *task) static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock __IP_DECL__) + int detect_deadlock) { struct rt_mutex_waiter waiter; int ret = 0; @@ -615,7 +614,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, spin_lock(&lock->wait_lock); /* Try to acquire the lock again: */ - if (try_to_take_rt_mutex(lock __IP__)) { + if (try_to_take_rt_mutex(lock)) { spin_unlock(&lock->wait_lock); return 0; } @@ -629,7 +628,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, for (;;) { /* Try to acquire the lock: */ - if (try_to_take_rt_mutex(lock __IP__)) + if (try_to_take_rt_mutex(lock)) break; /* @@ -653,7 +652,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, */ if (!waiter.task) { ret = task_blocks_on_rt_mutex(lock, &waiter, - detect_deadlock __IP__); + detect_deadlock); /* * If we got woken up by the owner then start loop * all over without going into schedule to try @@ -680,7 +679,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, set_current_state(TASK_RUNNING); if (unlikely(waiter.task)) - remove_waiter(lock, &waiter __IP__); + remove_waiter(lock, &waiter); /* * try_to_take_rt_mutex() sets the waiter bit @@ -711,7 +710,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, * Slow path try-lock function: */ static inline int -rt_mutex_slowtrylock(struct rt_mutex *lock __IP_DECL__) +rt_mutex_slowtrylock(struct rt_mutex *lock) { int ret = 0; @@ -719,7 +718,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock __IP_DECL__) if (likely(rt_mutex_owner(lock) != current)) { - ret = try_to_take_rt_mutex(lock __IP__); + ret = try_to_take_rt_mutex(lock); /* * try_to_take_rt_mutex() sets the lock waiters * bit unconditionally. Clean this up. @@ -769,13 +768,13 @@ rt_mutex_fastlock(struct rt_mutex *lock, int state, int detect_deadlock, int (*slowfn)(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock __IP_DECL__)) + int detect_deadlock)) { if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; } else - return slowfn(lock, state, NULL, detect_deadlock __RET_IP__); + return slowfn(lock, state, NULL, detect_deadlock); } static inline int @@ -783,24 +782,24 @@ rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, int detect_deadlock, int (*slowfn)(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock __IP_DECL__)) + int detect_deadlock)) { if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; } else - return slowfn(lock, state, timeout, detect_deadlock __RET_IP__); + return slowfn(lock, state, timeout, detect_deadlock); } static inline int rt_mutex_fasttrylock(struct rt_mutex *lock, - int (*slowfn)(struct rt_mutex *lock __IP_DECL__)) + int (*slowfn)(struct rt_mutex *lock)) { if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 1; } - return slowfn(lock __RET_IP__); + return slowfn(lock); } static inline void @@ -948,7 +947,7 @@ void rt_mutex_init_proxy_locked(struct rt_mutex *lock, struct task_struct *proxy_owner) { __rt_mutex_init(lock, NULL); - debug_rt_mutex_proxy_lock(lock, proxy_owner __RET_IP__); + debug_rt_mutex_proxy_lock(lock, proxy_owner); rt_mutex_set_owner(lock, proxy_owner, 0); rt_mutex_deadlock_account_lock(lock, proxy_owner); } diff --git a/kernel/rtmutex.h b/kernel/rtmutex.h index 1e0fca13ff7..a1a1dd06421 100644 --- a/kernel/rtmutex.h +++ b/kernel/rtmutex.h @@ -10,9 +10,6 @@ * Non-debug version. */ -#define __IP_DECL__ -#define __IP__ -#define __RET_IP__ #define rt_mutex_deadlock_check(l) (0) #define rt_mutex_deadlock_account_lock(m, t) do { } while (0) #define rt_mutex_deadlock_account_unlock(l) do { } while (0) diff --git a/kernel/rwsem.c b/kernel/rwsem.c new file mode 100644 index 00000000000..291ded556aa --- /dev/null +++ b/kernel/rwsem.c @@ -0,0 +1,147 @@ +/* kernel/rwsem.c: R/W semaphores, public implementation + * + * Written by David Howells (dhowells@redhat.com). + * Derived from asm-i386/semaphore.h + */ + +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/rwsem.h> + +#include <asm/system.h> +#include <asm/atomic.h> + +/* + * lock for reading + */ +void down_read(struct rw_semaphore *sem) +{ + might_sleep(); + rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); + + __down_read(sem); +} + +EXPORT_SYMBOL(down_read); + +/* + * trylock for reading -- returns 1 if successful, 0 if contention + */ +int down_read_trylock(struct rw_semaphore *sem) +{ + int ret = __down_read_trylock(sem); + + if (ret == 1) + rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); + return ret; +} + +EXPORT_SYMBOL(down_read_trylock); + +/* + * lock for writing + */ +void down_write(struct rw_semaphore *sem) +{ + might_sleep(); + rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); + + __down_write(sem); +} + +EXPORT_SYMBOL(down_write); + +/* + * trylock for writing -- returns 1 if successful, 0 if contention + */ +int down_write_trylock(struct rw_semaphore *sem) +{ + int ret = __down_write_trylock(sem); + + if (ret == 1) + rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); + return ret; +} + +EXPORT_SYMBOL(down_write_trylock); + +/* + * release a read lock + */ +void up_read(struct rw_semaphore *sem) +{ + rwsem_release(&sem->dep_map, 1, _RET_IP_); + + __up_read(sem); +} + +EXPORT_SYMBOL(up_read); + +/* + * release a write lock + */ +void up_write(struct rw_semaphore *sem) +{ + rwsem_release(&sem->dep_map, 1, _RET_IP_); + + __up_write(sem); +} + +EXPORT_SYMBOL(up_write); + +/* + * downgrade write lock to read lock + */ +void downgrade_write(struct rw_semaphore *sem) +{ + /* + * lockdep: a downgraded write will live on as a write + * dependency. + */ + __downgrade_write(sem); +} + +EXPORT_SYMBOL(downgrade_write); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + +void down_read_nested(struct rw_semaphore *sem, int subclass) +{ + might_sleep(); + rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); + + __down_read(sem); +} + +EXPORT_SYMBOL(down_read_nested); + +void down_read_non_owner(struct rw_semaphore *sem) +{ + might_sleep(); + + __down_read(sem); +} + +EXPORT_SYMBOL(down_read_non_owner); + +void down_write_nested(struct rw_semaphore *sem, int subclass) +{ + might_sleep(); + rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); + + __down_write_nested(sem, subclass); +} + +EXPORT_SYMBOL(down_write_nested); + +void up_read_non_owner(struct rw_semaphore *sem) +{ + __up_read(sem); +} + +EXPORT_SYMBOL(up_read_non_owner); + +#endif + + diff --git a/kernel/sched.c b/kernel/sched.c index d5e37072ea5..4ee400f9d56 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -30,6 +30,7 @@ #include <linux/capability.h> #include <linux/completion.h> #include <linux/kernel_stat.h> +#include <linux/debug_locks.h> #include <linux/security.h> #include <linux/notifier.h> #include <linux/profile.h> @@ -178,20 +179,15 @@ static unsigned int static_prio_timeslice(int static_prio) return SCALE_PRIO(DEF_TIMESLICE, static_prio); } -static inline unsigned int task_timeslice(task_t *p) +static inline unsigned int task_timeslice(struct task_struct *p) { return static_prio_timeslice(p->static_prio); } -#define task_hot(p, now, sd) ((long long) ((now) - (p)->last_ran) \ - < (long long) (sd)->cache_hot_time) - /* * These are the runqueue data structures: */ -typedef struct runqueue runqueue_t; - struct prio_array { unsigned int nr_active; DECLARE_BITMAP(bitmap, MAX_PRIO+1); /* include 1 bit for delimiter */ @@ -205,7 +201,7 @@ struct prio_array { * (such as the load balancing or the thread migration code), lock * acquire operations must be ordered by ascending &runqueue. */ -struct runqueue { +struct rq { spinlock_t lock; /* @@ -229,9 +225,9 @@ struct runqueue { unsigned long expired_timestamp; unsigned long long timestamp_last_tick; - task_t *curr, *idle; + struct task_struct *curr, *idle; struct mm_struct *prev_mm; - prio_array_t *active, *expired, arrays[2]; + struct prio_array *active, *expired, arrays[2]; int best_expired_prio; atomic_t nr_iowait; @@ -242,7 +238,7 @@ struct runqueue { int active_balance; int push_cpu; - task_t *migration_thread; + struct task_struct *migration_thread; struct list_head migration_queue; #endif @@ -265,9 +261,10 @@ struct runqueue { unsigned long ttwu_cnt; unsigned long ttwu_local; #endif + struct lock_class_key rq_lock_key; }; -static DEFINE_PER_CPU(struct runqueue, runqueues); +static DEFINE_PER_CPU(struct rq, runqueues); /* * The domain tree (rq->sd) is protected by RCU's quiescent state transition. @@ -276,8 +273,8 @@ static DEFINE_PER_CPU(struct runqueue, runqueues); * The domain tree of any CPU may only be accessed from within * preempt-disabled sections. */ -#define for_each_domain(cpu, domain) \ -for (domain = rcu_dereference(cpu_rq(cpu)->sd); domain; domain = domain->parent) +#define for_each_domain(cpu, __sd) \ + for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent) #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) #define this_rq() (&__get_cpu_var(runqueues)) @@ -292,26 +289,33 @@ for (domain = rcu_dereference(cpu_rq(cpu)->sd); domain; domain = domain->parent) #endif #ifndef __ARCH_WANT_UNLOCKED_CTXSW -static inline int task_running(runqueue_t *rq, task_t *p) +static inline int task_running(struct rq *rq, struct task_struct *p) { return rq->curr == p; } -static inline void prepare_lock_switch(runqueue_t *rq, task_t *next) +static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { } -static inline void finish_lock_switch(runqueue_t *rq, task_t *prev) +static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) { #ifdef CONFIG_DEBUG_SPINLOCK /* this is a valid case when another task releases the spinlock */ rq->lock.owner = current; #endif + /* + * If we are tracking spinlock dependencies then we have to + * fix up the runqueue lock - which gets 'carried over' from + * prev into current: + */ + spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); + spin_unlock_irq(&rq->lock); } #else /* __ARCH_WANT_UNLOCKED_CTXSW */ -static inline int task_running(runqueue_t *rq, task_t *p) +static inline int task_running(struct rq *rq, struct task_struct *p) { #ifdef CONFIG_SMP return p->oncpu; @@ -320,7 +324,7 @@ static inline int task_running(runqueue_t *rq, task_t *p) #endif } -static inline void prepare_lock_switch(runqueue_t *rq, task_t *next) +static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { #ifdef CONFIG_SMP /* @@ -337,7 +341,7 @@ static inline void prepare_lock_switch(runqueue_t *rq, task_t *next) #endif } -static inline void finish_lock_switch(runqueue_t *rq, task_t *prev) +static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) { #ifdef CONFIG_SMP /* @@ -358,10 +362,10 @@ static inline void finish_lock_switch(runqueue_t *rq, task_t *prev) * __task_rq_lock - lock the runqueue a given task resides on. * Must be called interrupts disabled. */ -static inline runqueue_t *__task_rq_lock(task_t *p) +static inline struct rq *__task_rq_lock(struct task_struct *p) __acquires(rq->lock) { - struct runqueue *rq; + struct rq *rq; repeat_lock_task: rq = task_rq(p); @@ -378,10 +382,10 @@ repeat_lock_task: * interrupts. Note the ordering: we can safely lookup the task_rq without * explicitly disabling preemption. */ -static runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) +static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) __acquires(rq->lock) { - struct runqueue *rq; + struct rq *rq; repeat_lock_task: local_irq_save(*flags); @@ -394,13 +398,13 @@ repeat_lock_task: return rq; } -static inline void __task_rq_unlock(runqueue_t *rq) +static inline void __task_rq_unlock(struct rq *rq) __releases(rq->lock) { spin_unlock(&rq->lock); } -static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags) +static inline void task_rq_unlock(struct rq *rq, unsigned long *flags) __releases(rq->lock) { spin_unlock_irqrestore(&rq->lock, *flags); @@ -420,7 +424,7 @@ static int show_schedstat(struct seq_file *seq, void *v) seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION); seq_printf(seq, "timestamp %lu\n", jiffies); for_each_online_cpu(cpu) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); #ifdef CONFIG_SMP struct sched_domain *sd; int dcnt = 0; @@ -507,10 +511,10 @@ struct file_operations proc_schedstat_operations = { /* * rq_lock - lock a given runqueue and disable interrupts. */ -static inline runqueue_t *this_rq_lock(void) +static inline struct rq *this_rq_lock(void) __acquires(rq->lock) { - runqueue_t *rq; + struct rq *rq; local_irq_disable(); rq = this_rq(); @@ -535,7 +539,7 @@ static inline runqueue_t *this_rq_lock(void) * long it was from the *first* time it was queued to the time that it * finally hit a cpu. */ -static inline void sched_info_dequeued(task_t *t) +static inline void sched_info_dequeued(struct task_struct *t) { t->sched_info.last_queued = 0; } @@ -545,10 +549,10 @@ static inline void sched_info_dequeued(task_t *t) * long it was waiting to run. We also note when it began so that we * can keep stats on how long its timeslice is. */ -static void sched_info_arrive(task_t *t) +static void sched_info_arrive(struct task_struct *t) { unsigned long now = jiffies, diff = 0; - struct runqueue *rq = task_rq(t); + struct rq *rq = task_rq(t); if (t->sched_info.last_queued) diff = now - t->sched_info.last_queued; @@ -579,7 +583,7 @@ static void sched_info_arrive(task_t *t) * the timestamp if it is already not set. It's assumed that * sched_info_dequeued() will clear that stamp when appropriate. */ -static inline void sched_info_queued(task_t *t) +static inline void sched_info_queued(struct task_struct *t) { if (!t->sched_info.last_queued) t->sched_info.last_queued = jiffies; @@ -589,9 +593,9 @@ static inline void sched_info_queued(task_t *t) * Called when a process ceases being the active-running process, either * voluntarily or involuntarily. Now we can calculate how long we ran. */ -static inline void sched_info_depart(task_t *t) +static inline void sched_info_depart(struct task_struct *t) { - struct runqueue *rq = task_rq(t); + struct rq *rq = task_rq(t); unsigned long diff = jiffies - t->sched_info.last_arrival; t->sched_info.cpu_time += diff; @@ -605,9 +609,10 @@ static inline void sched_info_depart(task_t *t) * their time slice. (This may also be called when switching to or from * the idle task.) We are only called when prev != next. */ -static inline void sched_info_switch(task_t *prev, task_t *next) +static inline void +sched_info_switch(struct task_struct *prev, struct task_struct *next) { - struct runqueue *rq = task_rq(prev); + struct rq *rq = task_rq(prev); /* * prev now departs the cpu. It's not interesting to record @@ -628,7 +633,7 @@ static inline void sched_info_switch(task_t *prev, task_t *next) /* * Adding/removing a task to/from a priority array: */ -static void dequeue_task(struct task_struct *p, prio_array_t *array) +static void dequeue_task(struct task_struct *p, struct prio_array *array) { array->nr_active--; list_del(&p->run_list); @@ -636,7 +641,7 @@ static void dequeue_task(struct task_struct *p, prio_array_t *array) __clear_bit(p->prio, array->bitmap); } -static void enqueue_task(struct task_struct *p, prio_array_t *array) +static void enqueue_task(struct task_struct *p, struct prio_array *array) { sched_info_queued(p); list_add_tail(&p->run_list, array->queue + p->prio); @@ -649,12 +654,13 @@ static void enqueue_task(struct task_struct *p, prio_array_t *array) * Put task to the end of the run list without the overhead of dequeue * followed by enqueue. */ -static void requeue_task(struct task_struct *p, prio_array_t *array) +static void requeue_task(struct task_struct *p, struct prio_array *array) { list_move_tail(&p->run_list, array->queue + p->prio); } -static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array) +static inline void +enqueue_task_head(struct task_struct *p, struct prio_array *array) { list_add(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); @@ -677,7 +683,7 @@ static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array) * Both properties are important to certain workloads. */ -static inline int __normal_prio(task_t *p) +static inline int __normal_prio(struct task_struct *p) { int bonus, prio; @@ -713,7 +719,7 @@ static inline int __normal_prio(task_t *p) #define RTPRIO_TO_LOAD_WEIGHT(rp) \ (PRIO_TO_LOAD_WEIGHT(MAX_RT_PRIO) + LOAD_WEIGHT(rp)) -static void set_load_weight(task_t *p) +static void set_load_weight(struct task_struct *p) { if (has_rt_policy(p)) { #ifdef CONFIG_SMP @@ -731,23 +737,25 @@ static void set_load_weight(task_t *p) p->load_weight = PRIO_TO_LOAD_WEIGHT(p->static_prio); } -static inline void inc_raw_weighted_load(runqueue_t *rq, const task_t *p) +static inline void +inc_raw_weighted_load(struct rq *rq, const struct task_struct *p) { rq->raw_weighted_load += p->load_weight; } -static inline void dec_raw_weighted_load(runqueue_t *rq, const task_t *p) +static inline void +dec_raw_weighted_load(struct rq *rq, const struct task_struct *p) { rq->raw_weighted_load -= p->load_weight; } -static inline void inc_nr_running(task_t *p, runqueue_t *rq) +static inline void inc_nr_running(struct task_struct *p, struct rq *rq) { rq->nr_running++; inc_raw_weighted_load(rq, p); } -static inline void dec_nr_running(task_t *p, runqueue_t *rq) +static inline void dec_nr_running(struct task_struct *p, struct rq *rq) { rq->nr_running--; dec_raw_weighted_load(rq, p); @@ -760,7 +768,7 @@ static inline void dec_nr_running(task_t *p, runqueue_t *rq) * setprio syscalls, and whenever the interactivity * estimator recalculates. */ -static inline int normal_prio(task_t *p) +static inline int normal_prio(struct task_struct *p) { int prio; @@ -778,7 +786,7 @@ static inline int normal_prio(task_t *p) * interactivity modifiers. Will be RT if the task got * RT-boosted. If not then it returns p->normal_prio. */ -static int effective_prio(task_t *p) +static int effective_prio(struct task_struct *p) { p->normal_prio = normal_prio(p); /* @@ -794,9 +802,9 @@ static int effective_prio(task_t *p) /* * __activate_task - move a task to the runqueue. */ -static void __activate_task(task_t *p, runqueue_t *rq) +static void __activate_task(struct task_struct *p, struct rq *rq) { - prio_array_t *target = rq->active; + struct prio_array *target = rq->active; if (batch_task(p)) target = rq->expired; @@ -807,7 +815,7 @@ static void __activate_task(task_t *p, runqueue_t *rq) /* * __activate_idle_task - move idle task to the _front_ of runqueue. */ -static inline void __activate_idle_task(task_t *p, runqueue_t *rq) +static inline void __activate_idle_task(struct task_struct *p, struct rq *rq) { enqueue_task_head(p, rq->active); inc_nr_running(p, rq); @@ -817,7 +825,7 @@ static inline void __activate_idle_task(task_t *p, runqueue_t *rq) * Recalculate p->normal_prio and p->prio after having slept, * updating the sleep-average too: */ -static int recalc_task_prio(task_t *p, unsigned long long now) +static int recalc_task_prio(struct task_struct *p, unsigned long long now) { /* Caller must always ensure 'now >= p->timestamp' */ unsigned long sleep_time = now - p->timestamp; @@ -889,7 +897,7 @@ static int recalc_task_prio(task_t *p, unsigned long long now) * Update all the scheduling statistics stuff. (sleep average * calculation, priority modifiers, etc.) */ -static void activate_task(task_t *p, runqueue_t *rq, int local) +static void activate_task(struct task_struct *p, struct rq *rq, int local) { unsigned long long now; @@ -897,7 +905,7 @@ static void activate_task(task_t *p, runqueue_t *rq, int local) #ifdef CONFIG_SMP if (!local) { /* Compensate for drifting sched_clock */ - runqueue_t *this_rq = this_rq(); + struct rq *this_rq = this_rq(); now = (now - this_rq->timestamp_last_tick) + rq->timestamp_last_tick; } @@ -936,7 +944,7 @@ static void activate_task(task_t *p, runqueue_t *rq, int local) /* * deactivate_task - remove a task from the runqueue. */ -static void deactivate_task(struct task_struct *p, runqueue_t *rq) +static void deactivate_task(struct task_struct *p, struct rq *rq) { dec_nr_running(p, rq); dequeue_task(p, p->array); @@ -956,7 +964,7 @@ static void deactivate_task(struct task_struct *p, runqueue_t *rq) #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG) #endif -static void resched_task(task_t *p) +static void resched_task(struct task_struct *p) { int cpu; @@ -977,7 +985,7 @@ static void resched_task(task_t *p) smp_send_reschedule(cpu); } #else -static inline void resched_task(task_t *p) +static inline void resched_task(struct task_struct *p) { assert_spin_locked(&task_rq(p)->lock); set_tsk_need_resched(p); @@ -988,7 +996,7 @@ static inline void resched_task(task_t *p) * task_curr - is this task currently executing on a CPU? * @p: the task in question. */ -inline int task_curr(const task_t *p) +inline int task_curr(const struct task_struct *p) { return cpu_curr(task_cpu(p)) == p; } @@ -1000,22 +1008,23 @@ unsigned long weighted_cpuload(const int cpu) } #ifdef CONFIG_SMP -typedef struct { +struct migration_req { struct list_head list; - task_t *task; + struct task_struct *task; int dest_cpu; struct completion done; -} migration_req_t; +}; /* * The task's runqueue lock must be held. * Returns true if you have to wait for migration thread. */ -static int migrate_task(task_t *p, int dest_cpu, migration_req_t *req) +static int +migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req) { - runqueue_t *rq = task_rq(p); + struct rq *rq = task_rq(p); /* * If the task is not on a runqueue (and not running), then @@ -1030,6 +1039,7 @@ static int migrate_task(task_t *p, int dest_cpu, migration_req_t *req) req->task = p; req->dest_cpu = dest_cpu; list_add(&req->list, &rq->migration_queue); + return 1; } @@ -1042,10 +1052,10 @@ static int migrate_task(task_t *p, int dest_cpu, migration_req_t *req) * smp_call_function() if an IPI is sent by the same process we are * waiting to become inactive. */ -void wait_task_inactive(task_t *p) +void wait_task_inactive(struct task_struct *p) { unsigned long flags; - runqueue_t *rq; + struct rq *rq; int preempted; repeat: @@ -1076,7 +1086,7 @@ repeat: * to another CPU then no harm is done and the purpose has been * achieved as well. */ -void kick_process(task_t *p) +void kick_process(struct task_struct *p) { int cpu; @@ -1096,7 +1106,7 @@ void kick_process(task_t *p) */ static inline unsigned long source_load(int cpu, int type) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); if (type == 0) return rq->raw_weighted_load; @@ -1110,7 +1120,7 @@ static inline unsigned long source_load(int cpu, int type) */ static inline unsigned long target_load(int cpu, int type) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); if (type == 0) return rq->raw_weighted_load; @@ -1123,10 +1133,10 @@ static inline unsigned long target_load(int cpu, int type) */ static inline unsigned long cpu_avg_load_per_task(int cpu) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); unsigned long n = rq->nr_running; - return n ? rq->raw_weighted_load / n : SCHED_LOAD_SCALE; + return n ? rq->raw_weighted_load / n : SCHED_LOAD_SCALE; } /* @@ -1279,7 +1289,7 @@ nextlevel: * Returns the CPU we should wake onto. */ #if defined(ARCH_HAS_SCHED_WAKE_IDLE) -static int wake_idle(int cpu, task_t *p) +static int wake_idle(int cpu, struct task_struct *p) { cpumask_t tmp; struct sched_domain *sd; @@ -1302,7 +1312,7 @@ static int wake_idle(int cpu, task_t *p) return cpu; } #else -static inline int wake_idle(int cpu, task_t *p) +static inline int wake_idle(int cpu, struct task_struct *p) { return cpu; } @@ -1322,15 +1332,15 @@ static inline int wake_idle(int cpu, task_t *p) * * returns failure only if the task is already active. */ -static int try_to_wake_up(task_t *p, unsigned int state, int sync) +static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) { int cpu, this_cpu, success = 0; unsigned long flags; long old_state; - runqueue_t *rq; + struct rq *rq; #ifdef CONFIG_SMP - unsigned long load, this_load; struct sched_domain *sd, *this_sd = NULL; + unsigned long load, this_load; int new_cpu; #endif @@ -1480,15 +1490,14 @@ out: return success; } -int fastcall wake_up_process(task_t *p) +int fastcall wake_up_process(struct task_struct *p) { return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); } - EXPORT_SYMBOL(wake_up_process); -int fastcall wake_up_state(task_t *p, unsigned int state) +int fastcall wake_up_state(struct task_struct *p, unsigned int state) { return try_to_wake_up(p, state, 0); } @@ -1497,7 +1506,7 @@ int fastcall wake_up_state(task_t *p, unsigned int state) * Perform scheduler related setup for a newly forked process p. * p is forked by current. */ -void fastcall sched_fork(task_t *p, int clone_flags) +void fastcall sched_fork(struct task_struct *p, int clone_flags) { int cpu = get_cpu(); @@ -1565,11 +1574,11 @@ void fastcall sched_fork(task_t *p, int clone_flags) * that must be done for every newly created context, then puts the task * on the runqueue and wakes it. */ -void fastcall wake_up_new_task(task_t *p, unsigned long clone_flags) +void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) { + struct rq *rq, *this_rq; unsigned long flags; int this_cpu, cpu; - runqueue_t *rq, *this_rq; rq = task_rq_lock(p, &flags); BUG_ON(p->state != TASK_RUNNING); @@ -1649,10 +1658,10 @@ void fastcall wake_up_new_task(task_t *p, unsigned long clone_flags) * artificially, because any timeslice recovered here * was given away by the parent in the first place.) */ -void fastcall sched_exit(task_t *p) +void fastcall sched_exit(struct task_struct *p) { unsigned long flags; - runqueue_t *rq; + struct rq *rq; /* * If the child was a (relative-) CPU hog then decrease @@ -1683,7 +1692,7 @@ void fastcall sched_exit(task_t *p) * prepare_task_switch sets up locking and calls architecture specific * hooks. */ -static inline void prepare_task_switch(runqueue_t *rq, task_t *next) +static inline void prepare_task_switch(struct rq *rq, struct task_struct *next) { prepare_lock_switch(rq, next); prepare_arch_switch(next); @@ -1704,7 +1713,7 @@ static inline void prepare_task_switch(runqueue_t *rq, task_t *next) * with the lock held can cause deadlocks; see schedule() for * details.) */ -static inline void finish_task_switch(runqueue_t *rq, task_t *prev) +static inline void finish_task_switch(struct rq *rq, struct task_struct *prev) __releases(rq->lock) { struct mm_struct *mm = rq->prev_mm; @@ -1742,10 +1751,11 @@ static inline void finish_task_switch(runqueue_t *rq, task_t *prev) * schedule_tail - first thing a freshly forked thread must call. * @prev: the thread we just switched away from. */ -asmlinkage void schedule_tail(task_t *prev) +asmlinkage void schedule_tail(struct task_struct *prev) __releases(rq->lock) { - runqueue_t *rq = this_rq(); + struct rq *rq = this_rq(); + finish_task_switch(rq, prev); #ifdef __ARCH_WANT_UNLOCKED_CTXSW /* In this case, finish_task_switch does not reenable preemption */ @@ -1759,8 +1769,9 @@ asmlinkage void schedule_tail(task_t *prev) * context_switch - switch to the new MM and the new * thread's register state. */ -static inline -task_t * context_switch(runqueue_t *rq, task_t *prev, task_t *next) +static inline struct task_struct * +context_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) { struct mm_struct *mm = next->mm; struct mm_struct *oldmm = prev->active_mm; @@ -1777,6 +1788,7 @@ task_t * context_switch(runqueue_t *rq, task_t *prev, task_t *next) WARN_ON(rq->prev_mm); rq->prev_mm = oldmm; } + spin_release(&rq->lock.dep_map, 1, _THIS_IP_); /* Here we just switch the register state and the stack. */ switch_to(prev, next, prev); @@ -1857,12 +1869,21 @@ unsigned long nr_active(void) #ifdef CONFIG_SMP /* + * Is this task likely cache-hot: + */ +static inline int +task_hot(struct task_struct *p, unsigned long long now, struct sched_domain *sd) +{ + return (long long)(now - p->last_ran) < (long long)sd->cache_hot_time; +} + +/* * double_rq_lock - safely lock two runqueues * * Note this does not disable interrupts like task_rq_lock, * you need to do so manually before calling. */ -static void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2) +static void double_rq_lock(struct rq *rq1, struct rq *rq2) __acquires(rq1->lock) __acquires(rq2->lock) { @@ -1886,7 +1907,7 @@ static void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2) * Note this does not restore interrupts like task_rq_unlock, * you need to do so manually after calling. */ -static void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2) +static void double_rq_unlock(struct rq *rq1, struct rq *rq2) __releases(rq1->lock) __releases(rq2->lock) { @@ -1900,7 +1921,7 @@ static void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2) /* * double_lock_balance - lock the busiest runqueue, this_rq is locked already. */ -static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest) +static void double_lock_balance(struct rq *this_rq, struct rq *busiest) __releases(this_rq->lock) __acquires(busiest->lock) __acquires(this_rq->lock) @@ -1921,11 +1942,11 @@ static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest) * allow dest_cpu, which will force the cpu onto dest_cpu. Then * the cpu_allowed mask is restored. */ -static void sched_migrate_task(task_t *p, int dest_cpu) +static void sched_migrate_task(struct task_struct *p, int dest_cpu) { - migration_req_t req; - runqueue_t *rq; + struct migration_req req; unsigned long flags; + struct rq *rq; rq = task_rq_lock(p, &flags); if (!cpu_isset(dest_cpu, p->cpus_allowed) @@ -1936,11 +1957,13 @@ static void sched_migrate_task(task_t *p, int dest_cpu) if (migrate_task(p, dest_cpu, &req)) { /* Need to wait for migration thread (might exit: take ref). */ struct task_struct *mt = rq->migration_thread; + get_task_struct(mt); task_rq_unlock(rq, &flags); wake_up_process(mt); put_task_struct(mt); wait_for_completion(&req.done); + return; } out: @@ -1964,9 +1987,9 @@ void sched_exec(void) * pull_task - move a task from a remote runqueue to the local runqueue. * Both runqueues must be locked. */ -static -void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, - runqueue_t *this_rq, prio_array_t *this_array, int this_cpu) +static void pull_task(struct rq *src_rq, struct prio_array *src_array, + struct task_struct *p, struct rq *this_rq, + struct prio_array *this_array, int this_cpu) { dequeue_task(p, src_array); dec_nr_running(p, src_rq); @@ -1987,7 +2010,7 @@ void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ static -int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu, +int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, struct sched_domain *sd, enum idle_type idle, int *all_pinned) { @@ -2019,6 +2042,7 @@ int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu, } #define rq_best_prio(rq) min((rq)->curr->prio, (rq)->best_expired_prio) + /* * move_tasks tries to move up to max_nr_move tasks and max_load_move weighted * load from busiest to this_rq, as part of a balancing operation within @@ -2026,18 +2050,17 @@ int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu, * * Called with both runqueues locked. */ -static int move_tasks(runqueue_t *this_rq, int this_cpu, runqueue_t *busiest, +static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_nr_move, unsigned long max_load_move, struct sched_domain *sd, enum idle_type idle, int *all_pinned) { - prio_array_t *array, *dst_array; + int idx, pulled = 0, pinned = 0, this_best_prio, best_prio, + best_prio_seen, skip_for_load; + struct prio_array *array, *dst_array; struct list_head *head, *curr; - int idx, pulled = 0, pinned = 0, this_best_prio, busiest_best_prio; - int busiest_best_prio_seen; - int skip_for_load; /* skip the task based on weighted load issues */ + struct task_struct *tmp; long rem_load_move; - task_t *tmp; if (max_nr_move == 0 || max_load_move == 0) goto out; @@ -2045,15 +2068,15 @@ static int move_tasks(runqueue_t *this_rq, int this_cpu, runqueue_t *busiest, rem_load_move = max_load_move; pinned = 1; this_best_prio = rq_best_prio(this_rq); - busiest_best_prio = rq_best_prio(busiest); + best_prio = rq_best_prio(busiest); /* * Enable handling of the case where there is more than one task * with the best priority. If the current running task is one - * of those with prio==busiest_best_prio we know it won't be moved + * of those with prio==best_prio we know it won't be moved * and therefore it's safe to override the skip (based on load) of * any task we find with that prio. */ - busiest_best_prio_seen = busiest_best_prio == busiest->curr->prio; + best_prio_seen = best_prio == busiest->curr->prio; /* * We first consider expired tasks. Those will likely not be @@ -2089,7 +2112,7 @@ skip_bitmap: head = array->queue + idx; curr = head->prev; skip_queue: - tmp = list_entry(curr, task_t, run_list); + tmp = list_entry(curr, struct task_struct, run_list); curr = curr->prev; @@ -2100,10 +2123,11 @@ skip_queue: */ skip_for_load = tmp->load_weight > rem_load_move; if (skip_for_load && idx < this_best_prio) - skip_for_load = !busiest_best_prio_seen && idx == busiest_best_prio; + skip_for_load = !best_prio_seen && idx == best_prio; if (skip_for_load || !can_migrate_task(tmp, busiest, this_cpu, sd, idle, &pinned)) { - busiest_best_prio_seen |= idx == busiest_best_prio; + + best_prio_seen |= idx == best_prio; if (curr != head) goto skip_queue; idx++; @@ -2146,8 +2170,8 @@ out: /* * find_busiest_group finds and returns the busiest CPU group within the - * domain. It calculates and returns the amount of weighted load which should be - * moved to restore balance via the imbalance parameter. + * domain. It calculates and returns the amount of weighted load which + * should be moved to restore balance via the imbalance parameter. */ static struct sched_group * find_busiest_group(struct sched_domain *sd, int this_cpu, @@ -2188,7 +2212,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, sum_weighted_load = sum_nr_running = avg_load = 0; for_each_cpu_mask(i, group->cpumask) { - runqueue_t *rq = cpu_rq(i); + struct rq *rq = cpu_rq(i); if (*sd_idle && !idle_cpu(i)) *sd_idle = 0; @@ -2269,7 +2293,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, * capacity but still has some space to pick up some load * from other group and save more power */ - if (sum_nr_running <= group_capacity - 1) + if (sum_nr_running <= group_capacity - 1) { if (sum_nr_running > leader_nr_running || (sum_nr_running == leader_nr_running && first_cpu(group->cpumask) > @@ -2277,7 +2301,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, group_leader = group; leader_nr_running = sum_nr_running; } - + } group_next: #endif group = group->next; @@ -2332,8 +2356,7 @@ group_next: * moved */ if (*imbalance < busiest_load_per_task) { - unsigned long pwr_now, pwr_move; - unsigned long tmp; + unsigned long tmp, pwr_now, pwr_move; unsigned int imbn; small_imbalance: @@ -2405,22 +2428,23 @@ ret: /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ -static runqueue_t *find_busiest_queue(struct sched_group *group, - enum idle_type idle, unsigned long imbalance) +static struct rq * +find_busiest_queue(struct sched_group *group, enum idle_type idle, + unsigned long imbalance) { + struct rq *busiest = NULL, *rq; unsigned long max_load = 0; - runqueue_t *busiest = NULL, *rqi; int i; for_each_cpu_mask(i, group->cpumask) { - rqi = cpu_rq(i); + rq = cpu_rq(i); - if (rqi->nr_running == 1 && rqi->raw_weighted_load > imbalance) + if (rq->nr_running == 1 && rq->raw_weighted_load > imbalance) continue; - if (rqi->raw_weighted_load > max_load) { - max_load = rqi->raw_weighted_load; - busiest = rqi; + if (rq->raw_weighted_load > max_load) { + max_load = rq->raw_weighted_load; + busiest = rq; } } @@ -2433,22 +2457,24 @@ static runqueue_t *find_busiest_queue(struct sched_group *group, */ #define MAX_PINNED_INTERVAL 512 -#define minus_1_or_zero(n) ((n) > 0 ? (n) - 1 : 0) +static inline unsigned long minus_1_or_zero(unsigned long n) +{ + return n > 0 ? n - 1 : 0; +} + /* * Check this_cpu to ensure it is balanced within domain. Attempt to move * tasks if there is an imbalance. * * Called with this_rq unlocked. */ -static int load_balance(int this_cpu, runqueue_t *this_rq, +static int load_balance(int this_cpu, struct rq *this_rq, struct sched_domain *sd, enum idle_type idle) { + int nr_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; struct sched_group *group; - runqueue_t *busiest; unsigned long imbalance; - int nr_moved, all_pinned = 0; - int active_balance = 0; - int sd_idle = 0; + struct rq *busiest; if (idle != NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER && !sched_smt_power_savings) @@ -2482,8 +2508,8 @@ static int load_balance(int this_cpu, runqueue_t *this_rq, */ double_rq_lock(this_rq, busiest); nr_moved = move_tasks(this_rq, this_cpu, busiest, - minus_1_or_zero(busiest->nr_running), - imbalance, sd, idle, &all_pinned); + minus_1_or_zero(busiest->nr_running), + imbalance, sd, idle, &all_pinned); double_rq_unlock(this_rq, busiest); /* All tasks on this runqueue were pinned by CPU affinity */ @@ -2556,7 +2582,8 @@ out_one_pinned: (sd->balance_interval < sd->max_interval)) sd->balance_interval *= 2; - if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && !sched_smt_power_savings) + if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && + !sched_smt_power_savings) return -1; return 0; } @@ -2568,11 +2595,11 @@ out_one_pinned: * Called from schedule when this_rq is about to become idle (NEWLY_IDLE). * this_rq is locked. */ -static int load_balance_newidle(int this_cpu, runqueue_t *this_rq, - struct sched_domain *sd) +static int +load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd) { struct sched_group *group; - runqueue_t *busiest = NULL; + struct rq *busiest = NULL; unsigned long imbalance; int nr_moved = 0; int sd_idle = 0; @@ -2618,9 +2645,11 @@ static int load_balance_newidle(int this_cpu, runqueue_t *this_rq, out_balanced: schedstat_inc(sd, lb_balanced[NEWLY_IDLE]); - if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && !sched_smt_power_savings) + if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && + !sched_smt_power_savings) return -1; sd->nr_balance_failed = 0; + return 0; } @@ -2628,16 +2657,15 @@ out_balanced: * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */ -static void idle_balance(int this_cpu, runqueue_t *this_rq) +static void idle_balance(int this_cpu, struct rq *this_rq) { struct sched_domain *sd; for_each_domain(this_cpu, sd) { if (sd->flags & SD_BALANCE_NEWIDLE) { - if (load_balance_newidle(this_cpu, this_rq, sd)) { - /* We've pulled tasks over so stop searching */ + /* If we've pulled tasks over stop searching: */ + if (load_balance_newidle(this_cpu, this_rq, sd)) break; - } } } } @@ -2650,14 +2678,14 @@ static void idle_balance(int this_cpu, runqueue_t *this_rq) * * Called with busiest_rq locked. */ -static void active_load_balance(runqueue_t *busiest_rq, int busiest_cpu) +static void active_load_balance(struct rq *busiest_rq, int busiest_cpu) { - struct sched_domain *sd; - runqueue_t *target_rq; int target_cpu = busiest_rq->push_cpu; + struct sched_domain *sd; + struct rq *target_rq; + /* Is there any task to move? */ if (busiest_rq->nr_running <= 1) - /* no task to move */ return; target_rq = cpu_rq(target_cpu); @@ -2675,21 +2703,20 @@ static void active_load_balance(runqueue_t *busiest_rq, int busiest_cpu) /* Search for an sd spanning us and the target CPU. */ for_each_domain(target_cpu, sd) { if ((sd->flags & SD_LOAD_BALANCE) && - cpu_isset(busiest_cpu, sd->span)) + cpu_isset(busiest_cpu, sd->span)) break; } - if (unlikely(sd == NULL)) - goto out; - - schedstat_inc(sd, alb_cnt); + if (likely(sd)) { + schedstat_inc(sd, alb_cnt); - if (move_tasks(target_rq, target_cpu, busiest_rq, 1, - RTPRIO_TO_LOAD_WEIGHT(100), sd, SCHED_IDLE, NULL)) - schedstat_inc(sd, alb_pushed); - else - schedstat_inc(sd, alb_failed); -out: + if (move_tasks(target_rq, target_cpu, busiest_rq, 1, + RTPRIO_TO_LOAD_WEIGHT(100), sd, SCHED_IDLE, + NULL)) + schedstat_inc(sd, alb_pushed); + else + schedstat_inc(sd, alb_failed); + } spin_unlock(&target_rq->lock); } @@ -2702,23 +2729,27 @@ out: * Balancing parameters are set up in arch_init_sched_domains. */ -/* Don't have all balancing operations going off at once */ -#define CPU_OFFSET(cpu) (HZ * cpu / NR_CPUS) +/* Don't have all balancing operations going off at once: */ +static inline unsigned long cpu_offset(int cpu) +{ + return jiffies + cpu * HZ / NR_CPUS; +} -static void rebalance_tick(int this_cpu, runqueue_t *this_rq, - enum idle_type idle) +static void +rebalance_tick(int this_cpu, struct rq *this_rq, enum idle_type idle) { - unsigned long old_load, this_load; - unsigned long j = jiffies + CPU_OFFSET(this_cpu); + unsigned long this_load, interval, j = cpu_offset(this_cpu); struct sched_domain *sd; - int i; + int i, scale; this_load = this_rq->raw_weighted_load; - /* Update our load */ - for (i = 0; i < 3; i++) { - unsigned long new_load = this_load; - int scale = 1 << i; + + /* Update our load: */ + for (i = 0, scale = 1; i < 3; i++, scale <<= 1) { + unsigned long old_load, new_load; + old_load = this_rq->cpu_load[i]; + new_load = this_load; /* * Round up the averaging division if load is increasing. This * prevents us from getting stuck on 9 if the load is 10, for @@ -2730,8 +2761,6 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq, } for_each_domain(this_cpu, sd) { - unsigned long interval; - if (!(sd->flags & SD_LOAD_BALANCE)) continue; @@ -2761,17 +2790,18 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq, /* * on UP we do not need to balance between CPUs: */ -static inline void rebalance_tick(int cpu, runqueue_t *rq, enum idle_type idle) +static inline void rebalance_tick(int cpu, struct rq *rq, enum idle_type idle) { } -static inline void idle_balance(int cpu, runqueue_t *rq) +static inline void idle_balance(int cpu, struct rq *rq) { } #endif -static inline int wake_priority_sleeper(runqueue_t *rq) +static inline int wake_priority_sleeper(struct rq *rq) { int ret = 0; + #ifdef CONFIG_SCHED_SMT spin_lock(&rq->lock); /* @@ -2795,25 +2825,26 @@ EXPORT_PER_CPU_SYMBOL(kstat); * This is called on clock ticks and on context switches. * Bank in p->sched_time the ns elapsed since the last tick or switch. */ -static inline void update_cpu_clock(task_t *p, runqueue_t *rq, - unsigned long long now) +static inline void +update_cpu_clock(struct task_struct *p, struct rq *rq, unsigned long long now) { - unsigned long long last = max(p->timestamp, rq->timestamp_last_tick); - p->sched_time += now - last; + p->sched_time += now - max(p->timestamp, rq->timestamp_last_tick); } /* * Return current->sched_time plus any more ns on the sched_clock * that have not yet been banked. */ -unsigned long long current_sched_time(const task_t *tsk) +unsigned long long current_sched_time(const struct task_struct *p) { unsigned long long ns; unsigned long flags; + local_irq_save(flags); - ns = max(tsk->timestamp, task_rq(tsk)->timestamp_last_tick); - ns = tsk->sched_time + (sched_clock() - ns); + ns = max(p->timestamp, task_rq(p)->timestamp_last_tick); + ns = p->sched_time + sched_clock() - ns; local_irq_restore(flags); + return ns; } @@ -2827,11 +2858,16 @@ unsigned long long current_sched_time(const task_t *tsk) * increasing number of running tasks. We also ignore the interactivity * if a better static_prio task has expired: */ -#define EXPIRED_STARVING(rq) \ - ((STARVATION_LIMIT && ((rq)->expired_timestamp && \ - (jiffies - (rq)->expired_timestamp >= \ - STARVATION_LIMIT * ((rq)->nr_running) + 1))) || \ - ((rq)->curr->static_prio > (rq)->best_expired_prio)) +static inline int expired_starving(struct rq *rq) +{ + if (rq->curr->static_prio > rq->best_expired_prio) + return 1; + if (!STARVATION_LIMIT || !rq->expired_timestamp) + return 0; + if (jiffies - rq->expired_timestamp > STARVATION_LIMIT * rq->nr_running) + return 1; + return 0; +} /* * Account user cpu time to a process. @@ -2864,7 +2900,7 @@ void account_system_time(struct task_struct *p, int hardirq_offset, cputime_t cputime) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; - runqueue_t *rq = this_rq(); + struct rq *rq = this_rq(); cputime64_t tmp; p->stime = cputime_add(p->stime, cputime); @@ -2894,7 +2930,7 @@ void account_steal_time(struct task_struct *p, cputime_t steal) { struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; cputime64_t tmp = cputime_to_cputime64(steal); - runqueue_t *rq = this_rq(); + struct rq *rq = this_rq(); if (p == rq->idle) { p->stime = cputime_add(p->stime, steal); @@ -2915,10 +2951,10 @@ void account_steal_time(struct task_struct *p, cputime_t steal) */ void scheduler_tick(void) { - int cpu = smp_processor_id(); - runqueue_t *rq = this_rq(); - task_t *p = current; unsigned long long now = sched_clock(); + struct task_struct *p = current; + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); update_cpu_clock(p, rq, now); @@ -2968,7 +3004,7 @@ void scheduler_tick(void) if (!rq->expired_timestamp) rq->expired_timestamp = jiffies; - if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) { + if (!TASK_INTERACTIVE(p) || expired_starving(rq)) { enqueue_task(p, rq->expired); if (p->static_prio < rq->best_expired_prio) rq->best_expired_prio = p->static_prio; @@ -3007,7 +3043,7 @@ out: } #ifdef CONFIG_SCHED_SMT -static inline void wakeup_busy_runqueue(runqueue_t *rq) +static inline void wakeup_busy_runqueue(struct rq *rq) { /* If an SMT runqueue is sleeping due to priority reasons wake it up */ if (rq->curr == rq->idle && rq->nr_running) @@ -3033,7 +3069,7 @@ static void wake_sleeping_dependent(int this_cpu) return; for_each_cpu_mask(i, sd->span) { - runqueue_t *smt_rq = cpu_rq(i); + struct rq *smt_rq = cpu_rq(i); if (i == this_cpu) continue; @@ -3050,7 +3086,8 @@ static void wake_sleeping_dependent(int this_cpu) * utilize, if another task runs on a sibling. This models the * slowdown effect of other tasks running on siblings: */ -static inline unsigned long smt_slice(task_t *p, struct sched_domain *sd) +static inline unsigned long +smt_slice(struct task_struct *p, struct sched_domain *sd) { return p->time_slice * (100 - sd->per_cpu_gain) / 100; } @@ -3061,7 +3098,8 @@ static inline unsigned long smt_slice(task_t *p, struct sched_domain *sd) * acquire their lock. As we only trylock the normal locking order does not * need to be obeyed. */ -static int dependent_sleeper(int this_cpu, runqueue_t *this_rq, task_t *p) +static int +dependent_sleeper(int this_cpu, struct rq *this_rq, struct task_struct *p) { struct sched_domain *tmp, *sd = NULL; int ret = 0, i; @@ -3081,8 +3119,8 @@ static int dependent_sleeper(int this_cpu, runqueue_t *this_rq, task_t *p) return 0; for_each_cpu_mask(i, sd->span) { - runqueue_t *smt_rq; - task_t *smt_curr; + struct task_struct *smt_curr; + struct rq *smt_rq; if (i == this_cpu) continue; @@ -3127,9 +3165,8 @@ unlock: static inline void wake_sleeping_dependent(int this_cpu) { } - -static inline int dependent_sleeper(int this_cpu, runqueue_t *this_rq, - task_t *p) +static inline int +dependent_sleeper(int this_cpu, struct rq *this_rq, struct task_struct *p) { return 0; } @@ -3142,12 +3179,13 @@ void fastcall add_preempt_count(int val) /* * Underflow? */ - BUG_ON((preempt_count() < 0)); + if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + return; preempt_count() += val; /* * Spinlock count overflowing soon? */ - BUG_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK-10); + DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK-10); } EXPORT_SYMBOL(add_preempt_count); @@ -3156,11 +3194,15 @@ void fastcall sub_preempt_count(int val) /* * Underflow? */ - BUG_ON(val > preempt_count()); + if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + return; /* * Is the spinlock portion underflowing? */ - BUG_ON((val < PREEMPT_MASK) && !(preempt_count() & PREEMPT_MASK)); + if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && + !(preempt_count() & PREEMPT_MASK))) + return; + preempt_count() -= val; } EXPORT_SYMBOL(sub_preempt_count); @@ -3178,14 +3220,14 @@ static inline int interactive_sleep(enum sleep_type sleep_type) */ asmlinkage void __sched schedule(void) { - long *switch_count; - task_t *prev, *next; - runqueue_t *rq; - prio_array_t *array; + struct task_struct *prev, *next; + struct prio_array *array; struct list_head *queue; unsigned long long now; unsigned long run_time; int cpu, idx, new_prio; + long *switch_count; + struct rq *rq; /* * Test if we are atomic. Since do_exit() needs to call into @@ -3275,7 +3317,7 @@ need_resched_nonpreemptible: idx = sched_find_first_bit(array->bitmap); queue = array->queue + idx; - next = list_entry(queue->next, task_t, run_list); + next = list_entry(queue->next, struct task_struct, run_list); if (!rt_task(next) && interactive_sleep(next->sleep_type)) { unsigned long long delta = now - next->timestamp; @@ -3338,7 +3380,6 @@ switch_tasks: if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) goto need_resched; } - EXPORT_SYMBOL(schedule); #ifdef CONFIG_PREEMPT @@ -3383,7 +3424,6 @@ need_resched: if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) goto need_resched; } - EXPORT_SYMBOL(preempt_schedule); /* @@ -3432,10 +3472,8 @@ need_resched: int default_wake_function(wait_queue_t *curr, unsigned mode, int sync, void *key) { - task_t *p = curr->private; - return try_to_wake_up(p, mode, sync); + return try_to_wake_up(curr->private, mode, sync); } - EXPORT_SYMBOL(default_wake_function); /* @@ -3453,13 +3491,11 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, struct list_head *tmp, *next; list_for_each_safe(tmp, next, &q->task_list) { - wait_queue_t *curr; - unsigned flags; - curr = list_entry(tmp, wait_queue_t, task_list); - flags = curr->flags; + wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list); + unsigned flags = curr->flags; + if (curr->func(curr, mode, sync, key) && - (flags & WQ_FLAG_EXCLUSIVE) && - !--nr_exclusive) + (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) break; } } @@ -3480,7 +3516,6 @@ void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode, __wake_up_common(q, mode, nr_exclusive, 0, key); spin_unlock_irqrestore(&q->lock, flags); } - EXPORT_SYMBOL(__wake_up); /* @@ -3549,6 +3584,7 @@ EXPORT_SYMBOL(complete_all); void fastcall __sched wait_for_completion(struct completion *x) { might_sleep(); + spin_lock_irq(&x->wait.lock); if (!x->done) { DECLARE_WAITQUEUE(wait, current); @@ -3693,7 +3729,6 @@ void fastcall __sched interruptible_sleep_on(wait_queue_head_t *q) schedule(); SLEEP_ON_TAIL } - EXPORT_SYMBOL(interruptible_sleep_on); long fastcall __sched @@ -3709,7 +3744,6 @@ interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) return timeout; } - EXPORT_SYMBOL(interruptible_sleep_on_timeout); void fastcall __sched sleep_on(wait_queue_head_t *q) @@ -3722,7 +3756,6 @@ void fastcall __sched sleep_on(wait_queue_head_t *q) schedule(); SLEEP_ON_TAIL } - EXPORT_SYMBOL(sleep_on); long fastcall __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) @@ -3752,11 +3785,11 @@ EXPORT_SYMBOL(sleep_on_timeout); * * Used by the rt_mutex code to implement priority inheritance logic. */ -void rt_mutex_setprio(task_t *p, int prio) +void rt_mutex_setprio(struct task_struct *p, int prio) { + struct prio_array *array; unsigned long flags; - prio_array_t *array; - runqueue_t *rq; + struct rq *rq; int oldprio; BUG_ON(prio < 0 || prio > MAX_PRIO); @@ -3793,12 +3826,12 @@ void rt_mutex_setprio(task_t *p, int prio) #endif -void set_user_nice(task_t *p, long nice) +void set_user_nice(struct task_struct *p, long nice) { - unsigned long flags; - prio_array_t *array; - runqueue_t *rq; + struct prio_array *array; int old_prio, delta; + unsigned long flags; + struct rq *rq; if (TASK_NICE(p) == nice || nice < -20 || nice > 19) return; @@ -3849,10 +3882,11 @@ EXPORT_SYMBOL(set_user_nice); * @p: task * @nice: nice value */ -int can_nice(const task_t *p, const int nice) +int can_nice(const struct task_struct *p, const int nice) { /* convert nice value [19,-20] to rlimit style value [1,40] */ int nice_rlim = 20 - nice; + return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur || capable(CAP_SYS_NICE)); } @@ -3868,8 +3902,7 @@ int can_nice(const task_t *p, const int nice) */ asmlinkage long sys_nice(int increment) { - int retval; - long nice; + long nice, retval; /* * Setpriority might change our priority at the same moment. @@ -3908,7 +3941,7 @@ asmlinkage long sys_nice(int increment) * RT tasks are offset by -200. Normal tasks are centered * around 0, value goes from -16 to +15. */ -int task_prio(const task_t *p) +int task_prio(const struct task_struct *p) { return p->prio - MAX_RT_PRIO; } @@ -3917,7 +3950,7 @@ int task_prio(const task_t *p) * task_nice - return the nice value of a given task. * @p: the task in question. */ -int task_nice(const task_t *p) +int task_nice(const struct task_struct *p) { return TASK_NICE(p); } @@ -3936,7 +3969,7 @@ int idle_cpu(int cpu) * idle_task - return the idle task for a given cpu. * @cpu: the processor in question. */ -task_t *idle_task(int cpu) +struct task_struct *idle_task(int cpu) { return cpu_rq(cpu)->idle; } @@ -3945,7 +3978,7 @@ task_t *idle_task(int cpu) * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. */ -static inline task_t *find_process_by_pid(pid_t pid) +static inline struct task_struct *find_process_by_pid(pid_t pid) { return pid ? find_task_by_pid(pid) : current; } @@ -3954,6 +3987,7 @@ static inline task_t *find_process_by_pid(pid_t pid) static void __setscheduler(struct task_struct *p, int policy, int prio) { BUG_ON(p->array); + p->policy = policy; p->rt_priority = prio; p->normal_prio = normal_prio(p); @@ -3977,11 +4011,10 @@ static void __setscheduler(struct task_struct *p, int policy, int prio) int sched_setscheduler(struct task_struct *p, int policy, struct sched_param *param) { - int retval; - int oldprio, oldpolicy = -1; - prio_array_t *array; + int retval, oldprio, oldpolicy = -1; + struct prio_array *array; unsigned long flags; - runqueue_t *rq; + struct rq *rq; /* may grab non-irq protected spin_locks */ BUG_ON(in_interrupt()); @@ -4079,9 +4112,9 @@ EXPORT_SYMBOL_GPL(sched_setscheduler); static int do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) { - int retval; struct sched_param lparam; struct task_struct *p; + int retval; if (!param || pid < 0) return -EINVAL; @@ -4097,6 +4130,7 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) read_unlock_irq(&tasklist_lock); retval = sched_setscheduler(p, policy, &lparam); put_task_struct(p); + return retval; } @@ -4132,8 +4166,8 @@ asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) */ asmlinkage long sys_sched_getscheduler(pid_t pid) { + struct task_struct *p; int retval = -EINVAL; - task_t *p; if (pid < 0) goto out_nounlock; @@ -4160,8 +4194,8 @@ out_nounlock: asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) { struct sched_param lp; + struct task_struct *p; int retval = -EINVAL; - task_t *p; if (!param || pid < 0) goto out_nounlock; @@ -4194,9 +4228,9 @@ out_unlock: long sched_setaffinity(pid_t pid, cpumask_t new_mask) { - task_t *p; - int retval; cpumask_t cpus_allowed; + struct task_struct *p; + int retval; lock_cpu_hotplug(); read_lock(&tasklist_lock); @@ -4282,8 +4316,8 @@ cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL; long sched_getaffinity(pid_t pid, cpumask_t *mask) { + struct task_struct *p; int retval; - task_t *p; lock_cpu_hotplug(); read_lock(&tasklist_lock); @@ -4342,9 +4376,8 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, */ asmlinkage long sys_sched_yield(void) { - runqueue_t *rq = this_rq_lock(); - prio_array_t *array = current->array; - prio_array_t *target = rq->expired; + struct rq *rq = this_rq_lock(); + struct prio_array *array = current->array, *target = rq->expired; schedstat_inc(rq, yld_cnt); /* @@ -4378,6 +4411,7 @@ asmlinkage long sys_sched_yield(void) * no need to preempt or enable interrupts: */ __release(rq->lock); + spin_release(&rq->lock.dep_map, 1, _THIS_IP_); _raw_spin_unlock(&rq->lock); preempt_enable_no_resched(); @@ -4441,6 +4475,7 @@ int cond_resched_lock(spinlock_t *lock) spin_lock(lock); } if (need_resched() && __resched_legal()) { + spin_release(&lock->dep_map, 1, _THIS_IP_); _raw_spin_unlock(lock); preempt_enable_no_resched(); __cond_resched(); @@ -4456,7 +4491,9 @@ int __sched cond_resched_softirq(void) BUG_ON(!in_softirq()); if (need_resched() && __resched_legal()) { - __local_bh_enable(); + raw_local_irq_disable(); + _local_bh_enable(); + raw_local_irq_enable(); __cond_resched(); local_bh_disable(); return 1; @@ -4476,7 +4513,6 @@ void __sched yield(void) set_current_state(TASK_RUNNING); sys_sched_yield(); } - EXPORT_SYMBOL(yield); /* @@ -4488,18 +4524,17 @@ EXPORT_SYMBOL(yield); */ void __sched io_schedule(void) { - struct runqueue *rq = &__raw_get_cpu_var(runqueues); + struct rq *rq = &__raw_get_cpu_var(runqueues); atomic_inc(&rq->nr_iowait); schedule(); atomic_dec(&rq->nr_iowait); } - EXPORT_SYMBOL(io_schedule); long __sched io_schedule_timeout(long timeout) { - struct runqueue *rq = &__raw_get_cpu_var(runqueues); + struct rq *rq = &__raw_get_cpu_var(runqueues); long ret; atomic_inc(&rq->nr_iowait); @@ -4566,9 +4601,9 @@ asmlinkage long sys_sched_get_priority_min(int policy) asmlinkage long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) { + struct task_struct *p; int retval = -EINVAL; struct timespec t; - task_t *p; if (pid < 0) goto out_nounlock; @@ -4596,28 +4631,32 @@ out_unlock: static inline struct task_struct *eldest_child(struct task_struct *p) { - if (list_empty(&p->children)) return NULL; + if (list_empty(&p->children)) + return NULL; return list_entry(p->children.next,struct task_struct,sibling); } static inline struct task_struct *older_sibling(struct task_struct *p) { - if (p->sibling.prev==&p->parent->children) return NULL; + if (p->sibling.prev==&p->parent->children) + return NULL; return list_entry(p->sibling.prev,struct task_struct,sibling); } static inline struct task_struct *younger_sibling(struct task_struct *p) { - if (p->sibling.next==&p->parent->children) return NULL; + if (p->sibling.next==&p->parent->children) + return NULL; return list_entry(p->sibling.next,struct task_struct,sibling); } -static void show_task(task_t *p) +static const char *stat_nam[] = { "R", "S", "D", "T", "t", "Z", "X" }; + +static void show_task(struct task_struct *p) { - task_t *relative; - unsigned state; + struct task_struct *relative; unsigned long free = 0; - static const char *stat_nam[] = { "R", "S", "D", "T", "t", "Z", "X" }; + unsigned state; printk("%-13.13s ", p->comm); state = p->state ? __ffs(p->state) + 1 : 0; @@ -4668,7 +4707,7 @@ static void show_task(task_t *p) void show_state(void) { - task_t *g, *p; + struct task_struct *g, *p; #if (BITS_PER_LONG == 32) printk("\n" @@ -4690,7 +4729,7 @@ void show_state(void) } while_each_thread(g, p); read_unlock(&tasklist_lock); - mutex_debug_show_all_locks(); + debug_show_all_locks(); } /** @@ -4701,9 +4740,9 @@ void show_state(void) * NOTE: this function does not set the idle thread's NEED_RESCHED * flag, to make booting more robust. */ -void __devinit init_idle(task_t *idle, int cpu) +void __devinit init_idle(struct task_struct *idle, int cpu) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); unsigned long flags; idle->timestamp = sched_clock(); @@ -4742,7 +4781,7 @@ cpumask_t nohz_cpu_mask = CPU_MASK_NONE; /* * This is how migration works: * - * 1) we queue a migration_req_t structure in the source CPU's + * 1) we queue a struct migration_req structure in the source CPU's * runqueue and wake up that CPU's migration thread. * 2) we down() the locked semaphore => thread blocks. * 3) migration thread wakes up (implicitly it forces the migrated @@ -4764,12 +4803,12 @@ cpumask_t nohz_cpu_mask = CPU_MASK_NONE; * task must not exit() & deallocate itself prematurely. The * call is not atomic; no spinlocks may be held. */ -int set_cpus_allowed(task_t *p, cpumask_t new_mask) +int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) { + struct migration_req req; unsigned long flags; + struct rq *rq; int ret = 0; - migration_req_t req; - runqueue_t *rq; rq = task_rq_lock(p, &flags); if (!cpus_intersects(new_mask, cpu_online_map)) { @@ -4792,9 +4831,9 @@ int set_cpus_allowed(task_t *p, cpumask_t new_mask) } out: task_rq_unlock(rq, &flags); + return ret; } - EXPORT_SYMBOL_GPL(set_cpus_allowed); /* @@ -4810,7 +4849,7 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed); */ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) { - runqueue_t *rq_dest, *rq_src; + struct rq *rq_dest, *rq_src; int ret = 0; if (unlikely(cpu_is_offline(dest_cpu))) @@ -4855,16 +4894,16 @@ out: */ static int migration_thread(void *data) { - runqueue_t *rq; int cpu = (long)data; + struct rq *rq; rq = cpu_rq(cpu); BUG_ON(rq->migration_thread != current); set_current_state(TASK_INTERRUPTIBLE); while (!kthread_should_stop()) { + struct migration_req *req; struct list_head *head; - migration_req_t *req; try_to_freeze(); @@ -4888,7 +4927,7 @@ static int migration_thread(void *data) set_current_state(TASK_INTERRUPTIBLE); continue; } - req = list_entry(head->next, migration_req_t, list); + req = list_entry(head->next, struct migration_req, list); list_del_init(head->next); spin_unlock(&rq->lock); @@ -4913,28 +4952,28 @@ wait_to_die: #ifdef CONFIG_HOTPLUG_CPU /* Figure out where task on dead CPU should go, use force if neccessary. */ -static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *tsk) +static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) { - runqueue_t *rq; unsigned long flags; - int dest_cpu; cpumask_t mask; + struct rq *rq; + int dest_cpu; restart: /* On same node? */ mask = node_to_cpumask(cpu_to_node(dead_cpu)); - cpus_and(mask, mask, tsk->cpus_allowed); + cpus_and(mask, mask, p->cpus_allowed); dest_cpu = any_online_cpu(mask); /* On any allowed CPU? */ if (dest_cpu == NR_CPUS) - dest_cpu = any_online_cpu(tsk->cpus_allowed); + dest_cpu = any_online_cpu(p->cpus_allowed); /* No more Mr. Nice Guy. */ if (dest_cpu == NR_CPUS) { - rq = task_rq_lock(tsk, &flags); - cpus_setall(tsk->cpus_allowed); - dest_cpu = any_online_cpu(tsk->cpus_allowed); + rq = task_rq_lock(p, &flags); + cpus_setall(p->cpus_allowed); + dest_cpu = any_online_cpu(p->cpus_allowed); task_rq_unlock(rq, &flags); /* @@ -4942,12 +4981,12 @@ restart: * kernel threads (both mm NULL), since they never * leave kernel. */ - if (tsk->mm && printk_ratelimit()) + if (p->mm && printk_ratelimit()) printk(KERN_INFO "process %d (%s) no " "longer affine to cpu%d\n", - tsk->pid, tsk->comm, dead_cpu); + p->pid, p->comm, dead_cpu); } - if (!__migrate_task(tsk, dead_cpu, dest_cpu)) + if (!__migrate_task(p, dead_cpu, dest_cpu)) goto restart; } @@ -4958,9 +4997,9 @@ restart: * their home CPUs. So we just add the counter to another CPU's counter, * to keep the global sum constant after CPU-down: */ -static void migrate_nr_uninterruptible(runqueue_t *rq_src) +static void migrate_nr_uninterruptible(struct rq *rq_src) { - runqueue_t *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL)); + struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL)); unsigned long flags; local_irq_save(flags); @@ -4974,48 +5013,51 @@ static void migrate_nr_uninterruptible(runqueue_t *rq_src) /* Run through task list and migrate tasks from the dead cpu. */ static void migrate_live_tasks(int src_cpu) { - struct task_struct *tsk, *t; + struct task_struct *p, *t; write_lock_irq(&tasklist_lock); - do_each_thread(t, tsk) { - if (tsk == current) + do_each_thread(t, p) { + if (p == current) continue; - if (task_cpu(tsk) == src_cpu) - move_task_off_dead_cpu(src_cpu, tsk); - } while_each_thread(t, tsk); + if (task_cpu(p) == src_cpu) + move_task_off_dead_cpu(src_cpu, p); + } while_each_thread(t, p); write_unlock_irq(&tasklist_lock); } /* Schedules idle task to be the next runnable task on current CPU. * It does so by boosting its priority to highest possible and adding it to - * the _front_ of runqueue. Used by CPU offline code. + * the _front_ of the runqueue. Used by CPU offline code. */ void sched_idle_next(void) { - int cpu = smp_processor_id(); - runqueue_t *rq = this_rq(); + int this_cpu = smp_processor_id(); + struct rq *rq = cpu_rq(this_cpu); struct task_struct *p = rq->idle; unsigned long flags; /* cpu has to be offline */ - BUG_ON(cpu_online(cpu)); + BUG_ON(cpu_online(this_cpu)); - /* Strictly not necessary since rest of the CPUs are stopped by now - * and interrupts disabled on current cpu. + /* + * Strictly not necessary since rest of the CPUs are stopped by now + * and interrupts disabled on the current cpu. */ spin_lock_irqsave(&rq->lock, flags); __setscheduler(p, SCHED_FIFO, MAX_RT_PRIO-1); - /* Add idle task to _front_ of it's priority queue */ + + /* Add idle task to the _front_ of its priority queue: */ __activate_idle_task(p, rq); spin_unlock_irqrestore(&rq->lock, flags); } -/* Ensures that the idle task is using init_mm right before its cpu goes +/* + * Ensures that the idle task is using init_mm right before its cpu goes * offline. */ void idle_task_exit(void) @@ -5029,17 +5071,17 @@ void idle_task_exit(void) mmdrop(mm); } -static void migrate_dead(unsigned int dead_cpu, task_t *tsk) +static void migrate_dead(unsigned int dead_cpu, struct task_struct *p) { - struct runqueue *rq = cpu_rq(dead_cpu); + struct rq *rq = cpu_rq(dead_cpu); /* Must be exiting, otherwise would be on tasklist. */ - BUG_ON(tsk->exit_state != EXIT_ZOMBIE && tsk->exit_state != EXIT_DEAD); + BUG_ON(p->exit_state != EXIT_ZOMBIE && p->exit_state != EXIT_DEAD); /* Cannot have done final schedule yet: would have vanished. */ - BUG_ON(tsk->flags & PF_DEAD); + BUG_ON(p->flags & PF_DEAD); - get_task_struct(tsk); + get_task_struct(p); /* * Drop lock around migration; if someone else moves it, @@ -5047,25 +5089,25 @@ static void migrate_dead(unsigned int dead_cpu, task_t *tsk) * fine. */ spin_unlock_irq(&rq->lock); - move_task_off_dead_cpu(dead_cpu, tsk); + move_task_off_dead_cpu(dead_cpu, p); spin_lock_irq(&rq->lock); - put_task_struct(tsk); + put_task_struct(p); } /* release_task() removes task from tasklist, so we won't find dead tasks. */ static void migrate_dead_tasks(unsigned int dead_cpu) { - unsigned arr, i; - struct runqueue *rq = cpu_rq(dead_cpu); + struct rq *rq = cpu_rq(dead_cpu); + unsigned int arr, i; for (arr = 0; arr < 2; arr++) { for (i = 0; i < MAX_PRIO; i++) { struct list_head *list = &rq->arrays[arr].queue[i]; + while (!list_empty(list)) - migrate_dead(dead_cpu, - list_entry(list->next, task_t, - run_list)); + migrate_dead(dead_cpu, list_entry(list->next, + struct task_struct, run_list)); } } } @@ -5075,14 +5117,13 @@ static void migrate_dead_tasks(unsigned int dead_cpu) * migration_call - callback that gets triggered when a CPU is added. * Here we can start up the necessary migration thread for the new CPU. */ -static int __cpuinit migration_call(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int __cpuinit +migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) { - int cpu = (long)hcpu; struct task_struct *p; - struct runqueue *rq; + int cpu = (long)hcpu; unsigned long flags; + struct rq *rq; switch (action) { case CPU_UP_PREPARE: @@ -5097,10 +5138,12 @@ static int __cpuinit migration_call(struct notifier_block *nfb, task_rq_unlock(rq, &flags); cpu_rq(cpu)->migration_thread = p; break; + case CPU_ONLINE: /* Strictly unneccessary, as first user will wake it. */ wake_up_process(cpu_rq(cpu)->migration_thread); break; + #ifdef CONFIG_HOTPLUG_CPU case CPU_UP_CANCELED: if (!cpu_rq(cpu)->migration_thread) @@ -5111,6 +5154,7 @@ static int __cpuinit migration_call(struct notifier_block *nfb, kthread_stop(cpu_rq(cpu)->migration_thread); cpu_rq(cpu)->migration_thread = NULL; break; + case CPU_DEAD: migrate_live_tasks(cpu); rq = cpu_rq(cpu); @@ -5131,9 +5175,10 @@ static int __cpuinit migration_call(struct notifier_block *nfb, * the requestors. */ spin_lock_irq(&rq->lock); while (!list_empty(&rq->migration_queue)) { - migration_req_t *req; + struct migration_req *req; + req = list_entry(rq->migration_queue.next, - migration_req_t, list); + struct migration_req, list); list_del_init(&req->list); complete(&req->done); } @@ -5155,10 +5200,12 @@ static struct notifier_block __cpuinitdata migration_notifier = { int __init migration_init(void) { void *cpu = (void *)(long)smp_processor_id(); - /* Start one for boot CPU. */ + + /* Start one for the boot CPU: */ migration_call(&migration_notifier, CPU_UP_PREPARE, cpu); migration_call(&migration_notifier, CPU_ONLINE, cpu); register_cpu_notifier(&migration_notifier); + return 0; } #endif @@ -5254,7 +5301,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu) } while (sd); } #else -#define sched_domain_debug(sd, cpu) {} +# define sched_domain_debug(sd, cpu) do { } while (0) #endif static int sd_degenerate(struct sched_domain *sd) @@ -5280,8 +5327,8 @@ static int sd_degenerate(struct sched_domain *sd) return 1; } -static int sd_parent_degenerate(struct sched_domain *sd, - struct sched_domain *parent) +static int +sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) { unsigned long cflags = sd->flags, pflags = parent->flags; @@ -5314,7 +5361,7 @@ static int sd_parent_degenerate(struct sched_domain *sd, */ static void cpu_attach_domain(struct sched_domain *sd, int cpu) { - runqueue_t *rq = cpu_rq(cpu); + struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; /* Remove the sched domains which do not contribute to scheduling. */ @@ -5576,8 +5623,8 @@ static void touch_cache(void *__cache, unsigned long __size) /* * Measure the cache-cost of one task migration. Returns in units of nsec. */ -static unsigned long long measure_one(void *cache, unsigned long size, - int source, int target) +static unsigned long long +measure_one(void *cache, unsigned long size, int source, int target) { cpumask_t mask, saved_mask; unsigned long long t0, t1, t2, t3, cost; @@ -5927,9 +5974,9 @@ static int find_next_best_node(int node, unsigned long *used_nodes) */ static cpumask_t sched_domain_node_span(int node) { - int i; - cpumask_t span, nodemask; DECLARE_BITMAP(used_nodes, MAX_NUMNODES); + cpumask_t span, nodemask; + int i; cpus_clear(span); bitmap_zero(used_nodes, MAX_NUMNODES); @@ -5940,6 +5987,7 @@ static cpumask_t sched_domain_node_span(int node) for (i = 1; i < SD_NODES_PER_DOMAIN; i++) { int next_node = find_next_best_node(node, used_nodes); + nodemask = node_to_cpumask(next_node); cpus_or(span, span, nodemask); } @@ -5949,19 +5997,23 @@ static cpumask_t sched_domain_node_span(int node) #endif int sched_smt_power_savings = 0, sched_mc_power_savings = 0; + /* - * At the moment, CONFIG_SCHED_SMT is never defined, but leave it in so we - * can switch it on easily if needed. + * SMT sched-domains: */ #ifdef CONFIG_SCHED_SMT static DEFINE_PER_CPU(struct sched_domain, cpu_domains); static struct sched_group sched_group_cpus[NR_CPUS]; + static int cpu_to_cpu_group(int cpu) { return cpu; } #endif +/* + * multi-core sched-domains: + */ #ifdef CONFIG_SCHED_MC static DEFINE_PER_CPU(struct sched_domain, core_domains); static struct sched_group *sched_group_core_bycpu[NR_CPUS]; @@ -5981,9 +6033,10 @@ static int cpu_to_core_group(int cpu) static DEFINE_PER_CPU(struct sched_domain, phys_domains); static struct sched_group *sched_group_phys_bycpu[NR_CPUS]; + static int cpu_to_phys_group(int cpu) { -#if defined(CONFIG_SCHED_MC) +#ifdef CONFIG_SCHED_MC cpumask_t mask = cpu_coregroup_map(cpu); return first_cpu(mask); #elif defined(CONFIG_SCHED_SMT) @@ -6529,6 +6582,7 @@ static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) { int err = 0; + #ifdef CONFIG_SCHED_SMT if (smt_capable()) err = sysfs_create_file(&cls->kset.kobj, @@ -6548,7 +6602,8 @@ static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page) { return sprintf(page, "%u\n", sched_mc_power_savings); } -static ssize_t sched_mc_power_savings_store(struct sys_device *dev, const char *buf, size_t count) +static ssize_t sched_mc_power_savings_store(struct sys_device *dev, + const char *buf, size_t count) { return sched_power_savings_store(buf, count, 0); } @@ -6561,7 +6616,8 @@ static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page) { return sprintf(page, "%u\n", sched_smt_power_savings); } -static ssize_t sched_smt_power_savings_store(struct sys_device *dev, const char *buf, size_t count) +static ssize_t sched_smt_power_savings_store(struct sys_device *dev, + const char *buf, size_t count) { return sched_power_savings_store(buf, count, 1); } @@ -6623,6 +6679,7 @@ int in_sched_functions(unsigned long addr) { /* Linker adds these: start and end of __sched functions */ extern char __sched_text_start[], __sched_text_end[]; + return in_lock_functions(addr) || (addr >= (unsigned long)__sched_text_start && addr < (unsigned long)__sched_text_end); @@ -6630,14 +6687,15 @@ int in_sched_functions(unsigned long addr) void __init sched_init(void) { - runqueue_t *rq; int i, j, k; for_each_possible_cpu(i) { - prio_array_t *array; + struct prio_array *array; + struct rq *rq; rq = cpu_rq(i); spin_lock_init(&rq->lock); + lockdep_set_class(&rq->lock, &rq->rq_lock_key); rq->nr_running = 0; rq->active = rq->arrays; rq->expired = rq->arrays + 1; @@ -6684,7 +6742,7 @@ void __init sched_init(void) #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP void __might_sleep(char *file, int line) { -#if defined(in_atomic) +#ifdef in_atomic static unsigned long prev_jiffy; /* ratelimiting */ if ((in_atomic() || irqs_disabled()) && @@ -6706,10 +6764,10 @@ EXPORT_SYMBOL(__might_sleep); #ifdef CONFIG_MAGIC_SYSRQ void normalize_rt_tasks(void) { + struct prio_array *array; struct task_struct *p; - prio_array_t *array; unsigned long flags; - runqueue_t *rq; + struct rq *rq; read_lock_irq(&tasklist_lock); for_each_process(p) { @@ -6753,7 +6811,7 @@ void normalize_rt_tasks(void) * * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! */ -task_t *curr_task(int cpu) +struct task_struct *curr_task(int cpu) { return cpu_curr(cpu); } @@ -6773,7 +6831,7 @@ task_t *curr_task(int cpu) * * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! */ -void set_curr_task(int cpu, task_t *p) +void set_curr_task(int cpu, struct task_struct *p) { cpu_curr(cpu) = p; } diff --git a/kernel/softirq.c b/kernel/softirq.c index 8f03e3b89b5..215541e26c1 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -62,6 +62,119 @@ static inline void wakeup_softirqd(void) } /* + * This one is for softirq.c-internal use, + * where hardirqs are disabled legitimately: + */ +static void __local_bh_disable(unsigned long ip) +{ + unsigned long flags; + + WARN_ON_ONCE(in_irq()); + + raw_local_irq_save(flags); + add_preempt_count(SOFTIRQ_OFFSET); + /* + * Were softirqs turned off above: + */ + if (softirq_count() == SOFTIRQ_OFFSET) + trace_softirqs_off(ip); + raw_local_irq_restore(flags); +} + +void local_bh_disable(void) +{ + __local_bh_disable((unsigned long)__builtin_return_address(0)); +} + +EXPORT_SYMBOL(local_bh_disable); + +void __local_bh_enable(void) +{ + WARN_ON_ONCE(in_irq()); + + /* + * softirqs should never be enabled by __local_bh_enable(), + * it always nests inside local_bh_enable() sections: + */ + WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET); + + sub_preempt_count(SOFTIRQ_OFFSET); +} +EXPORT_SYMBOL_GPL(__local_bh_enable); + +/* + * Special-case - softirqs can safely be enabled in + * cond_resched_softirq(), or by __do_softirq(), + * without processing still-pending softirqs: + */ +void _local_bh_enable(void) +{ + WARN_ON_ONCE(in_irq()); + WARN_ON_ONCE(!irqs_disabled()); + + if (softirq_count() == SOFTIRQ_OFFSET) + trace_softirqs_on((unsigned long)__builtin_return_address(0)); + sub_preempt_count(SOFTIRQ_OFFSET); +} + +EXPORT_SYMBOL(_local_bh_enable); + +void local_bh_enable(void) +{ + unsigned long flags; + + WARN_ON_ONCE(in_irq()); + WARN_ON_ONCE(irqs_disabled()); + + local_irq_save(flags); + /* + * Are softirqs going to be turned on now: + */ + if (softirq_count() == SOFTIRQ_OFFSET) + trace_softirqs_on((unsigned long)__builtin_return_address(0)); + /* + * Keep preemption disabled until we are done with + * softirq processing: + */ + sub_preempt_count(SOFTIRQ_OFFSET - 1); + + if (unlikely(!in_interrupt() && local_softirq_pending())) + do_softirq(); + + dec_preempt_count(); + local_irq_restore(flags); + preempt_check_resched(); +} +EXPORT_SYMBOL(local_bh_enable); + +void local_bh_enable_ip(unsigned long ip) +{ + unsigned long flags; + + WARN_ON_ONCE(in_irq()); + + local_irq_save(flags); + /* + * Are softirqs going to be turned on now: + */ + if (softirq_count() == SOFTIRQ_OFFSET) + trace_softirqs_on(ip); + /* + * Keep preemption disabled until we are done with + * softirq processing: + */ + sub_preempt_count(SOFTIRQ_OFFSET - 1); + + if (unlikely(!in_interrupt() && local_softirq_pending())) + do_softirq(); + + dec_preempt_count(); + local_irq_restore(flags); + preempt_check_resched(); +} +EXPORT_SYMBOL(local_bh_enable_ip); + +/* * We restart softirq processing MAX_SOFTIRQ_RESTART times, * and we fall back to softirqd after that. * @@ -80,8 +193,11 @@ asmlinkage void __do_softirq(void) int cpu; pending = local_softirq_pending(); + account_system_vtime(current); + + __local_bh_disable((unsigned long)__builtin_return_address(0)); + trace_softirq_enter(); - local_bh_disable(); cpu = smp_processor_id(); restart: /* Reset the pending bitmask before enabling irqs */ @@ -109,7 +225,10 @@ restart: if (pending) wakeup_softirqd(); - __local_bh_enable(); + trace_softirq_exit(); + + account_system_vtime(current); + _local_bh_enable(); } #ifndef __ARCH_HAS_DO_SOFTIRQ @@ -136,23 +255,6 @@ EXPORT_SYMBOL(do_softirq); #endif -void local_bh_enable(void) -{ - WARN_ON(irqs_disabled()); - /* - * Keep preemption disabled until we are done with - * softirq processing: - */ - sub_preempt_count(SOFTIRQ_OFFSET - 1); - - if (unlikely(!in_interrupt() && local_softirq_pending())) - do_softirq(); - - dec_preempt_count(); - preempt_check_resched(); -} -EXPORT_SYMBOL(local_bh_enable); - #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED # define invoke_softirq() __do_softirq() #else @@ -165,6 +267,7 @@ EXPORT_SYMBOL(local_bh_enable); void irq_exit(void) { account_system_vtime(current); + trace_hardirq_exit(); sub_preempt_count(IRQ_EXIT_OFFSET); if (!in_interrupt() && local_softirq_pending()) invoke_softirq(); diff --git a/kernel/spinlock.c b/kernel/spinlock.c index b31e54eadf5..bfd6ad9c033 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c @@ -13,6 +13,7 @@ #include <linux/preempt.h> #include <linux/spinlock.h> #include <linux/interrupt.h> +#include <linux/debug_locks.h> #include <linux/module.h> /* @@ -29,8 +30,10 @@ EXPORT_SYMBOL(generic__raw_read_trylock); int __lockfunc _spin_trylock(spinlock_t *lock) { preempt_disable(); - if (_raw_spin_trylock(lock)) + if (_raw_spin_trylock(lock)) { + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); return 1; + } preempt_enable(); return 0; @@ -40,8 +43,10 @@ EXPORT_SYMBOL(_spin_trylock); int __lockfunc _read_trylock(rwlock_t *lock) { preempt_disable(); - if (_raw_read_trylock(lock)) + if (_raw_read_trylock(lock)) { + rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_); return 1; + } preempt_enable(); return 0; @@ -51,19 +56,28 @@ EXPORT_SYMBOL(_read_trylock); int __lockfunc _write_trylock(rwlock_t *lock) { preempt_disable(); - if (_raw_write_trylock(lock)) + if (_raw_write_trylock(lock)) { + rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_); return 1; + } preempt_enable(); return 0; } EXPORT_SYMBOL(_write_trylock); -#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) +/* + * If lockdep is enabled then we use the non-preemption spin-ops + * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are + * not re-enabled during lock-acquire (which the preempt-spin-ops do): + */ +#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \ + defined(CONFIG_PROVE_LOCKING) void __lockfunc _read_lock(rwlock_t *lock) { preempt_disable(); + rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); _raw_read_lock(lock); } EXPORT_SYMBOL(_read_lock); @@ -74,7 +88,17 @@ unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) local_irq_save(flags); preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + /* + * On lockdep we dont want the hand-coded irq-enable of + * _raw_spin_lock_flags() code, because lockdep assumes + * that interrupts are not re-enabled during lock-acquire: + */ +#ifdef CONFIG_PROVE_LOCKING + _raw_spin_lock(lock); +#else _raw_spin_lock_flags(lock, &flags); +#endif return flags; } EXPORT_SYMBOL(_spin_lock_irqsave); @@ -83,6 +107,7 @@ void __lockfunc _spin_lock_irq(spinlock_t *lock) { local_irq_disable(); preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_spin_lock(lock); } EXPORT_SYMBOL(_spin_lock_irq); @@ -91,6 +116,7 @@ void __lockfunc _spin_lock_bh(spinlock_t *lock) { local_bh_disable(); preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_spin_lock(lock); } EXPORT_SYMBOL(_spin_lock_bh); @@ -101,6 +127,7 @@ unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) local_irq_save(flags); preempt_disable(); + rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); _raw_read_lock(lock); return flags; } @@ -110,6 +137,7 @@ void __lockfunc _read_lock_irq(rwlock_t *lock) { local_irq_disable(); preempt_disable(); + rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); _raw_read_lock(lock); } EXPORT_SYMBOL(_read_lock_irq); @@ -118,6 +146,7 @@ void __lockfunc _read_lock_bh(rwlock_t *lock) { local_bh_disable(); preempt_disable(); + rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); _raw_read_lock(lock); } EXPORT_SYMBOL(_read_lock_bh); @@ -128,6 +157,7 @@ unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) local_irq_save(flags); preempt_disable(); + rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_write_lock(lock); return flags; } @@ -137,6 +167,7 @@ void __lockfunc _write_lock_irq(rwlock_t *lock) { local_irq_disable(); preempt_disable(); + rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_write_lock(lock); } EXPORT_SYMBOL(_write_lock_irq); @@ -145,6 +176,7 @@ void __lockfunc _write_lock_bh(rwlock_t *lock) { local_bh_disable(); preempt_disable(); + rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_write_lock(lock); } EXPORT_SYMBOL(_write_lock_bh); @@ -152,6 +184,7 @@ EXPORT_SYMBOL(_write_lock_bh); void __lockfunc _spin_lock(spinlock_t *lock) { preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_spin_lock(lock); } @@ -160,6 +193,7 @@ EXPORT_SYMBOL(_spin_lock); void __lockfunc _write_lock(rwlock_t *lock) { preempt_disable(); + rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); _raw_write_lock(lock); } @@ -255,8 +289,22 @@ BUILD_LOCK_OPS(write, rwlock); #endif /* CONFIG_PREEMPT */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + +void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) +{ + preempt_disable(); + spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); + _raw_spin_lock(lock); +} + +EXPORT_SYMBOL(_spin_lock_nested); + +#endif + void __lockfunc _spin_unlock(spinlock_t *lock) { + spin_release(&lock->dep_map, 1, _RET_IP_); _raw_spin_unlock(lock); preempt_enable(); } @@ -264,6 +312,7 @@ EXPORT_SYMBOL(_spin_unlock); void __lockfunc _write_unlock(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_write_unlock(lock); preempt_enable(); } @@ -271,6 +320,7 @@ EXPORT_SYMBOL(_write_unlock); void __lockfunc _read_unlock(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_read_unlock(lock); preempt_enable(); } @@ -278,6 +328,7 @@ EXPORT_SYMBOL(_read_unlock); void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) { + spin_release(&lock->dep_map, 1, _RET_IP_); _raw_spin_unlock(lock); local_irq_restore(flags); preempt_enable(); @@ -286,6 +337,7 @@ EXPORT_SYMBOL(_spin_unlock_irqrestore); void __lockfunc _spin_unlock_irq(spinlock_t *lock) { + spin_release(&lock->dep_map, 1, _RET_IP_); _raw_spin_unlock(lock); local_irq_enable(); preempt_enable(); @@ -294,14 +346,16 @@ EXPORT_SYMBOL(_spin_unlock_irq); void __lockfunc _spin_unlock_bh(spinlock_t *lock) { + spin_release(&lock->dep_map, 1, _RET_IP_); _raw_spin_unlock(lock); preempt_enable_no_resched(); - local_bh_enable(); + local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } EXPORT_SYMBOL(_spin_unlock_bh); void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_read_unlock(lock); local_irq_restore(flags); preempt_enable(); @@ -310,6 +364,7 @@ EXPORT_SYMBOL(_read_unlock_irqrestore); void __lockfunc _read_unlock_irq(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_read_unlock(lock); local_irq_enable(); preempt_enable(); @@ -318,14 +373,16 @@ EXPORT_SYMBOL(_read_unlock_irq); void __lockfunc _read_unlock_bh(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_read_unlock(lock); preempt_enable_no_resched(); - local_bh_enable(); + local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } EXPORT_SYMBOL(_read_unlock_bh); void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_write_unlock(lock); local_irq_restore(flags); preempt_enable(); @@ -334,6 +391,7 @@ EXPORT_SYMBOL(_write_unlock_irqrestore); void __lockfunc _write_unlock_irq(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_write_unlock(lock); local_irq_enable(); preempt_enable(); @@ -342,9 +400,10 @@ EXPORT_SYMBOL(_write_unlock_irq); void __lockfunc _write_unlock_bh(rwlock_t *lock) { + rwlock_release(&lock->dep_map, 1, _RET_IP_); _raw_write_unlock(lock); preempt_enable_no_resched(); - local_bh_enable(); + local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } EXPORT_SYMBOL(_write_unlock_bh); @@ -352,11 +411,13 @@ int __lockfunc _spin_trylock_bh(spinlock_t *lock) { local_bh_disable(); preempt_disable(); - if (_raw_spin_trylock(lock)) + if (_raw_spin_trylock(lock)) { + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); return 1; + } preempt_enable_no_resched(); - local_bh_enable(); + local_bh_enable_ip((unsigned long)__builtin_return_address(0)); return 0; } EXPORT_SYMBOL(_spin_trylock_bh); diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c new file mode 100644 index 00000000000..b71816e47a3 --- /dev/null +++ b/kernel/stacktrace.c @@ -0,0 +1,24 @@ +/* + * kernel/stacktrace.c + * + * Stack trace management functions + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + */ +#include <linux/sched.h> +#include <linux/kallsyms.h> +#include <linux/stacktrace.h> + +void print_stack_trace(struct stack_trace *trace, int spaces) +{ + int i, j; + + for (i = 0; i < trace->nr_entries; i++) { + unsigned long ip = trace->entries[i]; + + for (j = 0; j < spaces + 1; j++) + printk(" "); + print_ip_sym(ip); + } +} + diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 99a58f27907..362a0cc3713 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -932,6 +932,17 @@ static ctl_table vm_table[] = { .strategy = &sysctl_intvec, .extra1 = &zero, }, + { + .ctl_name = VM_MIN_UNMAPPED, + .procname = "min_unmapped_ratio", + .data = &sysctl_min_unmapped_ratio, + .maxlen = sizeof(sysctl_min_unmapped_ratio), + .mode = 0644, + .proc_handler = &sysctl_min_unmapped_ratio_sysctl_handler, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one_hundred, + }, #endif #ifdef CONFIG_X86_32 { diff --git a/kernel/timer.c b/kernel/timer.c index 5a896025306..396a3c024c2 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1208,7 +1208,7 @@ unsigned long wall_jiffies = INITIAL_JIFFIES; * playing with xtime and avenrun. */ #ifndef ARCH_HAVE_XTIME_LOCK -seqlock_t xtime_lock __cacheline_aligned_in_smp = SEQLOCK_UNLOCKED; +__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); EXPORT_SYMBOL(xtime_lock); #endif @@ -1368,7 +1368,7 @@ asmlinkage long sys_getegid(void) static void process_timeout(unsigned long __data) { - wake_up_process((task_t *)__data); + wake_up_process((struct task_struct *)__data); } /** @@ -1559,6 +1559,13 @@ asmlinkage long sys_sysinfo(struct sysinfo __user *info) return 0; } +/* + * lockdep: we want to track each per-CPU base as a separate lock-class, + * but timer-bases are kmalloc()-ed, so we need to attach separate + * keys to them: + */ +static struct lock_class_key base_lock_keys[NR_CPUS]; + static int __devinit init_timers_cpu(int cpu) { int j; @@ -1594,6 +1601,8 @@ static int __devinit init_timers_cpu(int cpu) } spin_lock_init(&base->lock); + lockdep_set_class(&base->lock, base_lock_keys + cpu); + for (j = 0; j < TVN_SIZE; j++) { INIT_LIST_HEAD(base->tv5.vec + j); INIT_LIST_HEAD(base->tv4.vec + j); diff --git a/kernel/wait.c b/kernel/wait.c index 5985d866531..a1d57aeb7f7 100644 --- a/kernel/wait.c +++ b/kernel/wait.c @@ -10,6 +10,10 @@ #include <linux/wait.h> #include <linux/hash.h> +struct lock_class_key waitqueue_lock_key; + +EXPORT_SYMBOL(waitqueue_lock_key); + void fastcall add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) { unsigned long flags; diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 59f0b42bd89..90d2c600165 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -51,7 +51,7 @@ struct cpu_workqueue_struct { wait_queue_head_t work_done; struct workqueue_struct *wq; - task_t *thread; + struct task_struct *thread; int run_depth; /* Detect run_workqueue() recursion depth */ } ____cacheline_aligned; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index e4fcbd12cf6..e5889b1a33f 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -48,7 +48,7 @@ config DEBUG_KERNEL config LOG_BUF_SHIFT int "Kernel log buffer size (16 => 64KB, 17 => 128KB)" if DEBUG_KERNEL range 12 21 - default 17 if S390 + default 17 if S390 || LOCKDEP default 16 if X86_NUMAQ || IA64 default 15 if SMP default 14 @@ -107,7 +107,7 @@ config DEBUG_SLAB_LEAK config DEBUG_PREEMPT bool "Debug preemptible kernel" - depends on DEBUG_KERNEL && PREEMPT + depends on DEBUG_KERNEL && PREEMPT && TRACE_IRQFLAGS_SUPPORT default y help If you say Y here then the kernel will use a debug variant of the @@ -115,14 +115,6 @@ config DEBUG_PREEMPT if kernel code uses it in a preemption-unsafe way. Also, the kernel will detect preemption count underflows. -config DEBUG_MUTEXES - bool "Mutex debugging, deadlock detection" - default n - depends on DEBUG_KERNEL - help - This allows mutex semantics violations and mutex related deadlocks - (lockups) to be detected and reported automatically. - config DEBUG_RT_MUTEXES bool "RT Mutex debugging, deadlock detection" depends on DEBUG_KERNEL && RT_MUTEXES @@ -142,7 +134,7 @@ config RT_MUTEX_TESTER This option enables a rt-mutex tester. config DEBUG_SPINLOCK - bool "Spinlock debugging" + bool "Spinlock and rw-lock debugging: basic checks" depends on DEBUG_KERNEL help Say Y here and build SMP to catch missing spinlock initialization @@ -150,13 +142,122 @@ config DEBUG_SPINLOCK best used in conjunction with the NMI watchdog so that spinlock deadlocks are also debuggable. +config DEBUG_MUTEXES + bool "Mutex debugging: basic checks" + depends on DEBUG_KERNEL + help + This feature allows mutex semantics violations to be detected and + reported. + +config DEBUG_RWSEMS + bool "RW-sem debugging: basic checks" + depends on DEBUG_KERNEL + help + This feature allows read-write semaphore semantics violations to + be detected and reported. + +config DEBUG_LOCK_ALLOC + bool "Lock debugging: detect incorrect freeing of live locks" + depends on TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT + select DEBUG_SPINLOCK + select DEBUG_MUTEXES + select DEBUG_RWSEMS + select LOCKDEP + help + This feature will check whether any held lock (spinlock, rwlock, + mutex or rwsem) is incorrectly freed by the kernel, via any of the + memory-freeing routines (kfree(), kmem_cache_free(), free_pages(), + vfree(), etc.), whether a live lock is incorrectly reinitialized via + spin_lock_init()/mutex_init()/etc., or whether there is any lock + held during task exit. + +config PROVE_LOCKING + bool "Lock debugging: prove locking correctness" + depends on TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT + select LOCKDEP + select DEBUG_SPINLOCK + select DEBUG_MUTEXES + select DEBUG_RWSEMS + select DEBUG_LOCK_ALLOC + default n + help + This feature enables the kernel to prove that all locking + that occurs in the kernel runtime is mathematically + correct: that under no circumstance could an arbitrary (and + not yet triggered) combination of observed locking + sequences (on an arbitrary number of CPUs, running an + arbitrary number of tasks and interrupt contexts) cause a + deadlock. + + In short, this feature enables the kernel to report locking + related deadlocks before they actually occur. + + The proof does not depend on how hard and complex a + deadlock scenario would be to trigger: how many + participant CPUs, tasks and irq-contexts would be needed + for it to trigger. The proof also does not depend on + timing: if a race and a resulting deadlock is possible + theoretically (no matter how unlikely the race scenario + is), it will be proven so and will immediately be + reported by the kernel (once the event is observed that + makes the deadlock theoretically possible). + + If a deadlock is impossible (i.e. the locking rules, as + observed by the kernel, are mathematically correct), the + kernel reports nothing. + + NOTE: this feature can also be enabled for rwlocks, mutexes + and rwsems - in which case all dependencies between these + different locking variants are observed and mapped too, and + the proof of observed correctness is also maintained for an + arbitrary combination of these separate locking variants. + + For more details, see Documentation/lockdep-design.txt. + +config LOCKDEP + bool + depends on TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT + select STACKTRACE + select FRAME_POINTER + select KALLSYMS + select KALLSYMS_ALL + +config DEBUG_LOCKDEP + bool "Lock dependency engine debugging" + depends on LOCKDEP + help + If you say Y here, the lock dependency engine will do + additional runtime checks to debug itself, at the price + of more runtime overhead. + +config TRACE_IRQFLAGS + bool + default y + depends on TRACE_IRQFLAGS_SUPPORT + depends on PROVE_LOCKING + config DEBUG_SPINLOCK_SLEEP - bool "Sleep-inside-spinlock checking" + bool "Spinlock debugging: sleep-inside-spinlock checking" depends on DEBUG_KERNEL help If you say Y here, various routines which may sleep will become very noisy if they are called with a spinlock held. +config DEBUG_LOCKING_API_SELFTESTS + bool "Locking API boot-time self-tests" + depends on DEBUG_KERNEL + help + Say Y here if you want the kernel to run a short self-test during + bootup. The self-test checks whether common types of locking bugs + are detected by debugging mechanisms or not. (if you disable + lock debugging then those bugs wont be detected of course.) + The following locking APIs are covered: spinlocks, rwlocks, + mutexes and rwsems. + +config STACKTRACE + bool + depends on STACKTRACE_SUPPORT + config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL @@ -212,7 +313,7 @@ config DEBUG_VM config FRAME_POINTER bool "Compile the kernel with frame pointers" - depends on DEBUG_KERNEL && (X86 || CRIS || M68K || M68KNOMMU || FRV || UML) + depends on DEBUG_KERNEL && (X86 || CRIS || M68K || M68KNOMMU || FRV || UML || S390) default y if DEBUG_INFO && UML help If you say Y here the resulting kernel image will be slightly larger diff --git a/lib/Makefile b/lib/Makefile index 10c13c9d782..be9719ae82d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -11,13 +11,14 @@ lib-$(CONFIG_SMP) += cpumask.o lib-y += kobject.o kref.o kobject_uevent.o klist.o -obj-y += sort.o parser.o halfmd4.o iomap_copy.o +obj-y += sort.o parser.o halfmd4.o iomap_copy.o debug_locks.o ifeq ($(CONFIG_DEBUG_KOBJECT),y) CFLAGS_kobject.o += -DDEBUG CFLAGS_kobject_uevent.o += -DDEBUG endif +obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o diff --git a/lib/debug_locks.c b/lib/debug_locks.c new file mode 100644 index 00000000000..0ef01d14727 --- /dev/null +++ b/lib/debug_locks.c @@ -0,0 +1,45 @@ +/* + * lib/debug_locks.c + * + * Generic place for common debugging facilities for various locks: + * spinlocks, rwlocks, mutexes and rwsems. + * + * Started by Ingo Molnar: + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + */ +#include <linux/rwsem.h> +#include <linux/mutex.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <linux/debug_locks.h> + +/* + * We want to turn all lock-debugging facilities on/off at once, + * via a global flag. The reason is that once a single bug has been + * detected and reported, there might be cascade of followup bugs + * that would just muddy the log. So we report the first one and + * shut up after that. + */ +int debug_locks = 1; + +/* + * The locking-testsuite uses <debug_locks_silent> to get a + * 'silent failure': nothing is printed to the console when + * a locking bug is detected. + */ +int debug_locks_silent; + +/* + * Generic 'turn off all lock debugging' function: + */ +int debug_locks_off(void) +{ + if (xchg(&debug_locks, 0)) { + if (!debug_locks_silent) { + console_verbose(); + return 1; + } + } + return 0; +} diff --git a/lib/kernel_lock.c b/lib/kernel_lock.c index e713e86811a..e0fdfddb406 100644 --- a/lib/kernel_lock.c +++ b/lib/kernel_lock.c @@ -177,7 +177,12 @@ static inline void __lock_kernel(void) static inline void __unlock_kernel(void) { - spin_unlock(&kernel_flag); + /* + * the BKL is not covered by lockdep, so we open-code the + * unlocking sequence (and thus avoid the dep-chain ops): + */ + _raw_spin_unlock(&kernel_flag); + preempt_enable(); } /* diff --git a/lib/locking-selftest-hardirq.h b/lib/locking-selftest-hardirq.h new file mode 100644 index 00000000000..10d4a150b25 --- /dev/null +++ b/lib/locking-selftest-hardirq.h @@ -0,0 +1,9 @@ +#undef IRQ_DISABLE +#undef IRQ_ENABLE +#undef IRQ_ENTER +#undef IRQ_EXIT + +#define IRQ_ENABLE HARDIRQ_ENABLE +#define IRQ_DISABLE HARDIRQ_DISABLE +#define IRQ_ENTER HARDIRQ_ENTER +#define IRQ_EXIT HARDIRQ_EXIT diff --git a/lib/locking-selftest-mutex.h b/lib/locking-selftest-mutex.h new file mode 100644 index 00000000000..68601b6f584 --- /dev/null +++ b/lib/locking-selftest-mutex.h @@ -0,0 +1,11 @@ +#undef LOCK +#define LOCK ML + +#undef UNLOCK +#define UNLOCK MU + +#undef RLOCK +#undef WLOCK + +#undef INIT +#define INIT MI diff --git a/lib/locking-selftest-rlock-hardirq.h b/lib/locking-selftest-rlock-hardirq.h new file mode 100644 index 00000000000..9f517ebcb78 --- /dev/null +++ b/lib/locking-selftest-rlock-hardirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-rlock.h" +#include "locking-selftest-hardirq.h" diff --git a/lib/locking-selftest-rlock-softirq.h b/lib/locking-selftest-rlock-softirq.h new file mode 100644 index 00000000000..981455db7ff --- /dev/null +++ b/lib/locking-selftest-rlock-softirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-rlock.h" +#include "locking-selftest-softirq.h" diff --git a/lib/locking-selftest-rlock.h b/lib/locking-selftest-rlock.h new file mode 100644 index 00000000000..6789044f4d0 --- /dev/null +++ b/lib/locking-selftest-rlock.h @@ -0,0 +1,14 @@ +#undef LOCK +#define LOCK RL + +#undef UNLOCK +#define UNLOCK RU + +#undef RLOCK +#define RLOCK RL + +#undef WLOCK +#define WLOCK WL + +#undef INIT +#define INIT RWI diff --git a/lib/locking-selftest-rsem.h b/lib/locking-selftest-rsem.h new file mode 100644 index 00000000000..62da886680c --- /dev/null +++ b/lib/locking-selftest-rsem.h @@ -0,0 +1,14 @@ +#undef LOCK +#define LOCK RSL + +#undef UNLOCK +#define UNLOCK RSU + +#undef RLOCK +#define RLOCK RSL + +#undef WLOCK +#define WLOCK WSL + +#undef INIT +#define INIT RWSI diff --git a/lib/locking-selftest-softirq.h b/lib/locking-selftest-softirq.h new file mode 100644 index 00000000000..a83de2a04ac --- /dev/null +++ b/lib/locking-selftest-softirq.h @@ -0,0 +1,9 @@ +#undef IRQ_DISABLE +#undef IRQ_ENABLE +#undef IRQ_ENTER +#undef IRQ_EXIT + +#define IRQ_DISABLE SOFTIRQ_DISABLE +#define IRQ_ENABLE SOFTIRQ_ENABLE +#define IRQ_ENTER SOFTIRQ_ENTER +#define IRQ_EXIT SOFTIRQ_EXIT diff --git a/lib/locking-selftest-spin-hardirq.h b/lib/locking-selftest-spin-hardirq.h new file mode 100644 index 00000000000..693198dce30 --- /dev/null +++ b/lib/locking-selftest-spin-hardirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-spin.h" +#include "locking-selftest-hardirq.h" diff --git a/lib/locking-selftest-spin-softirq.h b/lib/locking-selftest-spin-softirq.h new file mode 100644 index 00000000000..c472e2a87ff --- /dev/null +++ b/lib/locking-selftest-spin-softirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-spin.h" +#include "locking-selftest-softirq.h" diff --git a/lib/locking-selftest-spin.h b/lib/locking-selftest-spin.h new file mode 100644 index 00000000000..ccd1b4b0975 --- /dev/null +++ b/lib/locking-selftest-spin.h @@ -0,0 +1,11 @@ +#undef LOCK +#define LOCK L + +#undef UNLOCK +#define UNLOCK U + +#undef RLOCK +#undef WLOCK + +#undef INIT +#define INIT SI diff --git a/lib/locking-selftest-wlock-hardirq.h b/lib/locking-selftest-wlock-hardirq.h new file mode 100644 index 00000000000..2dd2e5122ca --- /dev/null +++ b/lib/locking-selftest-wlock-hardirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-wlock.h" +#include "locking-selftest-hardirq.h" diff --git a/lib/locking-selftest-wlock-softirq.h b/lib/locking-selftest-wlock-softirq.h new file mode 100644 index 00000000000..cb80d1cb944 --- /dev/null +++ b/lib/locking-selftest-wlock-softirq.h @@ -0,0 +1,2 @@ +#include "locking-selftest-wlock.h" +#include "locking-selftest-softirq.h" diff --git a/lib/locking-selftest-wlock.h b/lib/locking-selftest-wlock.h new file mode 100644 index 00000000000..0815322d99e --- /dev/null +++ b/lib/locking-selftest-wlock.h @@ -0,0 +1,14 @@ +#undef LOCK +#define LOCK WL + +#undef UNLOCK +#define UNLOCK WU + +#undef RLOCK +#define RLOCK RL + +#undef WLOCK +#define WLOCK WL + +#undef INIT +#define INIT RWI diff --git a/lib/locking-selftest-wsem.h b/lib/locking-selftest-wsem.h new file mode 100644 index 00000000000..b88c5f2dc5f --- /dev/null +++ b/lib/locking-selftest-wsem.h @@ -0,0 +1,14 @@ +#undef LOCK +#define LOCK WSL + +#undef UNLOCK +#define UNLOCK WSU + +#undef RLOCK +#define RLOCK RSL + +#undef WLOCK +#define WLOCK WSL + +#undef INIT +#define INIT RWSI diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c new file mode 100644 index 00000000000..7945787f439 --- /dev/null +++ b/lib/locking-selftest.c @@ -0,0 +1,1216 @@ +/* + * lib/locking-selftest.c + * + * Testsuite for various locking APIs: spinlocks, rwlocks, + * mutexes and rw-semaphores. + * + * It is checking both false positives and false negatives. + * + * Started by Ingo Molnar: + * + * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> + */ +#include <linux/rwsem.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/lockdep.h> +#include <linux/spinlock.h> +#include <linux/kallsyms.h> +#include <linux/interrupt.h> +#include <linux/debug_locks.h> +#include <linux/irqflags.h> + +/* + * Change this to 1 if you want to see the failure printouts: + */ +static unsigned int debug_locks_verbose; + +static int __init setup_debug_locks_verbose(char *str) +{ + get_option(&str, &debug_locks_verbose); + + return 1; +} + +__setup("debug_locks_verbose=", setup_debug_locks_verbose); + +#define FAILURE 0 +#define SUCCESS 1 + +#define LOCKTYPE_SPIN 0x1 +#define LOCKTYPE_RWLOCK 0x2 +#define LOCKTYPE_MUTEX 0x4 +#define LOCKTYPE_RWSEM 0x8 + +/* + * Normal standalone locks, for the circular and irq-context + * dependency tests: + */ +static DEFINE_SPINLOCK(lock_A); +static DEFINE_SPINLOCK(lock_B); +static DEFINE_SPINLOCK(lock_C); +static DEFINE_SPINLOCK(lock_D); + +static DEFINE_RWLOCK(rwlock_A); +static DEFINE_RWLOCK(rwlock_B); +static DEFINE_RWLOCK(rwlock_C); +static DEFINE_RWLOCK(rwlock_D); + +static DEFINE_MUTEX(mutex_A); +static DEFINE_MUTEX(mutex_B); +static DEFINE_MUTEX(mutex_C); +static DEFINE_MUTEX(mutex_D); + +static DECLARE_RWSEM(rwsem_A); +static DECLARE_RWSEM(rwsem_B); +static DECLARE_RWSEM(rwsem_C); +static DECLARE_RWSEM(rwsem_D); + +/* + * Locks that we initialize dynamically as well so that + * e.g. X1 and X2 becomes two instances of the same class, + * but X* and Y* are different classes. We do this so that + * we do not trigger a real lockup: + */ +static DEFINE_SPINLOCK(lock_X1); +static DEFINE_SPINLOCK(lock_X2); +static DEFINE_SPINLOCK(lock_Y1); +static DEFINE_SPINLOCK(lock_Y2); +static DEFINE_SPINLOCK(lock_Z1); +static DEFINE_SPINLOCK(lock_Z2); + +static DEFINE_RWLOCK(rwlock_X1); +static DEFINE_RWLOCK(rwlock_X2); +static DEFINE_RWLOCK(rwlock_Y1); +static DEFINE_RWLOCK(rwlock_Y2); +static DEFINE_RWLOCK(rwlock_Z1); +static DEFINE_RWLOCK(rwlock_Z2); + +static DEFINE_MUTEX(mutex_X1); +static DEFINE_MUTEX(mutex_X2); +static DEFINE_MUTEX(mutex_Y1); +static DEFINE_MUTEX(mutex_Y2); +static DEFINE_MUTEX(mutex_Z1); +static DEFINE_MUTEX(mutex_Z2); + +static DECLARE_RWSEM(rwsem_X1); +static DECLARE_RWSEM(rwsem_X2); +static DECLARE_RWSEM(rwsem_Y1); +static DECLARE_RWSEM(rwsem_Y2); +static DECLARE_RWSEM(rwsem_Z1); +static DECLARE_RWSEM(rwsem_Z2); + +/* + * non-inlined runtime initializers, to let separate locks share + * the same lock-class: + */ +#define INIT_CLASS_FUNC(class) \ +static noinline void \ +init_class_##class(spinlock_t *lock, rwlock_t *rwlock, struct mutex *mutex, \ + struct rw_semaphore *rwsem) \ +{ \ + spin_lock_init(lock); \ + rwlock_init(rwlock); \ + mutex_init(mutex); \ + init_rwsem(rwsem); \ +} + +INIT_CLASS_FUNC(X) +INIT_CLASS_FUNC(Y) +INIT_CLASS_FUNC(Z) + +static void init_shared_classes(void) +{ + init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1); + init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2); + + init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1); + init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2); + + init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1); + init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2); +} + +/* + * For spinlocks and rwlocks we also do hardirq-safe / softirq-safe tests. + * The following functions use a lock from a simulated hardirq/softirq + * context, causing the locks to be marked as hardirq-safe/softirq-safe: + */ + +#define HARDIRQ_DISABLE local_irq_disable +#define HARDIRQ_ENABLE local_irq_enable + +#define HARDIRQ_ENTER() \ + local_irq_disable(); \ + irq_enter(); \ + WARN_ON(!in_irq()); + +#define HARDIRQ_EXIT() \ + __irq_exit(); \ + local_irq_enable(); + +#define SOFTIRQ_DISABLE local_bh_disable +#define SOFTIRQ_ENABLE local_bh_enable + +#define SOFTIRQ_ENTER() \ + local_bh_disable(); \ + local_irq_disable(); \ + trace_softirq_enter(); \ + WARN_ON(!in_softirq()); + +#define SOFTIRQ_EXIT() \ + trace_softirq_exit(); \ + local_irq_enable(); \ + local_bh_enable(); + +/* + * Shortcuts for lock/unlock API variants, to keep + * the testcases compact: + */ +#define L(x) spin_lock(&lock_##x) +#define U(x) spin_unlock(&lock_##x) +#define LU(x) L(x); U(x) +#define SI(x) spin_lock_init(&lock_##x) + +#define WL(x) write_lock(&rwlock_##x) +#define WU(x) write_unlock(&rwlock_##x) +#define WLU(x) WL(x); WU(x) + +#define RL(x) read_lock(&rwlock_##x) +#define RU(x) read_unlock(&rwlock_##x) +#define RLU(x) RL(x); RU(x) +#define RWI(x) rwlock_init(&rwlock_##x) + +#define ML(x) mutex_lock(&mutex_##x) +#define MU(x) mutex_unlock(&mutex_##x) +#define MI(x) mutex_init(&mutex_##x) + +#define WSL(x) down_write(&rwsem_##x) +#define WSU(x) up_write(&rwsem_##x) + +#define RSL(x) down_read(&rwsem_##x) +#define RSU(x) up_read(&rwsem_##x) +#define RWSI(x) init_rwsem(&rwsem_##x) + +#define LOCK_UNLOCK_2(x,y) LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x) + +/* + * Generate different permutations of the same testcase, using + * the same basic lock-dependency/state events: + */ + +#define GENERATE_TESTCASE(name) \ + \ +static void name(void) { E(); } + +#define GENERATE_PERMUTATIONS_2_EVENTS(name) \ + \ +static void name##_12(void) { E1(); E2(); } \ +static void name##_21(void) { E2(); E1(); } + +#define GENERATE_PERMUTATIONS_3_EVENTS(name) \ + \ +static void name##_123(void) { E1(); E2(); E3(); } \ +static void name##_132(void) { E1(); E3(); E2(); } \ +static void name##_213(void) { E2(); E1(); E3(); } \ +static void name##_231(void) { E2(); E3(); E1(); } \ +static void name##_312(void) { E3(); E1(); E2(); } \ +static void name##_321(void) { E3(); E2(); E1(); } + +/* + * AA deadlock: + */ + +#define E() \ + \ + LOCK(X1); \ + LOCK(X2); /* this one should fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(AA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(AA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(AA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(AA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(AA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(AA_rsem) + +#undef E + +/* + * Special-case for read-locking, they are + * allowed to recurse on the same lock class: + */ +static void rlock_AA1(void) +{ + RL(X1); + RL(X1); // this one should NOT fail +} + +static void rlock_AA1B(void) +{ + RL(X1); + RL(X2); // this one should NOT fail +} + +static void rsem_AA1(void) +{ + RSL(X1); + RSL(X1); // this one should fail +} + +static void rsem_AA1B(void) +{ + RSL(X1); + RSL(X2); // this one should fail +} +/* + * The mixing of read and write locks is not allowed: + */ +static void rlock_AA2(void) +{ + RL(X1); + WL(X2); // this one should fail +} + +static void rsem_AA2(void) +{ + RSL(X1); + WSL(X2); // this one should fail +} + +static void rlock_AA3(void) +{ + WL(X1); + RL(X2); // this one should fail +} + +static void rsem_AA3(void) +{ + WSL(X1); + RSL(X2); // this one should fail +} + +/* + * ABBA deadlock: + */ + +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(B, A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABBA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABBA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABBA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABBA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABBA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABBA_rsem) + +#undef E + +/* + * AB BC CA deadlock: + */ + +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(B, C); \ + LOCK_UNLOCK_2(C, A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABBCCA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABBCCA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABBCCA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABBCCA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABBCCA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABBCCA_rsem) + +#undef E + +/* + * AB CA BC deadlock: + */ + +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(C, A); \ + LOCK_UNLOCK_2(B, C); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABCABC_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABCABC_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABCABC_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABCABC_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABCABC_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABCABC_rsem) + +#undef E + +/* + * AB BC CD DA deadlock: + */ + +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(B, C); \ + LOCK_UNLOCK_2(C, D); \ + LOCK_UNLOCK_2(D, A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABBCCDDA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABBCCDDA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABBCCDDA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABBCCDDA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABBCCDDA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABBCCDDA_rsem) + +#undef E + +/* + * AB CD BD DA deadlock: + */ +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(C, D); \ + LOCK_UNLOCK_2(B, D); \ + LOCK_UNLOCK_2(D, A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABCDBDDA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABCDBDDA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABCDBDDA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABCDBDDA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABCDBDDA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABCDBDDA_rsem) + +#undef E + +/* + * AB CD BC DA deadlock: + */ +#define E() \ + \ + LOCK_UNLOCK_2(A, B); \ + LOCK_UNLOCK_2(C, D); \ + LOCK_UNLOCK_2(B, C); \ + LOCK_UNLOCK_2(D, A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(ABCDBCDA_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(ABCDBCDA_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(ABCDBCDA_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(ABCDBCDA_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(ABCDBCDA_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(ABCDBCDA_rsem) + +#undef E + +/* + * Double unlock: + */ +#define E() \ + \ + LOCK(A); \ + UNLOCK(A); \ + UNLOCK(A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(double_unlock_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(double_unlock_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(double_unlock_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(double_unlock_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(double_unlock_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(double_unlock_rsem) + +#undef E + +/* + * Bad unlock ordering: + */ +#define E() \ + \ + LOCK(A); \ + LOCK(B); \ + UNLOCK(A); /* fail */ \ + UNLOCK(B); + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(bad_unlock_order_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(bad_unlock_order_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(bad_unlock_order_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(bad_unlock_order_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(bad_unlock_order_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(bad_unlock_order_rsem) + +#undef E + +/* + * initializing a held lock: + */ +#define E() \ + \ + LOCK(A); \ + INIT(A); /* fail */ + +/* + * 6 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_TESTCASE(init_held_spin) +#include "locking-selftest-wlock.h" +GENERATE_TESTCASE(init_held_wlock) +#include "locking-selftest-rlock.h" +GENERATE_TESTCASE(init_held_rlock) +#include "locking-selftest-mutex.h" +GENERATE_TESTCASE(init_held_mutex) +#include "locking-selftest-wsem.h" +GENERATE_TESTCASE(init_held_wsem) +#include "locking-selftest-rsem.h" +GENERATE_TESTCASE(init_held_rsem) + +#undef E + +/* + * locking an irq-safe lock with irqs enabled: + */ +#define E1() \ + \ + IRQ_ENTER(); \ + LOCK(A); \ + UNLOCK(A); \ + IRQ_EXIT(); + +#define E2() \ + \ + LOCK(A); \ + UNLOCK(A); + +/* + * Generate 24 testcases: + */ +#include "locking-selftest-spin-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) + +#include "locking-selftest-rlock-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock) + +#include "locking-selftest-wlock-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock) + +#include "locking-selftest-spin-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin) + +#include "locking-selftest-rlock-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock) + +#include "locking-selftest-wlock-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock) + +#undef E1 +#undef E2 + +/* + * Enabling hardirqs with a softirq-safe lock held: + */ +#define E1() \ + \ + SOFTIRQ_ENTER(); \ + LOCK(A); \ + UNLOCK(A); \ + SOFTIRQ_EXIT(); + +#define E2() \ + \ + HARDIRQ_DISABLE(); \ + LOCK(A); \ + HARDIRQ_ENABLE(); \ + UNLOCK(A); + +/* + * Generate 12 testcases: + */ +#include "locking-selftest-spin.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin) + +#include "locking-selftest-wlock.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock) + +#include "locking-selftest-rlock.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock) + +#undef E1 +#undef E2 + +/* + * Enabling irqs with an irq-safe lock held: + */ +#define E1() \ + \ + IRQ_ENTER(); \ + LOCK(A); \ + UNLOCK(A); \ + IRQ_EXIT(); + +#define E2() \ + \ + IRQ_DISABLE(); \ + LOCK(A); \ + IRQ_ENABLE(); \ + UNLOCK(A); + +/* + * Generate 24 testcases: + */ +#include "locking-selftest-spin-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin) + +#include "locking-selftest-rlock-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock) + +#include "locking-selftest-wlock-hardirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock) + +#include "locking-selftest-spin-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin) + +#include "locking-selftest-rlock-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock) + +#include "locking-selftest-wlock-softirq.h" +GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock) + +#undef E1 +#undef E2 + +/* + * Acquiring a irq-unsafe lock while holding an irq-safe-lock: + */ +#define E1() \ + \ + LOCK(A); \ + LOCK(B); \ + UNLOCK(B); \ + UNLOCK(A); \ + +#define E2() \ + \ + LOCK(B); \ + UNLOCK(B); + +#define E3() \ + \ + IRQ_ENTER(); \ + LOCK(A); \ + UNLOCK(A); \ + IRQ_EXIT(); + +/* + * Generate 36 testcases: + */ +#include "locking-selftest-spin-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin) + +#include "locking-selftest-rlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock) + +#include "locking-selftest-wlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock) + +#include "locking-selftest-spin-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin) + +#include "locking-selftest-rlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock) + +#include "locking-selftest-wlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock) + +#undef E1 +#undef E2 +#undef E3 + +/* + * If a lock turns into softirq-safe, but earlier it took + * a softirq-unsafe lock: + */ + +#define E1() \ + IRQ_DISABLE(); \ + LOCK(A); \ + LOCK(B); \ + UNLOCK(B); \ + UNLOCK(A); \ + IRQ_ENABLE(); + +#define E2() \ + LOCK(B); \ + UNLOCK(B); + +#define E3() \ + IRQ_ENTER(); \ + LOCK(A); \ + UNLOCK(A); \ + IRQ_EXIT(); + +/* + * Generate 36 testcases: + */ +#include "locking-selftest-spin-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin) + +#include "locking-selftest-rlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock) + +#include "locking-selftest-wlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock) + +#include "locking-selftest-spin-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin) + +#include "locking-selftest-rlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock) + +#include "locking-selftest-wlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock) + +#undef E1 +#undef E2 +#undef E3 + +/* + * read-lock / write-lock irq inversion. + * + * Deadlock scenario: + * + * CPU#1 is at #1, i.e. it has write-locked A, but has not + * taken B yet. + * + * CPU#2 is at #2, i.e. it has locked B. + * + * Hardirq hits CPU#2 at point #2 and is trying to read-lock A. + * + * The deadlock occurs because CPU#1 will spin on B, and CPU#2 + * will spin on A. + */ + +#define E1() \ + \ + IRQ_DISABLE(); \ + WL(A); \ + LOCK(B); \ + UNLOCK(B); \ + WU(A); \ + IRQ_ENABLE(); + +#define E2() \ + \ + LOCK(B); \ + UNLOCK(B); + +#define E3() \ + \ + IRQ_ENTER(); \ + RL(A); \ + RU(A); \ + IRQ_EXIT(); + +/* + * Generate 36 testcases: + */ +#include "locking-selftest-spin-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin) + +#include "locking-selftest-rlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock) + +#include "locking-selftest-wlock-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock) + +#include "locking-selftest-spin-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin) + +#include "locking-selftest-rlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock) + +#include "locking-selftest-wlock-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock) + +#undef E1 +#undef E2 +#undef E3 + +/* + * read-lock / write-lock recursion that is actually safe. + */ + +#define E1() \ + \ + IRQ_DISABLE(); \ + WL(A); \ + WU(A); \ + IRQ_ENABLE(); + +#define E2() \ + \ + RL(A); \ + RU(A); \ + +#define E3() \ + \ + IRQ_ENTER(); \ + RL(A); \ + L(B); \ + U(B); \ + RU(A); \ + IRQ_EXIT(); + +/* + * Generate 12 testcases: + */ +#include "locking-selftest-hardirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard) + +#include "locking-selftest-softirq.h" +GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) + +#undef E1 +#undef E2 +#undef E3 + +/* + * read-lock / write-lock recursion that is unsafe. + */ + +#define E1() \ + \ + IRQ_DISABLE(); \ + L(B); \ + WL(A); \ + WU(A); \ + U(B); \ + IRQ_ENABLE(); + +#define E2() \ + \ + RL(A); \ + RU(A); \ + +#define E3() \ + \ + IRQ_ENTER(); \ + L(B); \ + U(B); \ + IRQ_EXIT(); + +/* + * Generate 12 testcases: + */ +#include "locking-selftest-hardirq.h" +// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard) + +#include "locking-selftest-softirq.h" +// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft) + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define I_SPINLOCK(x) lockdep_reset_lock(&lock_##x.dep_map) +# define I_RWLOCK(x) lockdep_reset_lock(&rwlock_##x.dep_map) +# define I_MUTEX(x) lockdep_reset_lock(&mutex_##x.dep_map) +# define I_RWSEM(x) lockdep_reset_lock(&rwsem_##x.dep_map) +#else +# define I_SPINLOCK(x) +# define I_RWLOCK(x) +# define I_MUTEX(x) +# define I_RWSEM(x) +#endif + +#define I1(x) \ + do { \ + I_SPINLOCK(x); \ + I_RWLOCK(x); \ + I_MUTEX(x); \ + I_RWSEM(x); \ + } while (0) + +#define I2(x) \ + do { \ + spin_lock_init(&lock_##x); \ + rwlock_init(&rwlock_##x); \ + mutex_init(&mutex_##x); \ + init_rwsem(&rwsem_##x); \ + } while (0) + +static void reset_locks(void) +{ + local_irq_disable(); + I1(A); I1(B); I1(C); I1(D); + I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2); + lockdep_reset(); + I2(A); I2(B); I2(C); I2(D); + init_shared_classes(); + local_irq_enable(); +} + +#undef I + +static int testcase_total; +static int testcase_successes; +static int expected_testcase_failures; +static int unexpected_testcase_failures; + +static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) +{ + unsigned long saved_preempt_count = preempt_count(); + int expected_failure = 0; + + WARN_ON(irqs_disabled()); + + testcase_fn(); + /* + * Filter out expected failures: + */ +#ifndef CONFIG_PROVE_LOCKING + if ((lockclass_mask & LOCKTYPE_SPIN) && debug_locks != expected) + expected_failure = 1; + if ((lockclass_mask & LOCKTYPE_RWLOCK) && debug_locks != expected) + expected_failure = 1; + if ((lockclass_mask & LOCKTYPE_MUTEX) && debug_locks != expected) + expected_failure = 1; + if ((lockclass_mask & LOCKTYPE_RWSEM) && debug_locks != expected) + expected_failure = 1; +#endif + if (debug_locks != expected) { + if (expected_failure) { + expected_testcase_failures++; + printk("failed|"); + } else { + unexpected_testcase_failures++; + printk("FAILED|"); + } + } else { + testcase_successes++; + printk(" ok |"); + } + testcase_total++; + + if (debug_locks_verbose) + printk(" lockclass mask: %x, debug_locks: %d, expected: %d\n", + lockclass_mask, debug_locks, expected); + /* + * Some tests (e.g. double-unlock) might corrupt the preemption + * count, so restore it: + */ + preempt_count() = saved_preempt_count; +#ifdef CONFIG_TRACE_IRQFLAGS + if (softirq_count()) + current->softirqs_enabled = 0; + else + current->softirqs_enabled = 1; +#endif + + reset_locks(); +} + +static inline void print_testname(const char *testname) +{ + printk("%33s:", testname); +} + +#define DO_TESTCASE_1(desc, name, nr) \ + print_testname(desc"/"#nr); \ + dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ + printk("\n"); + +#define DO_TESTCASE_1B(desc, name, nr) \ + print_testname(desc"/"#nr); \ + dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \ + printk("\n"); + +#define DO_TESTCASE_3(desc, name, nr) \ + print_testname(desc"/"#nr); \ + dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN); \ + dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ + printk("\n"); + +#define DO_TESTCASE_3RW(desc, name, nr) \ + print_testname(desc"/"#nr); \ + dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\ + dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \ + printk("\n"); + +#define DO_TESTCASE_6(desc, name) \ + print_testname(desc); \ + dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \ + dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \ + dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \ + dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \ + printk("\n"); + +#define DO_TESTCASE_6_SUCCESS(desc, name) \ + print_testname(desc); \ + dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN); \ + dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX); \ + dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM); \ + dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM); \ + printk("\n"); + +/* + * 'read' variant: rlocks must not trigger. + */ +#define DO_TESTCASE_6R(desc, name) \ + print_testname(desc); \ + dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \ + dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK); \ + dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK); \ + dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \ + dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \ + dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \ + printk("\n"); + +#define DO_TESTCASE_2I(desc, name, nr) \ + DO_TESTCASE_1("hard-"desc, name##_hard, nr); \ + DO_TESTCASE_1("soft-"desc, name##_soft, nr); + +#define DO_TESTCASE_2IB(desc, name, nr) \ + DO_TESTCASE_1B("hard-"desc, name##_hard, nr); \ + DO_TESTCASE_1B("soft-"desc, name##_soft, nr); + +#define DO_TESTCASE_6I(desc, name, nr) \ + DO_TESTCASE_3("hard-"desc, name##_hard, nr); \ + DO_TESTCASE_3("soft-"desc, name##_soft, nr); + +#define DO_TESTCASE_6IRW(desc, name, nr) \ + DO_TESTCASE_3RW("hard-"desc, name##_hard, nr); \ + DO_TESTCASE_3RW("soft-"desc, name##_soft, nr); + +#define DO_TESTCASE_2x3(desc, name) \ + DO_TESTCASE_3(desc, name, 12); \ + DO_TESTCASE_3(desc, name, 21); + +#define DO_TESTCASE_2x6(desc, name) \ + DO_TESTCASE_6I(desc, name, 12); \ + DO_TESTCASE_6I(desc, name, 21); + +#define DO_TESTCASE_6x2(desc, name) \ + DO_TESTCASE_2I(desc, name, 123); \ + DO_TESTCASE_2I(desc, name, 132); \ + DO_TESTCASE_2I(desc, name, 213); \ + DO_TESTCASE_2I(desc, name, 231); \ + DO_TESTCASE_2I(desc, name, 312); \ + DO_TESTCASE_2I(desc, name, 321); + +#define DO_TESTCASE_6x2B(desc, name) \ + DO_TESTCASE_2IB(desc, name, 123); \ + DO_TESTCASE_2IB(desc, name, 132); \ + DO_TESTCASE_2IB(desc, name, 213); \ + DO_TESTCASE_2IB(desc, name, 231); \ + DO_TESTCASE_2IB(desc, name, 312); \ + DO_TESTCASE_2IB(desc, name, 321); + +#define DO_TESTCASE_6x6(desc, name) \ + DO_TESTCASE_6I(desc, name, 123); \ + DO_TESTCASE_6I(desc, name, 132); \ + DO_TESTCASE_6I(desc, name, 213); \ + DO_TESTCASE_6I(desc, name, 231); \ + DO_TESTCASE_6I(desc, name, 312); \ + DO_TESTCASE_6I(desc, name, 321); + +#define DO_TESTCASE_6x6RW(desc, name) \ + DO_TESTCASE_6IRW(desc, name, 123); \ + DO_TESTCASE_6IRW(desc, name, 132); \ + DO_TESTCASE_6IRW(desc, name, 213); \ + DO_TESTCASE_6IRW(desc, name, 231); \ + DO_TESTCASE_6IRW(desc, name, 312); \ + DO_TESTCASE_6IRW(desc, name, 321); + + +void locking_selftest(void) +{ + /* + * Got a locking failure before the selftest ran? + */ + if (!debug_locks) { + printk("----------------------------------\n"); + printk("| Locking API testsuite disabled |\n"); + printk("----------------------------------\n"); + return; + } + + /* + * Run the testsuite: + */ + printk("------------------------\n"); + printk("| Locking API testsuite:\n"); + printk("----------------------------------------------------------------------------\n"); + printk(" | spin |wlock |rlock |mutex | wsem | rsem |\n"); + printk(" --------------------------------------------------------------------------\n"); + + init_shared_classes(); + debug_locks_silent = !debug_locks_verbose; + + DO_TESTCASE_6R("A-A deadlock", AA); + DO_TESTCASE_6R("A-B-B-A deadlock", ABBA); + DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA); + DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC); + DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA); + DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA); + DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA); + DO_TESTCASE_6("double unlock", double_unlock); + DO_TESTCASE_6("initialize held", init_held); + DO_TESTCASE_6_SUCCESS("bad unlock order", bad_unlock_order); + + printk(" --------------------------------------------------------------------------\n"); + print_testname("recursive read-lock"); + printk(" |"); + dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK); + printk(" |"); + dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM); + printk("\n"); + + print_testname("recursive read-lock #2"); + printk(" |"); + dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK); + printk(" |"); + dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM); + printk("\n"); + + print_testname("mixed read-write-lock"); + printk(" |"); + dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK); + printk(" |"); + dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM); + printk("\n"); + + print_testname("mixed write-read-lock"); + printk(" |"); + dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK); + printk(" |"); + dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM); + printk("\n"); + + printk(" --------------------------------------------------------------------------\n"); + + /* + * irq-context testcases: + */ + DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1); + DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A); + DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B); + DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3); + DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4); + DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion); + + DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); +// DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); + + if (unexpected_testcase_failures) { + printk("-----------------------------------------------------------------\n"); + debug_locks = 0; + printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n", + unexpected_testcase_failures, testcase_total); + printk("-----------------------------------------------------------------\n"); + } else if (expected_testcase_failures && testcase_successes) { + printk("--------------------------------------------------------\n"); + printk("%3d out of %3d testcases failed, as expected. |\n", + expected_testcase_failures, testcase_total); + printk("----------------------------------------------------\n"); + debug_locks = 1; + } else if (expected_testcase_failures && !testcase_successes) { + printk("--------------------------------------------------------\n"); + printk("All %3d testcases failed, as expected. |\n", + expected_testcase_failures); + printk("----------------------------------------\n"); + debug_locks = 1; + } else { + printk("-------------------------------------------------------\n"); + printk("Good, all %3d testcases passed! |\n", + testcase_successes); + printk("---------------------------------\n"); + debug_locks = 1; + } + debug_locks_silent = 0; +} diff --git a/lib/rwsem-spinlock.c b/lib/rwsem-spinlock.c index 40ffde940a8..db4fed74b94 100644 --- a/lib/rwsem-spinlock.c +++ b/lib/rwsem-spinlock.c @@ -17,27 +17,22 @@ struct rwsem_waiter { #define RWSEM_WAITING_FOR_WRITE 0x00000002 }; -#if RWSEM_DEBUG -void rwsemtrace(struct rw_semaphore *sem, const char *str) -{ - if (sem->debug) - printk("[%d] %s({%d,%d})\n", - current->pid, str, sem->activity, - list_empty(&sem->wait_list) ? 0 : 1); -} -#endif - /* * initialise the semaphore */ -void fastcall init_rwsem(struct rw_semaphore *sem) +void __init_rwsem(struct rw_semaphore *sem, const char *name, + struct lock_class_key *key) { +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held semaphore: + */ + debug_check_no_locks_freed((void *)sem, sizeof(*sem)); + lockdep_init_map(&sem->dep_map, name, key); +#endif sem->activity = 0; spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif } /* @@ -56,8 +51,6 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) struct task_struct *tsk; int woken; - rwsemtrace(sem, "Entering __rwsem_do_wake"); - waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); if (!wakewrite) { @@ -104,7 +97,6 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) sem->activity += woken; out: - rwsemtrace(sem, "Leaving __rwsem_do_wake"); return sem; } @@ -138,8 +130,6 @@ void fastcall __sched __down_read(struct rw_semaphore *sem) struct rwsem_waiter waiter; struct task_struct *tsk; - rwsemtrace(sem, "Entering __down_read"); - spin_lock_irq(&sem->wait_lock); if (sem->activity >= 0 && list_empty(&sem->wait_list)) { @@ -171,9 +161,8 @@ void fastcall __sched __down_read(struct rw_semaphore *sem) } tsk->state = TASK_RUNNING; - out: - rwsemtrace(sem, "Leaving __down_read"); + ; } /* @@ -184,7 +173,6 @@ int fastcall __down_read_trylock(struct rw_semaphore *sem) unsigned long flags; int ret = 0; - rwsemtrace(sem, "Entering __down_read_trylock"); spin_lock_irqsave(&sem->wait_lock, flags); @@ -196,7 +184,6 @@ int fastcall __down_read_trylock(struct rw_semaphore *sem) spin_unlock_irqrestore(&sem->wait_lock, flags); - rwsemtrace(sem, "Leaving __down_read_trylock"); return ret; } @@ -204,13 +191,11 @@ int fastcall __down_read_trylock(struct rw_semaphore *sem) * get a write lock on the semaphore * - we increment the waiting count anyway to indicate an exclusive lock */ -void fastcall __sched __down_write(struct rw_semaphore *sem) +void fastcall __sched __down_write_nested(struct rw_semaphore *sem, int subclass) { struct rwsem_waiter waiter; struct task_struct *tsk; - rwsemtrace(sem, "Entering __down_write"); - spin_lock_irq(&sem->wait_lock); if (sem->activity == 0 && list_empty(&sem->wait_list)) { @@ -242,9 +227,13 @@ void fastcall __sched __down_write(struct rw_semaphore *sem) } tsk->state = TASK_RUNNING; - out: - rwsemtrace(sem, "Leaving __down_write"); + ; +} + +void fastcall __sched __down_write(struct rw_semaphore *sem) +{ + __down_write_nested(sem, 0); } /* @@ -255,8 +244,6 @@ int fastcall __down_write_trylock(struct rw_semaphore *sem) unsigned long flags; int ret = 0; - rwsemtrace(sem, "Entering __down_write_trylock"); - spin_lock_irqsave(&sem->wait_lock, flags); if (sem->activity == 0 && list_empty(&sem->wait_list)) { @@ -267,7 +254,6 @@ int fastcall __down_write_trylock(struct rw_semaphore *sem) spin_unlock_irqrestore(&sem->wait_lock, flags); - rwsemtrace(sem, "Leaving __down_write_trylock"); return ret; } @@ -278,16 +264,12 @@ void fastcall __up_read(struct rw_semaphore *sem) { unsigned long flags; - rwsemtrace(sem, "Entering __up_read"); - spin_lock_irqsave(&sem->wait_lock, flags); if (--sem->activity == 0 && !list_empty(&sem->wait_list)) sem = __rwsem_wake_one_writer(sem); spin_unlock_irqrestore(&sem->wait_lock, flags); - - rwsemtrace(sem, "Leaving __up_read"); } /* @@ -297,8 +279,6 @@ void fastcall __up_write(struct rw_semaphore *sem) { unsigned long flags; - rwsemtrace(sem, "Entering __up_write"); - spin_lock_irqsave(&sem->wait_lock, flags); sem->activity = 0; @@ -306,8 +286,6 @@ void fastcall __up_write(struct rw_semaphore *sem) sem = __rwsem_do_wake(sem, 1); spin_unlock_irqrestore(&sem->wait_lock, flags); - - rwsemtrace(sem, "Leaving __up_write"); } /* @@ -318,8 +296,6 @@ void fastcall __downgrade_write(struct rw_semaphore *sem) { unsigned long flags; - rwsemtrace(sem, "Entering __downgrade_write"); - spin_lock_irqsave(&sem->wait_lock, flags); sem->activity = 1; @@ -327,18 +303,14 @@ void fastcall __downgrade_write(struct rw_semaphore *sem) sem = __rwsem_do_wake(sem, 0); spin_unlock_irqrestore(&sem->wait_lock, flags); - - rwsemtrace(sem, "Leaving __downgrade_write"); } -EXPORT_SYMBOL(init_rwsem); +EXPORT_SYMBOL(__init_rwsem); EXPORT_SYMBOL(__down_read); EXPORT_SYMBOL(__down_read_trylock); +EXPORT_SYMBOL(__down_write_nested); EXPORT_SYMBOL(__down_write); EXPORT_SYMBOL(__down_write_trylock); EXPORT_SYMBOL(__up_read); EXPORT_SYMBOL(__up_write); EXPORT_SYMBOL(__downgrade_write); -#if RWSEM_DEBUG -EXPORT_SYMBOL(rwsemtrace); -#endif diff --git a/lib/rwsem.c b/lib/rwsem.c index 62fa4eba9ff..b322421c296 100644 --- a/lib/rwsem.c +++ b/lib/rwsem.c @@ -8,6 +8,26 @@ #include <linux/init.h> #include <linux/module.h> +/* + * Initialize an rwsem: + */ +void __init_rwsem(struct rw_semaphore *sem, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held semaphore: + */ + debug_check_no_locks_freed((void *)sem, sizeof(*sem)); + lockdep_init_map(&sem->dep_map, name, key); +#endif + sem->count = RWSEM_UNLOCKED_VALUE; + spin_lock_init(&sem->wait_lock); + INIT_LIST_HEAD(&sem->wait_list); +} + +EXPORT_SYMBOL(__init_rwsem); + struct rwsem_waiter { struct list_head list; struct task_struct *task; @@ -16,17 +36,6 @@ struct rwsem_waiter { #define RWSEM_WAITING_FOR_WRITE 0x00000002 }; -#if RWSEM_DEBUG -#undef rwsemtrace -void rwsemtrace(struct rw_semaphore *sem, const char *str) -{ - printk("sem=%p\n", sem); - printk("(sem)=%08lx\n", sem->count); - if (sem->debug) - printk("[%d] %s({%08lx})\n", current->pid, str, sem->count); -} -#endif - /* * handle the lock release when processes blocked on it that can now run * - if we come here from up_xxxx(), then: @@ -45,8 +54,6 @@ __rwsem_do_wake(struct rw_semaphore *sem, int downgrading) struct list_head *next; signed long oldcount, woken, loop; - rwsemtrace(sem, "Entering __rwsem_do_wake"); - if (downgrading) goto dont_wake_writers; @@ -127,7 +134,6 @@ __rwsem_do_wake(struct rw_semaphore *sem, int downgrading) next->prev = &sem->wait_list; out: - rwsemtrace(sem, "Leaving __rwsem_do_wake"); return sem; /* undo the change to count, but check for a transition 1->0 */ @@ -186,13 +192,9 @@ rwsem_down_read_failed(struct rw_semaphore *sem) { struct rwsem_waiter waiter; - rwsemtrace(sem, "Entering rwsem_down_read_failed"); - waiter.flags = RWSEM_WAITING_FOR_READ; rwsem_down_failed_common(sem, &waiter, RWSEM_WAITING_BIAS - RWSEM_ACTIVE_BIAS); - - rwsemtrace(sem, "Leaving rwsem_down_read_failed"); return sem; } @@ -204,12 +206,9 @@ rwsem_down_write_failed(struct rw_semaphore *sem) { struct rwsem_waiter waiter; - rwsemtrace(sem, "Entering rwsem_down_write_failed"); - waiter.flags = RWSEM_WAITING_FOR_WRITE; rwsem_down_failed_common(sem, &waiter, -RWSEM_ACTIVE_BIAS); - rwsemtrace(sem, "Leaving rwsem_down_write_failed"); return sem; } @@ -221,8 +220,6 @@ struct rw_semaphore fastcall *rwsem_wake(struct rw_semaphore *sem) { unsigned long flags; - rwsemtrace(sem, "Entering rwsem_wake"); - spin_lock_irqsave(&sem->wait_lock, flags); /* do nothing if list empty */ @@ -231,8 +228,6 @@ struct rw_semaphore fastcall *rwsem_wake(struct rw_semaphore *sem) spin_unlock_irqrestore(&sem->wait_lock, flags); - rwsemtrace(sem, "Leaving rwsem_wake"); - return sem; } @@ -245,8 +240,6 @@ struct rw_semaphore fastcall *rwsem_downgrade_wake(struct rw_semaphore *sem) { unsigned long flags; - rwsemtrace(sem, "Entering rwsem_downgrade_wake"); - spin_lock_irqsave(&sem->wait_lock, flags); /* do nothing if list empty */ @@ -255,7 +248,6 @@ struct rw_semaphore fastcall *rwsem_downgrade_wake(struct rw_semaphore *sem) spin_unlock_irqrestore(&sem->wait_lock, flags); - rwsemtrace(sem, "Leaving rwsem_downgrade_wake"); return sem; } @@ -263,6 +255,3 @@ EXPORT_SYMBOL(rwsem_down_read_failed); EXPORT_SYMBOL(rwsem_down_write_failed); EXPORT_SYMBOL(rwsem_wake); EXPORT_SYMBOL(rwsem_downgrade_wake); -#if RWSEM_DEBUG -EXPORT_SYMBOL(rwsemtrace); -#endif diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c index 93c15ee3f8e..3d9c4dc965e 100644 --- a/lib/spinlock_debug.c +++ b/lib/spinlock_debug.c @@ -8,38 +8,71 @@ #include <linux/spinlock.h> #include <linux/interrupt.h> +#include <linux/debug_locks.h> #include <linux/delay.h> +#include <linux/module.h> + +void __spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + lockdep_init_map(&lock->dep_map, name, key); +#endif + lock->raw_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; + lock->magic = SPINLOCK_MAGIC; + lock->owner = SPINLOCK_OWNER_INIT; + lock->owner_cpu = -1; +} + +EXPORT_SYMBOL(__spin_lock_init); + +void __rwlock_init(rwlock_t *lock, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + lockdep_init_map(&lock->dep_map, name, key); +#endif + lock->raw_lock = (raw_rwlock_t) __RAW_RW_LOCK_UNLOCKED; + lock->magic = RWLOCK_MAGIC; + lock->owner = SPINLOCK_OWNER_INIT; + lock->owner_cpu = -1; +} + +EXPORT_SYMBOL(__rwlock_init); static void spin_bug(spinlock_t *lock, const char *msg) { - static long print_once = 1; struct task_struct *owner = NULL; - if (xchg(&print_once, 0)) { - if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT) - owner = lock->owner; - printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n", - msg, raw_smp_processor_id(), - current->comm, current->pid); - printk(KERN_EMERG " lock: %p, .magic: %08x, .owner: %s/%d, " - ".owner_cpu: %d\n", - lock, lock->magic, - owner ? owner->comm : "<none>", - owner ? owner->pid : -1, - lock->owner_cpu); - dump_stack(); -#ifdef CONFIG_SMP - /* - * We cannot continue on SMP: - */ -// panic("bad locking"); -#endif - } + if (!debug_locks_off()) + return; + + if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT) + owner = lock->owner; + printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n", + msg, raw_smp_processor_id(), + current->comm, current->pid); + printk(KERN_EMERG " lock: %p, .magic: %08x, .owner: %s/%d, " + ".owner_cpu: %d\n", + lock, lock->magic, + owner ? owner->comm : "<none>", + owner ? owner->pid : -1, + lock->owner_cpu); + dump_stack(); } #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg) -static inline void debug_spin_lock_before(spinlock_t *lock) +static inline void +debug_spin_lock_before(spinlock_t *lock) { SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic"); SPIN_BUG_ON(lock->owner == current, lock, "recursion"); @@ -118,20 +151,13 @@ void _raw_spin_unlock(spinlock_t *lock) static void rwlock_bug(rwlock_t *lock, const char *msg) { - static long print_once = 1; - - if (xchg(&print_once, 0)) { - printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n", - msg, raw_smp_processor_id(), current->comm, - current->pid, lock); - dump_stack(); -#ifdef CONFIG_SMP - /* - * We cannot continue on SMP: - */ - panic("bad locking"); -#endif - } + if (!debug_locks_off()) + return; + + printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n", + msg, raw_smp_processor_id(), current->comm, + current->pid, lock); + dump_stack(); } #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg) diff --git a/mm/memory.c b/mm/memory.c index 7e2a4b1580e..c1e14c9e67e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -503,7 +503,7 @@ again: return -ENOMEM; src_pte = pte_offset_map_nested(src_pmd, addr); src_ptl = pte_lockptr(src_mm, src_pmd); - spin_lock(src_ptl); + spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); do { /* diff --git a/mm/mremap.c b/mm/mremap.c index 1903bdf65e4..7c15cf3373a 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -97,7 +97,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, new_pte = pte_offset_map_nested(new_pmd, new_addr); new_ptl = pte_lockptr(mm, new_pmd); if (new_ptl != old_ptl) - spin_lock(new_ptl); + spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE, new_pte++, new_addr += PAGE_SIZE) { diff --git a/mm/oom_kill.c b/mm/oom_kill.c index d46ed0f1dc0..b9af136e5cf 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,7 @@ static struct task_struct *select_bad_process(unsigned long *ppoints) * CAP_SYS_RAW_IO set, send SIGTERM instead (but it's unlikely that * we select a process with CAP_SYS_RAW_IO set). */ -static void __oom_kill_task(task_t *p, const char *message) +static void __oom_kill_task(struct task_struct *p, const char *message) { if (p->pid == 1) { WARN_ON(1); @@ -255,10 +255,10 @@ static void __oom_kill_task(task_t *p, const char *message) force_sig(SIGKILL, p); } -static int oom_kill_task(task_t *p, const char *message) +static int oom_kill_task(struct task_struct *p, const char *message) { struct mm_struct *mm; - task_t * g, * q; + struct task_struct *g, *q; mm = p->mm; @@ -316,7 +316,7 @@ static int oom_kill_process(struct task_struct *p, unsigned long points, */ void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask, int order) { - task_t *p; + struct task_struct *p; unsigned long points = 0; if (printk_ratelimit()) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3e792a583f3..54a4f5375bb 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2005,6 +2005,10 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat, zone->spanned_pages = size; zone->present_pages = realsize; +#ifdef CONFIG_NUMA + zone->min_unmapped_ratio = (realsize*sysctl_min_unmapped_ratio) + / 100; +#endif zone->name = zone_names[j]; spin_lock_init(&zone->lock); spin_lock_init(&zone->lru_lock); @@ -2298,6 +2302,24 @@ int min_free_kbytes_sysctl_handler(ctl_table *table, int write, return 0; } +#ifdef CONFIG_NUMA +int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write, + struct file *file, void __user *buffer, size_t *length, loff_t *ppos) +{ + struct zone *zone; + int rc; + + rc = proc_dointvec_minmax(table, write, file, buffer, length, ppos); + if (rc) + return rc; + + for_each_zone(zone) + zone->min_unmapped_ratio = (zone->present_pages * + sysctl_min_unmapped_ratio) / 100; + return 0; +} +#endif + /* * lowmem_reserve_ratio_sysctl_handler - just a wrapper around * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() diff --git a/mm/slab.c b/mm/slab.c index 3936af34454..85c2e03098a 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1021,7 +1021,8 @@ static void drain_alien_cache(struct kmem_cache *cachep, } } -static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) +static inline int cache_free_alien(struct kmem_cache *cachep, void *objp, + int nesting) { struct slab *slabp = virt_to_slab(objp); int nodeid = slabp->nodeid; @@ -1039,7 +1040,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) STATS_INC_NODEFREES(cachep); if (l3->alien && l3->alien[nodeid]) { alien = l3->alien[nodeid]; - spin_lock(&alien->lock); + spin_lock_nested(&alien->lock, nesting); if (unlikely(alien->avail == alien->limit)) { STATS_INC_ACOVERFLOW(cachep); __drain_alien_cache(cachep, alien, nodeid); @@ -1068,7 +1069,8 @@ static inline void free_alien_cache(struct array_cache **ac_ptr) { } -static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) +static inline int cache_free_alien(struct kmem_cache *cachep, void *objp, + int nesting) { return 0; } @@ -1272,6 +1274,11 @@ static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list, local_irq_disable(); memcpy(ptr, list, sizeof(struct kmem_list3)); + /* + * Do not assume that spinlocks can be initialized via memcpy: + */ + spin_lock_init(&ptr->list_lock); + MAKE_ALL_LISTS(cachep, ptr, nodeid); cachep->nodelists[nodeid] = ptr; local_irq_enable(); @@ -1398,7 +1405,7 @@ void __init kmem_cache_init(void) } /* 4) Replace the bootstrap head arrays */ { - void *ptr; + struct array_cache *ptr; ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); @@ -1406,6 +1413,11 @@ void __init kmem_cache_init(void) BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache); memcpy(ptr, cpu_cache_get(&cache_cache), sizeof(struct arraycache_init)); + /* + * Do not assume that spinlocks can be initialized via memcpy: + */ + spin_lock_init(&ptr->lock); + cache_cache.array[smp_processor_id()] = ptr; local_irq_enable(); @@ -1416,6 +1428,11 @@ void __init kmem_cache_init(void) != &initarray_generic.cache); memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), sizeof(struct arraycache_init)); + /* + * Do not assume that spinlocks can be initialized via memcpy: + */ + spin_lock_init(&ptr->lock); + malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = ptr; local_irq_enable(); @@ -1743,6 +1760,8 @@ static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp) } #endif +static void __cache_free(struct kmem_cache *cachep, void *objp, int nesting); + /** * slab_destroy - destroy and release all objects in a slab * @cachep: cache pointer being destroyed @@ -1766,8 +1785,17 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp) call_rcu(&slab_rcu->head, kmem_rcu_free); } else { kmem_freepages(cachep, addr); - if (OFF_SLAB(cachep)) - kmem_cache_free(cachep->slabp_cache, slabp); + if (OFF_SLAB(cachep)) { + unsigned long flags; + + /* + * lockdep: we may nest inside an already held + * ac->lock, so pass in a nesting flag: + */ + local_irq_save(flags); + __cache_free(cachep->slabp_cache, slabp, 1); + local_irq_restore(flags); + } } } @@ -3072,7 +3100,16 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, if (slabp->inuse == 0) { if (l3->free_objects > l3->free_limit) { l3->free_objects -= cachep->num; + /* + * It is safe to drop the lock. The slab is + * no longer linked to the cache. cachep + * cannot disappear - we are using it and + * all destruction of caches must be + * serialized properly by the user. + */ + spin_unlock(&l3->list_lock); slab_destroy(cachep, slabp); + spin_lock(&l3->list_lock); } else { list_add(&slabp->list, &l3->slabs_free); } @@ -3098,7 +3135,7 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) #endif check_irq_off(); l3 = cachep->nodelists[node]; - spin_lock(&l3->list_lock); + spin_lock_nested(&l3->list_lock, SINGLE_DEPTH_NESTING); if (l3->shared) { struct array_cache *shared_array = l3->shared; int max = shared_array->limit - shared_array->avail; @@ -3141,14 +3178,14 @@ free_done: * Release an obj back to its cache. If the obj has a constructed state, it must * be in this state _before_ it is released. Called with disabled ints. */ -static inline void __cache_free(struct kmem_cache *cachep, void *objp) +static void __cache_free(struct kmem_cache *cachep, void *objp, int nesting) { struct array_cache *ac = cpu_cache_get(cachep); check_irq_off(); objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0)); - if (cache_free_alien(cachep, objp)) + if (cache_free_alien(cachep, objp, nesting)) return; if (likely(ac->avail < ac->limit)) { @@ -3387,7 +3424,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) BUG_ON(virt_to_cache(objp) != cachep); local_irq_save(flags); - __cache_free(cachep, objp); + __cache_free(cachep, objp, 0); local_irq_restore(flags); } EXPORT_SYMBOL(kmem_cache_free); @@ -3412,7 +3449,7 @@ void kfree(const void *objp) kfree_debugcheck(objp); c = virt_to_cache(objp); debug_check_no_locks_freed(objp, obj_size(c)); - __cache_free(c, (void *)objp); + __cache_free(c, (void *)objp, 0); local_irq_restore(flags); } EXPORT_SYMBOL(kfree); diff --git a/mm/swap_state.c b/mm/swap_state.c index fccbd9bba77..5f7cf2a4cb5 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -38,7 +38,7 @@ static struct backing_dev_info swap_backing_dev_info = { struct address_space swapper_space = { .page_tree = RADIX_TREE_INIT(GFP_ATOMIC|__GFP_NOWARN), - .tree_lock = RW_LOCK_UNLOCKED, + .tree_lock = __RW_LOCK_UNLOCKED(swapper_space.tree_lock), .a_ops = &swap_aops, .i_mmap_nonlinear = LIST_HEAD_INIT(swapper_space.i_mmap_nonlinear), .backing_dev_info = &swap_backing_dev_info, diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 35f8553f893..7b450798b45 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -330,6 +330,8 @@ void __vunmap(void *addr, int deallocate_pages) return; } + debug_check_no_locks_freed(addr, area->size); + if (deallocate_pages) { int i; diff --git a/mm/vmscan.c b/mm/vmscan.c index ff2ebe9458a..5d4c4d02254 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1503,10 +1503,6 @@ module_init(kswapd_init) * * If non-zero call zone_reclaim when the number of free pages falls below * the watermarks. - * - * In the future we may add flags to the mode. However, the page allocator - * should only have to check that zone_reclaim_mode != 0 before calling - * zone_reclaim(). */ int zone_reclaim_mode __read_mostly; @@ -1524,6 +1520,12 @@ int zone_reclaim_mode __read_mostly; #define ZONE_RECLAIM_PRIORITY 4 /* + * Percentage of pages in a zone that must be unmapped for zone_reclaim to + * occur. + */ +int sysctl_min_unmapped_ratio = 1; + +/* * Try to free up some pages from this zone through reclaim. */ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) @@ -1590,18 +1592,17 @@ int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) int node_id; /* - * Do not reclaim if there are not enough reclaimable pages in this - * zone that would satify this allocations. + * Zone reclaim reclaims unmapped file backed pages. * - * All unmapped pagecache pages are reclaimable. - * - * Both counters may be temporarily off a bit so we use - * SWAP_CLUSTER_MAX as the boundary. It may also be good to - * leave a few frequently used unmapped pagecache pages around. + * A small portion of unmapped file backed pages is needed for + * file I/O otherwise pages read by file I/O will be immediately + * thrown out if the zone is overallocated. So we do not reclaim + * if less than a specified percentage of the zone is used by + * unmapped file backed pages. */ if (zone_page_state(zone, NR_FILE_PAGES) - - zone_page_state(zone, NR_FILE_MAPPED) < SWAP_CLUSTER_MAX) - return 0; + zone_page_state(zone, NR_FILE_MAPPED) <= zone->min_unmapped_ratio) + return 0; /* * Avoid concurrent zone reclaims, do not reclaim in a zone that does diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 3948949a609..458031bfff5 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -364,6 +364,14 @@ static void vlan_transfer_operstate(const struct net_device *dev, struct net_dev } } +/* + * vlan network devices have devices nesting below it, and are a special + * "super class" of normal network devices; split their locks off into a + * separate class since they always nest. + */ +static struct lock_class_key vlan_netdev_xmit_lock_key; + + /* Attach a VLAN device to a mac address (ie Ethernet Card). * Returns the device that was created, or NULL if there was * an error of some kind. @@ -460,6 +468,7 @@ static struct net_device *register_vlan_device(const char *eth_IF_name, new_dev = alloc_netdev(sizeof(struct vlan_dev_info), name, vlan_setup); + if (new_dev == NULL) goto out_unlock; @@ -518,6 +527,8 @@ static struct net_device *register_vlan_device(const char *eth_IF_name, if (register_netdevice(new_dev)) goto out_free_newdev; + lockdep_set_class(&new_dev->_xmit_lock, &vlan_netdev_xmit_lock_key); + new_dev->iflink = real_dev->ifindex; vlan_transfer_operstate(real_dev, new_dev); linkwatch_fire_event(new_dev); /* _MUST_ call rfc2863_policy() */ diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 7cfbdb215ba..44f6a181a75 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -71,6 +71,13 @@ static kmem_cache_t *skbuff_head_cache __read_mostly; static kmem_cache_t *skbuff_fclone_cache __read_mostly; /* + * lockdep: lock class key used by skb_queue_head_init(): + */ +struct lock_class_key skb_queue_lock_key; + +EXPORT_SYMBOL(skb_queue_lock_key); + +/* * Keep out-of-line to prevent kernel bloat. * __builtin_return_address is not used because it is not always * reliable. diff --git a/net/core/sock.c b/net/core/sock.c index 533b9317144..51fcfbc041a 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -129,6 +129,53 @@ #include <net/tcp.h> #endif +/* + * Each address family might have different locking rules, so we have + * one slock key per address family: + */ +static struct lock_class_key af_family_keys[AF_MAX]; +static struct lock_class_key af_family_slock_keys[AF_MAX]; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +/* + * Make lock validator output more readable. (we pre-construct these + * strings build-time, so that runtime initialization of socket + * locks is fast): + */ +static const char *af_family_key_strings[AF_MAX+1] = { + "sk_lock-AF_UNSPEC", "sk_lock-AF_UNIX" , "sk_lock-AF_INET" , + "sk_lock-AF_AX25" , "sk_lock-AF_IPX" , "sk_lock-AF_APPLETALK", + "sk_lock-AF_NETROM", "sk_lock-AF_BRIDGE" , "sk_lock-AF_ATMPVC" , + "sk_lock-AF_X25" , "sk_lock-AF_INET6" , "sk_lock-AF_ROSE" , + "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" , + "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" , + "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" , + "sk_lock-21" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" , + "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" , + "sk_lock-27" , "sk_lock-28" , "sk_lock-29" , + "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-AF_MAX" +}; +static const char *af_family_slock_key_strings[AF_MAX+1] = { + "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" , + "slock-AF_AX25" , "slock-AF_IPX" , "slock-AF_APPLETALK", + "slock-AF_NETROM", "slock-AF_BRIDGE" , "slock-AF_ATMPVC" , + "slock-AF_X25" , "slock-AF_INET6" , "slock-AF_ROSE" , + "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" , + "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" , + "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" , + "slock-21" , "slock-AF_SNA" , "slock-AF_IRDA" , + "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" , + "slock-27" , "slock-28" , "slock-29" , + "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_MAX" +}; +#endif + +/* + * sk_callback_lock locking rules are per-address-family, + * so split the lock classes by using a per-AF key: + */ +static struct lock_class_key af_callback_keys[AF_MAX]; + /* Take into consideration the size of the struct sk_buff overhead in the * determination of these values, since that is non-constant across * platforms. This makes socket queueing behavior and performance @@ -237,9 +284,16 @@ int sk_receive_skb(struct sock *sk, struct sk_buff *skb) skb->dev = NULL; bh_lock_sock(sk); - if (!sock_owned_by_user(sk)) + if (!sock_owned_by_user(sk)) { + /* + * trylock + unlock semantics: + */ + mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); + rc = sk->sk_backlog_rcv(sk, skb); - else + + mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); + } else sk_add_backlog(sk, skb); bh_unlock_sock(sk); out: @@ -749,6 +803,33 @@ lenout: return 0; } +/* + * Initialize an sk_lock. + * + * (We also register the sk_lock with the lock validator.) + */ +static void inline sock_lock_init(struct sock *sk) +{ + spin_lock_init(&sk->sk_lock.slock); + sk->sk_lock.owner = NULL; + init_waitqueue_head(&sk->sk_lock.wq); + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)&sk->sk_lock, sizeof(sk->sk_lock)); + + /* + * Mark both the sk_lock and the sk_lock.slock as a + * per-address-family lock class: + */ + lockdep_set_class_and_name(&sk->sk_lock.slock, + af_family_slock_keys + sk->sk_family, + af_family_slock_key_strings[sk->sk_family]); + lockdep_init_map(&sk->sk_lock.dep_map, + af_family_key_strings[sk->sk_family], + af_family_keys + sk->sk_family); +} + /** * sk_alloc - All socket objects are allocated here * @family: protocol family @@ -848,6 +929,8 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority) rwlock_init(&newsk->sk_dst_lock); rwlock_init(&newsk->sk_callback_lock); + lockdep_set_class(&newsk->sk_callback_lock, + af_callback_keys + newsk->sk_family); newsk->sk_dst_cache = NULL; newsk->sk_wmem_queued = 0; @@ -1422,6 +1505,8 @@ void sock_init_data(struct socket *sock, struct sock *sk) rwlock_init(&sk->sk_dst_lock); rwlock_init(&sk->sk_callback_lock); + lockdep_set_class(&sk->sk_callback_lock, + af_callback_keys + sk->sk_family); sk->sk_state_change = sock_def_wakeup; sk->sk_data_ready = sock_def_readable; @@ -1449,24 +1534,34 @@ void sock_init_data(struct socket *sock, struct sock *sk) void fastcall lock_sock(struct sock *sk) { might_sleep(); - spin_lock_bh(&(sk->sk_lock.slock)); + spin_lock_bh(&sk->sk_lock.slock); if (sk->sk_lock.owner) __lock_sock(sk); sk->sk_lock.owner = (void *)1; - spin_unlock_bh(&(sk->sk_lock.slock)); + spin_unlock(&sk->sk_lock.slock); + /* + * The sk_lock has mutex_lock() semantics here: + */ + mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); + local_bh_enable(); } EXPORT_SYMBOL(lock_sock); void fastcall release_sock(struct sock *sk) { - spin_lock_bh(&(sk->sk_lock.slock)); + /* + * The sk_lock has mutex_unlock() semantics: + */ + mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); + + spin_lock_bh(&sk->sk_lock.slock); if (sk->sk_backlog.tail) __release_sock(sk); sk->sk_lock.owner = NULL; - if (waitqueue_active(&(sk->sk_lock.wq))) - wake_up(&(sk->sk_lock.wq)); - spin_unlock_bh(&(sk->sk_lock.slock)); + if (waitqueue_active(&sk->sk_lock.wq)) + wake_up(&sk->sk_lock.wq); + spin_unlock_bh(&sk->sk_lock.slock); } EXPORT_SYMBOL(release_sock); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index da44fabf4dc..2dc6dbb2846 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -205,21 +205,27 @@ __u8 ip_tos2prio[16] = { struct rt_hash_bucket { struct rtable *chain; }; -#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \ + defined(CONFIG_PROVE_LOCKING) /* * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks * The size of this table is a power of two and depends on the number of CPUS. + * (on lockdep we have a quite big spinlock_t, so keep the size down there) */ -#if NR_CPUS >= 32 -#define RT_HASH_LOCK_SZ 4096 -#elif NR_CPUS >= 16 -#define RT_HASH_LOCK_SZ 2048 -#elif NR_CPUS >= 8 -#define RT_HASH_LOCK_SZ 1024 -#elif NR_CPUS >= 4 -#define RT_HASH_LOCK_SZ 512 +#ifdef CONFIG_LOCKDEP +# define RT_HASH_LOCK_SZ 256 #else -#define RT_HASH_LOCK_SZ 256 +# if NR_CPUS >= 32 +# define RT_HASH_LOCK_SZ 4096 +# elif NR_CPUS >= 16 +# define RT_HASH_LOCK_SZ 2048 +# elif NR_CPUS >= 8 +# define RT_HASH_LOCK_SZ 1024 +# elif NR_CPUS >= 4 +# define RT_HASH_LOCK_SZ 512 +# else +# define RT_HASH_LOCK_SZ 256 +# endif #endif static spinlock_t *rt_hash_locks; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 8355b729fa9..5a886e6efbb 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -90,7 +90,7 @@ static struct socket *tcp_socket; void tcp_v4_send_check(struct sock *sk, int len, struct sk_buff *skb); struct inet_hashinfo __cacheline_aligned tcp_hashinfo = { - .lhash_lock = RW_LOCK_UNLOCKED, + .lhash_lock = __RW_LOCK_UNLOCKED(tcp_hashinfo.lhash_lock), .lhash_users = ATOMIC_INIT(0), .lhash_wait = __WAIT_QUEUE_HEAD_INITIALIZER(tcp_hashinfo.lhash_wait), }; @@ -1090,7 +1090,7 @@ process: skb->dev = NULL; - bh_lock_sock(sk); + bh_lock_sock_nested(sk); ret = 0; if (!sock_owned_by_user(sk)) { #ifdef CONFIG_NET_DMA diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index e0851697ad5..0ccb7cb22b1 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -40,7 +40,7 @@ int sysctl_tcp_abort_on_overflow; struct inet_timewait_death_row tcp_death_row = { .sysctl_max_tw_buckets = NR_FILE * 2, .period = TCP_TIMEWAIT_LEN / INET_TWDR_TWKILL_SLOTS, - .death_lock = SPIN_LOCK_UNLOCKED, + .death_lock = __SPIN_LOCK_UNLOCKED(tcp_death_row.death_lock), .hashinfo = &tcp_hashinfo, .tw_timer = TIMER_INITIALIZER(inet_twdr_hangman, 0, (unsigned long)&tcp_death_row), diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 70cee82a98b..55c0adc8f11 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -156,7 +156,7 @@ static void netlink_sock_destruct(struct sock *sk) static void netlink_table_grab(void) { - write_lock_bh(&nl_table_lock); + write_lock_irq(&nl_table_lock); if (atomic_read(&nl_table_users)) { DECLARE_WAITQUEUE(wait, current); @@ -166,9 +166,9 @@ static void netlink_table_grab(void) set_current_state(TASK_UNINTERRUPTIBLE); if (atomic_read(&nl_table_users) == 0) break; - write_unlock_bh(&nl_table_lock); + write_unlock_irq(&nl_table_lock); schedule(); - write_lock_bh(&nl_table_lock); + write_lock_irq(&nl_table_lock); } __set_current_state(TASK_RUNNING); @@ -178,7 +178,7 @@ static void netlink_table_grab(void) static __inline__ void netlink_table_ungrab(void) { - write_unlock_bh(&nl_table_lock); + write_unlock_irq(&nl_table_lock); wake_up(&nl_table_wait); } diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c index 6db6006616c..dc6cb93c883 100644 --- a/net/sunrpc/rpc_pipe.c +++ b/net/sunrpc/rpc_pipe.c @@ -515,7 +515,7 @@ rpc_depopulate(struct dentry *parent) struct dentry *dentry, *dvec[10]; int n = 0; - mutex_lock(&dir->i_mutex); + mutex_lock_nested(&dir->i_mutex, I_MUTEX_CHILD); repeat: spin_lock(&dcache_lock); list_for_each_safe(pos, next, &parent->d_subdirs) { @@ -631,7 +631,7 @@ rpc_lookup_negative(char *path, struct nameidata *nd) if ((error = rpc_lookup_parent(path, nd)) != 0) return ERR_PTR(error); dir = nd->dentry->d_inode; - mutex_lock(&dir->i_mutex); + mutex_lock_nested(&dir->i_mutex, I_MUTEX_PARENT); dentry = lookup_one_len(nd->last.name, nd->dentry, nd->last.len); if (IS_ERR(dentry)) goto out_err; @@ -693,7 +693,7 @@ rpc_rmdir(char *path) if ((error = rpc_lookup_parent(path, &nd)) != 0) return error; dir = nd.dentry->d_inode; - mutex_lock(&dir->i_mutex); + mutex_lock_nested(&dir->i_mutex, I_MUTEX_PARENT); dentry = lookup_one_len(nd.last.name, nd.dentry, nd.last.len); if (IS_ERR(dentry)) { error = PTR_ERR(dentry); @@ -754,7 +754,7 @@ rpc_unlink(char *path) if ((error = rpc_lookup_parent(path, &nd)) != 0) return error; dir = nd.dentry->d_inode; - mutex_lock(&dir->i_mutex); + mutex_lock_nested(&dir->i_mutex, I_MUTEX_PARENT); dentry = lookup_one_len(nd.last.name, nd.dentry, nd.last.len); if (IS_ERR(dentry)) { error = PTR_ERR(dentry); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index aca65010942..e9a287bc314 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -565,6 +565,14 @@ static struct proto unix_proto = { .obj_size = sizeof(struct unix_sock), }; +/* + * AF_UNIX sockets do not interact with hardware, hence they + * dont trigger interrupts - so it's safe for them to have + * bh-unsafe locking for their sk_receive_queue.lock. Split off + * this special lock-class by reinitializing the spinlock key: + */ +static struct lock_class_key af_unix_sk_receive_queue_lock_key; + static struct sock * unix_create1(struct socket *sock) { struct sock *sk = NULL; @@ -580,6 +588,8 @@ static struct sock * unix_create1(struct socket *sock) atomic_inc(&unix_nr_socks); sock_init_data(sock,sk); + lockdep_set_class(&sk->sk_receive_queue.lock, + &af_unix_sk_receive_queue_lock_key); sk->sk_write_space = unix_write_space; sk->sk_max_ack_backlog = sysctl_unix_max_dgram_qlen; @@ -1045,7 +1055,7 @@ restart: goto out_unlock; } - unix_state_wlock(sk); + unix_state_wlock_nested(sk); if (sk->sk_state != st) { unix_state_wunlock(sk); diff --git a/sound/core/seq/seq_device.c b/sound/core/seq/seq_device.c index d812dc88636..4260de90f36 100644 --- a/sound/core/seq/seq_device.c +++ b/sound/core/seq/seq_device.c @@ -380,6 +380,12 @@ static struct ops_list * create_driver(char *id) /* set up driver entry */ strlcpy(ops->id, id, sizeof(ops->id)); mutex_init(&ops->reg_mutex); + /* + * The ->reg_mutex locking rules are per-driver, so we create + * separate per-driver lock classes: + */ + lockdep_set_class(&ops->reg_mutex, (struct lock_class_key *)id); + ops->driver = DRIVER_EMPTY; INIT_LIST_HEAD(&ops->dev_list); /* lock this instance */ diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c index d467b4f0ff2..8c64b58ff77 100644 --- a/sound/core/seq/seq_ports.c +++ b/sound/core/seq/seq_ports.c @@ -514,7 +514,7 @@ int snd_seq_port_connect(struct snd_seq_client *connector, atomic_set(&subs->ref_count, 2); down_write(&src->list_mutex); - down_write(&dest->list_mutex); + down_write_nested(&dest->list_mutex, SINGLE_DEPTH_NESTING); exclusive = info->flags & SNDRV_SEQ_PORT_SUBS_EXCLUSIVE ? 1 : 0; err = -EBUSY; @@ -587,7 +587,7 @@ int snd_seq_port_disconnect(struct snd_seq_client *connector, unsigned long flags; down_write(&src->list_mutex); - down_write(&dest->list_mutex); + down_write_nested(&dest->list_mutex, SINGLE_DEPTH_NESTING); /* look for the connection */ list_for_each(p, &src->list_head) { diff --git a/sound/sparc/amd7930.c b/sound/sparc/amd7930.c index db3e22efd02..2bd8e40b854 100644 --- a/sound/sparc/amd7930.c +++ b/sound/sparc/amd7930.c @@ -1033,10 +1033,10 @@ static int __init amd7930_attach_common(struct resource *rp, int irq) strcpy(card->driver, "AMD7930"); strcpy(card->shortname, "Sun AMD7930"); - sprintf(card->longname, "%s at 0x%02lx:0x%08lx, irq %d", + sprintf(card->longname, "%s at 0x%02lx:0x%08Lx, irq %d", card->shortname, rp->flags & 0xffL, - rp->start, + (unsigned long long)rp->start, irq); if ((err = snd_amd7930_create(card, rp, diff --git a/sound/sparc/cs4231.c b/sound/sparc/cs4231.c index 5018fcf41df..9a06c3bd694 100644 --- a/sound/sparc/cs4231.c +++ b/sound/sparc/cs4231.c @@ -2036,7 +2036,7 @@ static int __init cs4231_sbus_attach(struct sbus_dev *sdev) if (err) return err; - sprintf(card->longname, "%s at 0x%02lx:0x%016lx, irq %d", + sprintf(card->longname, "%s at 0x%02lx:0x%016Lx, irq %d", card->shortname, rp->flags & 0xffL, (unsigned long long)rp->start, diff --git a/sound/sparc/dbri.c b/sound/sparc/dbri.c index 59a02a0d9af..f3ae6e23610 100644 --- a/sound/sparc/dbri.c +++ b/sound/sparc/dbri.c @@ -2645,7 +2645,7 @@ static int __init dbri_attach(int prom_node, struct sbus_dev *sdev) strcpy(card->driver, "DBRI"); strcpy(card->shortname, "Sun DBRI"); rp = &sdev->resource[0]; - sprintf(card->longname, "%s at 0x%02lx:0x%016lx, irq %d", + sprintf(card->longname, "%s at 0x%02lx:0x%016Lx, irq %d", card->shortname, rp->flags & 0xffL, (unsigned long long)rp->start, irq.pri); |