aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--drivers/ata/libata-core.c96
-rw-r--r--drivers/scsi/scsi_scan.c3
-rw-r--r--drivers/scsi/sd.c109
-rw-r--r--fs/inode.c20
-rw-r--r--fs/super.c10
-rw-r--r--include/linux/async.h25
-rw-r--r--include/linux/fs.h5
-rw-r--r--init/do_mounts.c2
-rw-r--r--init/main.c5
-rw-r--r--kernel/Makefile3
-rw-r--r--kernel/async.c321
-rw-r--r--kernel/irq/autoprobe.c5
-rw-r--r--kernel/module.c2
13 files changed, 510 insertions, 96 deletions
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index fecca4223f8..f178a450ec0 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -56,6 +56,7 @@
#include <linux/workqueue.h>
#include <linux/scatterlist.h>
#include <linux/io.h>
+#include <linux/async.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_host.h>
@@ -5909,6 +5910,54 @@ void ata_host_init(struct ata_host *host, struct device *dev,
host->ops = ops;
}
+
+static void async_port_probe(void *data, async_cookie_t cookie)
+{
+ int rc;
+ struct ata_port *ap = data;
+ /* probe */
+ if (ap->ops->error_handler) {
+ struct ata_eh_info *ehi = &ap->link.eh_info;
+ unsigned long flags;
+
+ ata_port_probe(ap);
+
+ /* kick EH for boot probing */
+ spin_lock_irqsave(ap->lock, flags);
+
+ ehi->probe_mask |= ATA_ALL_DEVICES;
+ ehi->action |= ATA_EH_RESET | ATA_EH_LPM;
+ ehi->flags |= ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET;
+
+ ap->pflags &= ~ATA_PFLAG_INITIALIZING;
+ ap->pflags |= ATA_PFLAG_LOADING;
+ ata_port_schedule_eh(ap);
+
+ spin_unlock_irqrestore(ap->lock, flags);
+
+ /* wait for EH to finish */
+ ata_port_wait_eh(ap);
+ } else {
+ DPRINTK("ata%u: bus probe begin\n", ap->print_id);
+ rc = ata_bus_probe(ap);
+ DPRINTK("ata%u: bus probe end\n", ap->print_id);
+
+ if (rc) {
+ /* FIXME: do something useful here?
+ * Current libata behavior will
+ * tear down everything when
+ * the module is removed
+ * or the h/w is unplugged.
+ */
+ }
+ }
+
+ /* in order to keep device order, we need to synchronize at this point */
+ async_synchronize_cookie(cookie);
+
+ ata_scsi_scan_host(ap, 1);
+
+}
/**
* ata_host_register - register initialized ATA host
* @host: ATA host to register
@@ -5988,52 +6037,9 @@ int ata_host_register(struct ata_host *host, struct scsi_host_template *sht)
DPRINTK("probe begin\n");
for (i = 0; i < host->n_ports; i++) {
struct ata_port *ap = host->ports[i];
-
- /* probe */
- if (ap->ops->error_handler) {
- struct ata_eh_info *ehi = &ap->link.eh_info;
- unsigned long flags;
-
- ata_port_probe(ap);
-
- /* kick EH for boot probing */
- spin_lock_irqsave(ap->lock, flags);
-
- ehi->probe_mask |= ATA_ALL_DEVICES;
- ehi->action |= ATA_EH_RESET | ATA_EH_LPM;
- ehi->flags |= ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET;
-
- ap->pflags &= ~ATA_PFLAG_INITIALIZING;
- ap->pflags |= ATA_PFLAG_LOADING;
- ata_port_schedule_eh(ap);
-
- spin_unlock_irqrestore(ap->lock, flags);
-
- /* wait for EH to finish */
- ata_port_wait_eh(ap);
- } else {
- DPRINTK("ata%u: bus probe begin\n", ap->print_id);
- rc = ata_bus_probe(ap);
- DPRINTK("ata%u: bus probe end\n", ap->print_id);
-
- if (rc) {
- /* FIXME: do something useful here?
- * Current libata behavior will
- * tear down everything when
- * the module is removed
- * or the h/w is unplugged.
- */
- }
- }
- }
-
- /* probes are done, now scan each port's disk(s) */
- DPRINTK("host probe begin\n");
- for (i = 0; i < host->n_ports; i++) {
- struct ata_port *ap = host->ports[i];
-
- ata_scsi_scan_host(ap, 1);
+ async_schedule(async_port_probe, ap);
}
+ DPRINTK("probe end\n");
return 0;
}
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 18486b51668..17914a346f7 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -32,6 +32,7 @@
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/spinlock.h>
+#include <linux/async.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
@@ -179,6 +180,8 @@ int scsi_complete_async_scans(void)
spin_unlock(&async_scan_lock);
kfree(data);
+ /* Synchronize async operations globally */
+ async_synchronize_full();
return 0;
}
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 62b28d58e65..e035c111401 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -48,6 +48,7 @@
#include <linux/delay.h>
#include <linux/mutex.h>
#include <linux/string_helpers.h>
+#include <linux/async.h>
#include <asm/uaccess.h>
#include <scsi/scsi.h>
@@ -1802,6 +1803,71 @@ static int sd_format_disk_name(char *prefix, int index, char *buf, int buflen)
return 0;
}
+/*
+ * The asynchronous part of sd_probe
+ */
+static void sd_probe_async(void *data, async_cookie_t cookie)
+{
+ struct scsi_disk *sdkp = data;
+ struct scsi_device *sdp;
+ struct gendisk *gd;
+ u32 index;
+ struct device *dev;
+
+ sdp = sdkp->device;
+ gd = sdkp->disk;
+ index = sdkp->index;
+ dev = &sdp->sdev_gendev;
+
+ if (!sdp->request_queue->rq_timeout) {
+ if (sdp->type != TYPE_MOD)
+ blk_queue_rq_timeout(sdp->request_queue, SD_TIMEOUT);
+ else
+ blk_queue_rq_timeout(sdp->request_queue,
+ SD_MOD_TIMEOUT);
+ }
+
+ device_initialize(&sdkp->dev);
+ sdkp->dev.parent = &sdp->sdev_gendev;
+ sdkp->dev.class = &sd_disk_class;
+ strncpy(sdkp->dev.bus_id, sdp->sdev_gendev.bus_id, BUS_ID_SIZE);
+
+ if (device_add(&sdkp->dev))
+ goto out_free_index;
+
+ get_device(&sdp->sdev_gendev);
+
+ if (index < SD_MAX_DISKS) {
+ gd->major = sd_major((index & 0xf0) >> 4);
+ gd->first_minor = ((index & 0xf) << 4) | (index & 0xfff00);
+ gd->minors = SD_MINORS;
+ }
+ gd->fops = &sd_fops;
+ gd->private_data = &sdkp->driver;
+ gd->queue = sdkp->device->request_queue;
+
+ sd_revalidate_disk(gd);
+
+ blk_queue_prep_rq(sdp->request_queue, sd_prep_fn);
+
+ gd->driverfs_dev = &sdp->sdev_gendev;
+ gd->flags = GENHD_FL_EXT_DEVT | GENHD_FL_DRIVERFS;
+ if (sdp->removable)
+ gd->flags |= GENHD_FL_REMOVABLE;
+
+ dev_set_drvdata(dev, sdkp);
+ add_disk(gd);
+ sd_dif_config_host(sdkp);
+
+ sd_printk(KERN_NOTICE, sdkp, "Attached SCSI %sdisk\n",
+ sdp->removable ? "removable " : "");
+
+ return;
+
+ out_free_index:
+ ida_remove(&sd_index_ida, index);
+}
+
/**
* sd_probe - called during driver initialization and whenever a
* new scsi device is attached to the system. It is called once
@@ -1865,48 +1931,7 @@ static int sd_probe(struct device *dev)
sdkp->openers = 0;
sdkp->previous_state = 1;
- if (!sdp->request_queue->rq_timeout) {
- if (sdp->type != TYPE_MOD)
- blk_queue_rq_timeout(sdp->request_queue, SD_TIMEOUT);
- else
- blk_queue_rq_timeout(sdp->request_queue,
- SD_MOD_TIMEOUT);
- }
-
- device_initialize(&sdkp->dev);
- sdkp->dev.parent = &sdp->sdev_gendev;
- sdkp->dev.class = &sd_disk_class;
- strncpy(sdkp->dev.bus_id, sdp->sdev_gendev.bus_id, BUS_ID_SIZE);
-
- if (device_add(&sdkp->dev))
- goto out_free_index;
-
- get_device(&sdp->sdev_gendev);
-
- if (index < SD_MAX_DISKS) {
- gd->major = sd_major((index & 0xf0) >> 4);
- gd->first_minor = ((index & 0xf) << 4) | (index & 0xfff00);
- gd->minors = SD_MINORS;
- }
- gd->fops = &sd_fops;
- gd->private_data = &sdkp->driver;
- gd->queue = sdkp->device->request_queue;
-
- sd_revalidate_disk(gd);
-
- blk_queue_prep_rq(sdp->request_queue, sd_prep_fn);
-
- gd->driverfs_dev = &sdp->sdev_gendev;
- gd->flags = GENHD_FL_EXT_DEVT | GENHD_FL_DRIVERFS;
- if (sdp->removable)
- gd->flags |= GENHD_FL_REMOVABLE;
-
- dev_set_drvdata(dev, sdkp);
- add_disk(gd);
- sd_dif_config_host(sdkp);
-
- sd_printk(KERN_NOTICE, sdkp, "Attached SCSI %sdisk\n",
- sdp->removable ? "removable " : "");
+ async_schedule(sd_probe_async, sdkp);
return 0;
diff --git a/fs/inode.c b/fs/inode.c
index 7a6e8c2ff7b..0013ac1af8e 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -22,6 +22,7 @@
#include <linux/bootmem.h>
#include <linux/inotify.h>
#include <linux/mount.h>
+#include <linux/async.h>
/*
* This is needed for the following functions:
@@ -1138,16 +1139,11 @@ EXPORT_SYMBOL(remove_inode_hash);
* I_FREEING is set so that no-one will take a new reference to the inode while
* it is being deleted.
*/
-void generic_delete_inode(struct inode *inode)
+static void generic_delete_inode_async(void *data, async_cookie_t cookie)
{
+ struct inode *inode = data;
const struct super_operations *op = inode->i_sb->s_op;
- list_del_init(&inode->i_list);
- list_del_init(&inode->i_sb_list);
- inode->i_state |= I_FREEING;
- inodes_stat.nr_inodes--;
- spin_unlock(&inode_lock);
-
security_inode_delete(inode);
if (op->delete_inode) {
@@ -1171,6 +1167,16 @@ void generic_delete_inode(struct inode *inode)
destroy_inode(inode);
}
+void generic_delete_inode(struct inode *inode)
+{
+ list_del_init(&inode->i_list);
+ list_del_init(&inode->i_sb_list);
+ inode->i_state |= I_FREEING;
+ inodes_stat.nr_inodes--;
+ spin_unlock(&inode_lock);
+ async_schedule_special(generic_delete_inode_async, inode, &inode->i_sb->s_async_list);
+}
+
EXPORT_SYMBOL(generic_delete_inode);
static void generic_forget_inode(struct inode *inode)
diff --git a/fs/super.c b/fs/super.c
index ddba069d7a9..cb20744ec78 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -38,6 +38,7 @@
#include <linux/kobject.h>
#include <linux/mutex.h>
#include <linux/file.h>
+#include <linux/async.h>
#include <asm/uaccess.h>
#include "internal.h"
@@ -71,6 +72,7 @@ static struct super_block *alloc_super(struct file_system_type *type)
INIT_HLIST_HEAD(&s->s_anon);
INIT_LIST_HEAD(&s->s_inodes);
INIT_LIST_HEAD(&s->s_dentry_lru);
+ INIT_LIST_HEAD(&s->s_async_list);
init_rwsem(&s->s_umount);
mutex_init(&s->s_lock);
lockdep_set_class(&s->s_umount, &type->s_umount_key);
@@ -289,11 +291,18 @@ void generic_shutdown_super(struct super_block *sb)
{
const struct super_operations *sop = sb->s_op;
+
if (sb->s_root) {
shrink_dcache_for_umount(sb);
fsync_super(sb);
lock_super(sb);
sb->s_flags &= ~MS_ACTIVE;
+
+ /*
+ * wait for asynchronous fs operations to finish before going further
+ */
+ async_synchronize_full_special(&sb->s_async_list);
+
/* bad name - it should be evict_inodes() */
invalidate_inodes(sb);
lock_kernel();
@@ -449,6 +458,7 @@ void sync_filesystems(int wait)
if (sb->s_flags & MS_RDONLY)
continue;
sb->s_need_sync_fs = 1;
+ async_synchronize_full_special(&sb->s_async_list);
}
restart:
diff --git a/include/linux/async.h b/include/linux/async.h
new file mode 100644
index 00000000000..c4ecacd0b32
--- /dev/null
+++ b/include/linux/async.h
@@ -0,0 +1,25 @@
+/*
+ * async.h: Asynchronous function calls for boot performance
+ *
+ * (C) Copyright 2009 Intel Corporation
+ * Author: Arjan van de Ven <arjan@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#include <linux/types.h>
+#include <linux/list.h>
+
+typedef u64 async_cookie_t;
+typedef void (async_func_ptr) (void *data, async_cookie_t cookie);
+
+extern async_cookie_t async_schedule(async_func_ptr *ptr, void *data);
+extern async_cookie_t async_schedule_special(async_func_ptr *ptr, void *data, struct list_head *list);
+extern void async_synchronize_full(void);
+extern void async_synchronize_full_special(struct list_head *list);
+extern void async_synchronize_cookie(async_cookie_t cookie);
+extern void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *list);
+
diff --git a/include/linux/fs.h b/include/linux/fs.h
index d7eba77f666..e38a64d71ef 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1184,6 +1184,11 @@ struct super_block {
* generic_show_options()
*/
char *s_options;
+
+ /*
+ * storage for asynchronous operations
+ */
+ struct list_head s_async_list;
};
extern struct timespec current_fs_time(struct super_block *sb);
diff --git a/init/do_mounts.c b/init/do_mounts.c
index 5efca73b39f..708105e163d 100644
--- a/init/do_mounts.c
+++ b/init/do_mounts.c
@@ -13,6 +13,7 @@
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/initrd.h>
+#include <linux/async.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_fs_sb.h>
@@ -372,6 +373,7 @@ void __init prepare_namespace(void)
/* wait for the known devices to complete their probing */
while (driver_probe_done() != 0)
msleep(100);
+ async_synchronize_full();
md_run_setup();
diff --git a/init/main.c b/init/main.c
index 05b31328331..844209453c0 100644
--- a/init/main.c
+++ b/init/main.c
@@ -62,6 +62,7 @@
#include <linux/signal.h>
#include <linux/idr.h>
#include <linux/ftrace.h>
+#include <linux/async.h>
#include <trace/boot.h>
#include <asm/io.h>
@@ -685,7 +686,7 @@ asmlinkage void __init start_kernel(void)
rest_init();
}
-static int initcall_debug;
+int initcall_debug;
core_param(initcall_debug, initcall_debug, bool, 0644);
int do_one_initcall(initcall_t fn)
@@ -786,6 +787,8 @@ static void run_init_process(char *init_filename)
*/
static noinline int init_post(void)
{
+ /* need to finish all async __init code before freeing the memory */
+ async_synchronize_full();
free_initmem();
unlock_kernel();
mark_rodata_ro();
diff --git a/kernel/Makefile b/kernel/Makefile
index e1c5bf3365c..2921d90ce32 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -9,7 +9,8 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
rcupdate.o extable.o params.o posix-timers.o \
kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
- notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o
+ notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
+ async.o
ifdef CONFIG_FUNCTION_TRACER
# Do not trace debug files and internal ftrace files
diff --git a/kernel/async.c b/kernel/async.c
new file mode 100644
index 00000000000..97373380c9e
--- /dev/null
+++ b/kernel/async.c
@@ -0,0 +1,321 @@
+/*
+ * async.c: Asynchronous function calls for boot performance
+ *
+ * (C) Copyright 2009 Intel Corporation
+ * Author: Arjan van de Ven <arjan@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+
+/*
+
+Goals and Theory of Operation
+
+The primary goal of this feature is to reduce the kernel boot time,
+by doing various independent hardware delays and discovery operations
+decoupled and not strictly serialized.
+
+More specifically, the asynchronous function call concept allows
+certain operations (primarily during system boot) to happen
+asynchronously, out of order, while these operations still
+have their externally visible parts happen sequentially and in-order.
+(not unlike how out-of-order CPUs retire their instructions in order)
+
+Key to the asynchronous function call implementation is the concept of
+a "sequence cookie" (which, although it has an abstracted type, can be
+thought of as a monotonically incrementing number).
+
+The async core will assign each scheduled event such a sequence cookie and
+pass this to the called functions.
+
+The asynchronously called function should before doing a globally visible
+operation, such as registering device numbers, call the
+async_synchronize_cookie() function and pass in its own cookie. The
+async_synchronize_cookie() function will make sure that all asynchronous
+operations that were scheduled prior to the operation corresponding with the
+cookie have completed.
+
+Subsystem/driver initialization code that scheduled asynchronous probe
+functions, but which shares global resources with other drivers/subsystems
+that do not use the asynchronous call feature, need to do a full
+synchronization with the async_synchronize_full() function, before returning
+from their init function. This is to maintain strict ordering between the
+asynchronous and synchronous parts of the kernel.
+
+*/
+
+#include <linux/async.h>
+#include <linux/module.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/kthread.h>
+#include <asm/atomic.h>
+
+static async_cookie_t next_cookie = 1;
+
+#define MAX_THREADS 256
+#define MAX_WORK 32768
+
+static LIST_HEAD(async_pending);
+static LIST_HEAD(async_running);
+static DEFINE_SPINLOCK(async_lock);
+
+struct async_entry {
+ struct list_head list;
+ async_cookie_t cookie;
+ async_func_ptr *func;
+ void *data;
+ struct list_head *running;
+};
+
+static DECLARE_WAIT_QUEUE_HEAD(async_done);
+static DECLARE_WAIT_QUEUE_HEAD(async_new);
+
+static atomic_t entry_count;
+static atomic_t thread_count;
+
+extern int initcall_debug;
+
+
+/*
+ * MUST be called with the lock held!
+ */
+static async_cookie_t __lowest_in_progress(struct list_head *running)
+{
+ struct async_entry *entry;
+ if (!list_empty(&async_pending)) {
+ entry = list_first_entry(&async_pending,
+ struct async_entry, list);
+ return entry->cookie;
+ } else if (!list_empty(running)) {
+ entry = list_first_entry(running,
+ struct async_entry, list);
+ return entry->cookie;
+ } else {
+ /* nothing in progress... next_cookie is "infinity" */
+ return next_cookie;
+ }
+
+}
+/*
+ * pick the first pending entry and run it
+ */
+static void run_one_entry(void)
+{
+ unsigned long flags;
+ struct async_entry *entry;
+ ktime_t calltime, delta, rettime;
+
+ /* 1) pick one task from the pending queue */
+
+ spin_lock_irqsave(&async_lock, flags);
+ if (list_empty(&async_pending))
+ goto out;
+ entry = list_first_entry(&async_pending, struct async_entry, list);
+
+ /* 2) move it to the running queue */
+ list_del(&entry->list);
+ list_add_tail(&entry->list, &async_running);
+ spin_unlock_irqrestore(&async_lock, flags);
+
+ /* 3) run it (and print duration)*/
+ if (initcall_debug && system_state == SYSTEM_BOOTING) {
+ printk("calling %lli_%pF @ %i\n", entry->cookie, entry->func, task_pid_nr(current));
+ calltime = ktime_get();
+ }
+ entry->func(entry->data, entry->cookie);
+ if (initcall_debug && system_state == SYSTEM_BOOTING) {
+ rettime = ktime_get();
+ delta = ktime_sub(rettime, calltime);
+ printk("initcall %lli_%pF returned 0 after %lld usecs\n", entry->cookie,
+ entry->func, ktime_to_ns(delta) >> 10);
+ }
+
+ /* 4) remove it from the running queue */
+ spin_lock_irqsave(&async_lock, flags);
+ list_del(&entry->list);
+
+ /* 5) free the entry */
+ kfree(entry);
+ atomic_dec(&entry_count);
+
+ spin_unlock_irqrestore(&async_lock, flags);
+
+ /* 6) wake up any waiters. */
+ wake_up(&async_done);
+ return;
+
+out:
+ spin_unlock_irqrestore(&async_lock, flags);
+}
+
+
+static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct list_head *running)
+{
+ struct async_entry *entry;
+ unsigned long flags;
+ async_cookie_t newcookie;
+
+
+ /* allow irq-off callers */
+ entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
+
+ /*
+ * If we're out of memory or if there's too much work
+ * pending already, we execute synchronously.
+ */
+ if (!entry || atomic_read(&entry_count) > MAX_WORK) {
+ kfree(entry);
+ spin_lock_irqsave(&async_lock, flags);
+ newcookie = next_cookie++;
+ spin_unlock_irqrestore(&async_lock, flags);
+
+ /* low on memory.. run synchronously */
+ ptr(data, newcookie);
+ return newcookie;
+ }
+ entry->func = ptr;
+ entry->data = data;
+ entry->running = running;
+
+ spin_lock_irqsave(&async_lock, flags);
+ newcookie = entry->cookie = next_cookie++;
+ list_add_tail(&entry->list, &async_pending);
+ atomic_inc(&entry_count);
+ spin_unlock_irqrestore(&async_lock, flags);
+ wake_up(&async_new);
+ return newcookie;
+}
+
+async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
+{
+ return __async_schedule(ptr, data, &async_pending);
+}
+EXPORT_SYMBOL_GPL(async_schedule);
+
+async_cookie_t async_schedule_special(async_func_ptr *ptr, void *data, struct list_head *running)
+{
+ return __async_schedule(ptr, data, running);
+}
+EXPORT_SYMBOL_GPL(async_schedule_special);
+
+void async_synchronize_full(void)
+{
+ async_synchronize_cookie(next_cookie);
+}
+EXPORT_SYMBOL_GPL(async_synchronize_full);
+
+void async_synchronize_full_special(struct list_head *list)
+{
+ async_synchronize_cookie_special(next_cookie, list);
+}
+EXPORT_SYMBOL_GPL(async_synchronize_full_special);
+
+void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *running)
+{
+ ktime_t starttime, delta, endtime;
+
+ if (initcall_debug && system_state == SYSTEM_BOOTING) {
+ printk("async_waiting @ %i\n", task_pid_nr(current));
+ starttime = ktime_get();
+ }
+
+ wait_event(async_done, __lowest_in_progress(running) >= cookie);
+
+ if (initcall_debug && system_state == SYSTEM_BOOTING) {
+ endtime = ktime_get();
+ delta = ktime_sub(endtime, starttime);
+
+ printk("async_continuing @ %i after %lli usec\n",
+ task_pid_nr(current), ktime_to_ns(delta) >> 10);
+ }
+}
+EXPORT_SYMBOL_GPL(async_synchronize_cookie_special);
+
+void async_synchronize_cookie(async_cookie_t cookie)
+{
+ async_synchronize_cookie_special(cookie, &async_running);
+}
+EXPORT_SYMBOL_GPL(async_synchronize_cookie);
+
+
+static int async_thread(void *unused)
+{
+ DECLARE_WAITQUEUE(wq, current);
+ add_wait_queue(&async_new, &wq);
+
+ while (!kthread_should_stop()) {
+ int ret = HZ;
+ set_current_state(TASK_INTERRUPTIBLE);
+ /*
+ * check the list head without lock.. false positives
+ * are dealt with inside run_one_entry() while holding
+ * the lock.
+ */
+ rmb();
+ if (!list_empty(&async_pending))
+ run_one_entry();
+ else
+ ret = schedule_timeout(HZ);
+
+ if (ret == 0) {
+ /*
+ * we timed out, this means we as thread are redundant.
+ * we sign off and die, but we to avoid any races there
+ * is a last-straw check to see if work snuck in.
+ */
+ atomic_dec(&thread_count);
+ wmb(); /* manager must see our departure first */
+ if (list_empty(&async_pending))
+ break;
+ /*
+ * woops work came in between us timing out and us
+ * signing off; we need to stay alive and keep working.
+ */
+ atomic_inc(&thread_count);
+ }
+ }
+ remove_wait_queue(&async_new, &wq);
+
+ return 0;
+}
+
+static int async_manager_thread(void *unused)
+{
+ DECLARE_WAITQUEUE(wq, current);
+ add_wait_queue(&async_new, &wq);
+
+ while (!kthread_should_stop()) {
+ int tc, ec;
+
+ set_current_state(TASK_INTERRUPTIBLE);
+
+ tc = atomic_read(&thread_count);
+ rmb();
+ ec = atomic_read(&entry_count);
+
+ while (tc < ec && tc < MAX_THREADS) {
+ kthread_run(async_thread, NULL, "async/%i", tc);
+ atomic_inc(&thread_count);
+ tc++;
+ }
+
+ schedule();
+ }
+ remove_wait_queue(&async_new, &wq);
+
+ return 0;
+}
+
+static int __init async_init(void)
+{
+ kthread_run(async_manager_thread, NULL, "async/mgr");
+ return 0;
+}
+
+core_initcall(async_init);
diff --git a/kernel/irq/autoprobe.c b/kernel/irq/autoprobe.c
index cc0f7321b8c..1de9700f416 100644
--- a/kernel/irq/autoprobe.c
+++ b/kernel/irq/autoprobe.c
@@ -10,6 +10,7 @@
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
+#include <linux/async.h>
#include "internals.h"
@@ -34,6 +35,10 @@ unsigned long probe_irq_on(void)
unsigned int status;
int i;
+ /*
+ * quiesce the kernel, or at least the asynchronous portion
+ */
+ async_synchronize_full();
mutex_lock(&probing_active);
/*
* something may have generated an irq long ago and we want to
diff --git a/kernel/module.c b/kernel/module.c
index 496dcb57b60..c9332c90d5a 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -50,6 +50,7 @@
#include <asm/sections.h>
#include <linux/tracepoint.h>
#include <linux/ftrace.h>
+#include <linux/async.h>
#if 0
#define DEBUGP printk
@@ -816,6 +817,7 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
mod->exit();
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_GOING, mod);
+ async_synchronize_full();
mutex_lock(&module_mutex);
/* Store the name of the last unloaded module for diagnostic purposes */
strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));