diff options
Diffstat (limited to 'include/asm-s390')
-rw-r--r-- | include/asm-s390/bitops.h | 5 | ||||
-rw-r--r-- | include/asm-s390/cache.h | 2 | ||||
-rw-r--r-- | include/asm-s390/ccwdev.h | 77 | ||||
-rw-r--r-- | include/asm-s390/ccwgroup.h | 32 | ||||
-rw-r--r-- | include/asm-s390/cio.h | 288 | ||||
-rw-r--r-- | include/asm-s390/cmb.h | 73 | ||||
-rw-r--r-- | include/asm-s390/cpu.h | 25 | ||||
-rw-r--r-- | include/asm-s390/ipc.h | 1 | ||||
-rw-r--r-- | include/asm-s390/kdebug.h | 15 | ||||
-rw-r--r-- | include/asm-s390/kprobes.h | 2 | ||||
-rw-r--r-- | include/asm-s390/mmu_context.h | 50 | ||||
-rw-r--r-- | include/asm-s390/page.h | 7 | ||||
-rw-r--r-- | include/asm-s390/pgalloc.h | 250 | ||||
-rw-r--r-- | include/asm-s390/pgtable.h | 442 | ||||
-rw-r--r-- | include/asm-s390/processor.h | 20 | ||||
-rw-r--r-- | include/asm-s390/s390_ext.h | 8 | ||||
-rw-r--r-- | include/asm-s390/scatterlist.h | 5 | ||||
-rw-r--r-- | include/asm-s390/semaphore.h | 1 | ||||
-rw-r--r-- | include/asm-s390/system.h | 24 | ||||
-rw-r--r-- | include/asm-s390/tlb.h | 129 | ||||
-rw-r--r-- | include/asm-s390/tlbflush.h | 159 | ||||
-rw-r--r-- | include/asm-s390/types.h | 6 | ||||
-rw-r--r-- | include/asm-s390/zcrypt.h | 4 |
23 files changed, 864 insertions, 761 deletions
diff --git a/include/asm-s390/bitops.h b/include/asm-s390/bitops.h index f79c9b792af..34d9a6357c3 100644 --- a/include/asm-s390/bitops.h +++ b/include/asm-s390/bitops.h @@ -15,6 +15,10 @@ #ifdef __KERNEL__ +#ifndef _LINUX_BITOPS_H +#error only <linux/bitops.h> can be included directly +#endif + #include <linux/compiler.h> /* @@ -746,6 +750,7 @@ static inline int sched_find_first_bit(unsigned long *b) #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/hweight.h> +#include <asm-generic/bitops/lock.h> /* * ATTENTION: intel byte ordering convention for ext2 and minix !! diff --git a/include/asm-s390/cache.h b/include/asm-s390/cache.h index cdf431b061b..9b866816863 100644 --- a/include/asm-s390/cache.h +++ b/include/asm-s390/cache.h @@ -14,8 +14,6 @@ #define L1_CACHE_BYTES 256 #define L1_CACHE_SHIFT 8 -#define ARCH_KMALLOC_MINALIGN 8 - #define __read_mostly __attribute__((__section__(".data.read_mostly"))) #endif diff --git a/include/asm-s390/ccwdev.h b/include/asm-s390/ccwdev.h index 1aeda27d5a8..066aa70518c 100644 --- a/include/asm-s390/ccwdev.h +++ b/include/asm-s390/ccwdev.h @@ -67,36 +67,55 @@ ccw_device_id_match(const struct ccw_device_id *array, return NULL; } -/* The struct ccw device is our replacement for the globally accessible - * ioinfo array. ioinfo will mutate into a subchannel device later. +/** + * struct ccw_device - channel attached device + * @ccwlock: pointer to device lock + * @id: id of this device + * @drv: ccw driver for this device + * @dev: embedded device structure + * @online: online status of device + * @handler: interrupt handler * - * Reference: Documentation/s390/driver-model.txt */ + * @handler is a member of the device rather than the driver since a driver + * can have different interrupt handlers for different ccw devices + * (multi-subchannel drivers). + */ struct ccw_device { spinlock_t *ccwlock; +/* private: */ struct ccw_device_private *private; /* cio private information */ - struct ccw_device_id id; /* id of this device, driver_info is - set by ccw_find_driver */ - struct ccw_driver *drv; /* */ - struct device dev; /* */ +/* public: */ + struct ccw_device_id id; + struct ccw_driver *drv; + struct device dev; int online; - /* This is sick, but a driver can have different interrupt handlers - for different ccw_devices (multi-subchannel drivers)... */ void (*handler) (struct ccw_device *, unsigned long, struct irb *); }; -/* Each ccw driver registers with the ccw root bus */ +/** + * struct ccw driver - device driver for channel attached devices + * @owner: owning module + * @ids: ids supported by this driver + * @probe: function called on probe + * @remove: function called on remove + * @set_online: called when setting device online + * @set_offline: called when setting device offline + * @notify: notify driver of device state changes + * @shutdown: called at device shutdown + * @driver: embedded device driver structure + * @name: device driver name + */ struct ccw_driver { - struct module *owner; /* for automatic MOD_INC_USE_COUNT */ - struct ccw_device_id *ids; /* probe driver with these devs */ - int (*probe) (struct ccw_device *); /* ask driver to probe dev */ + struct module *owner; + struct ccw_device_id *ids; + int (*probe) (struct ccw_device *); void (*remove) (struct ccw_device *); - /* device is no longer available */ int (*set_online) (struct ccw_device *); int (*set_offline) (struct ccw_device *); int (*notify) (struct ccw_device *, int); - struct device_driver driver; /* higher level structure, don't init - this from your driver */ + void (*shutdown) (struct ccw_device *); + struct device_driver driver; char *name; }; @@ -124,36 +143,10 @@ extern void ccw_device_clear_options(struct ccw_device *, unsigned long); /* Allow forced onlining of boxed devices. */ #define CCWDEV_ALLOW_FORCE 0x0008 -/* - * ccw_device_start() - * - * Start a S/390 channel program. When the interrupt arrives, the - * IRQ handler is called, either immediately, delayed (dev-end missing, - * or sense required) or never (no IRQ handler registered). - * Depending on the action taken, ccw_device_start() returns: - * 0 - Success - * -EBUSY - Device busy, or status pending - * -ENODEV - Device not operational - * -EINVAL - Device invalid for operation - */ extern int ccw_device_start(struct ccw_device *, struct ccw1 *, unsigned long, __u8, unsigned long); -/* - * ccw_device_start_timeout() - * - * This function notifies the device driver if the channel program has not - * completed during the specified time. If a timeout occurs, the channel - * program is terminated via xsch(), hsch() or csch(). - */ extern int ccw_device_start_timeout(struct ccw_device *, struct ccw1 *, unsigned long, __u8, unsigned long, int); -/* - * ccw_device_start_key() - * ccw_device_start_key_timeout() - * - * Same as ccw_device_start() and ccw_device_start_timeout(), except a - * storage key != default key can be provided for the I/O. - */ extern int ccw_device_start_key(struct ccw_device *, struct ccw1 *, unsigned long, __u8, __u8, unsigned long); extern int ccw_device_start_timeout_key(struct ccw_device *, struct ccw1 *, diff --git a/include/asm-s390/ccwgroup.h b/include/asm-s390/ccwgroup.h index 925b3ddfa14..7109c7cab87 100644 --- a/include/asm-s390/ccwgroup.h +++ b/include/asm-s390/ccwgroup.h @@ -4,19 +4,41 @@ struct ccw_device; struct ccw_driver; +/** + * struct ccwgroup_device - ccw group device + * @creator_id: unique number of the driver + * @state: online/offline state + * @count: number of attached slave devices + * @dev: embedded device structure + * @cdev: variable number of slave devices, allocated as needed + */ struct ccwgroup_device { - unsigned long creator_id; /* unique number of the driver */ + unsigned long creator_id; enum { CCWGROUP_OFFLINE, CCWGROUP_ONLINE, } state; +/* private: */ atomic_t onoff; struct mutex reg_mutex; - unsigned int count; /* number of attached slave devices */ - struct device dev; /* master device */ - struct ccw_device *cdev[0]; /* variable number, allocate as needed */ +/* public: */ + unsigned int count; + struct device dev; + struct ccw_device *cdev[0]; }; +/** + * struct ccwgroup_driver - driver for ccw group devices + * @owner: driver owner + * @name: driver name + * @max_slaves: maximum number of slave devices + * @driver_id: unique id + * @probe: function called on probe + * @remove: function called on remove + * @set_online: function called when device is set online + * @set_offline: function called when device is set offline + * @driver: embedded driver structure + */ struct ccwgroup_driver { struct module *owner; char *name; @@ -28,7 +50,7 @@ struct ccwgroup_driver { int (*set_online) (struct ccwgroup_device *); int (*set_offline) (struct ccwgroup_device *); - struct device_driver driver; /* this driver */ + struct device_driver driver; }; extern int ccwgroup_driver_register (struct ccwgroup_driver *cdriver); diff --git a/include/asm-s390/cio.h b/include/asm-s390/cio.h index 1982fb34416..2f08c16e44a 100644 --- a/include/asm-s390/cio.h +++ b/include/asm-s390/cio.h @@ -15,30 +15,50 @@ #define LPM_ANYPATH 0xff #define __MAX_CSSID 0 -/* - * subchannel status word +/** + * struct scsw - subchannel status word + * @key: subchannel key + * @sctl: suspend control + * @eswf: esw format + * @cc: deferred condition code + * @fmt: format + * @pfch: prefetch + * @isic: initial-status interruption control + * @alcc: adress-limit checking control + * @ssi: supress-suspended interruption + * @zcc: zero condition code + * @ectl: extended control + * @pno: path not operational + * @res: reserved + * @fctl: function control + * @actl: activity control + * @stctl: status control + * @cpa: channel program address + * @dstat: device status + * @cstat: subchannel status + * @count: residual count */ struct scsw { - __u32 key : 4; /* subchannel key */ - __u32 sctl : 1; /* suspend control */ - __u32 eswf : 1; /* ESW format */ - __u32 cc : 2; /* deferred condition code */ - __u32 fmt : 1; /* format */ - __u32 pfch : 1; /* prefetch */ - __u32 isic : 1; /* initial-status interruption control */ - __u32 alcc : 1; /* address-limit checking control */ - __u32 ssi : 1; /* supress-suspended interruption */ - __u32 zcc : 1; /* zero condition code */ - __u32 ectl : 1; /* extended control */ - __u32 pno : 1; /* path not operational */ - __u32 res : 1; /* reserved */ - __u32 fctl : 3; /* function control */ - __u32 actl : 7; /* activity control */ - __u32 stctl : 5; /* status control */ - __u32 cpa; /* channel program address */ - __u32 dstat : 8; /* device status */ - __u32 cstat : 8; /* subchannel status */ - __u32 count : 16; /* residual count */ + __u32 key : 4; + __u32 sctl : 1; + __u32 eswf : 1; + __u32 cc : 2; + __u32 fmt : 1; + __u32 pfch : 1; + __u32 isic : 1; + __u32 alcc : 1; + __u32 ssi : 1; + __u32 zcc : 1; + __u32 ectl : 1; + __u32 pno : 1; + __u32 res : 1; + __u32 fctl : 3; + __u32 actl : 7; + __u32 stctl : 5; + __u32 cpa; + __u32 dstat : 8; + __u32 cstat : 8; + __u32 count : 16; } __attribute__ ((packed)); #define SCSW_FCTL_CLEAR_FUNC 0x1 @@ -110,11 +130,22 @@ struct scsw { #define SNS2_ENV_DATA_PRESENT 0x10 #define SNS2_INPRECISE_END 0x04 +/** + * struct ccw1 - channel command word + * @cmd_code: command code + * @flags: flags, like IDA adressing, etc. + * @count: byte count + * @cda: data address + * + * The ccw is the basic structure to build channel programs that perform + * operations with the device or the control unit. Only Format-1 channel + * command words are supported. + */ struct ccw1 { - __u8 cmd_code; /* command code */ - __u8 flags; /* flags, like IDA addressing, etc. */ - __u16 count; /* byte count */ - __u32 cda; /* data address */ + __u8 cmd_code; + __u8 flags; + __u16 count; + __u32 cda; } __attribute__ ((packed,aligned(8))); #define CCW_FLAG_DC 0x80 @@ -140,102 +171,162 @@ struct ccw1 { #define SENSE_MAX_COUNT 0x20 +/** + * struct erw - extended report word + * @res0: reserved + * @auth: authorization check + * @pvrf: path-verification-required flag + * @cpt: channel-path timeout + * @fsavf: failing storage address validity flag + * @cons: concurrent sense + * @scavf: secondary ccw address validity flag + * @fsaf: failing storage address format + * @scnt: sense count, if @cons == %1 + * @res16: reserved + */ struct erw { - __u32 res0 : 3; /* reserved */ - __u32 auth : 1; /* Authorization check */ - __u32 pvrf : 1; /* path-verification-required flag */ - __u32 cpt : 1; /* channel-path timeout */ - __u32 fsavf : 1; /* Failing storage address validity flag */ - __u32 cons : 1; /* concurrent-sense */ - __u32 scavf : 1; /* Secondary ccw address validity flag */ - __u32 fsaf : 1; /* Failing storage address format */ - __u32 scnt : 6; /* sense count if cons == 1 */ - __u32 res16 : 16; /* reserved */ + __u32 res0 : 3; + __u32 auth : 1; + __u32 pvrf : 1; + __u32 cpt : 1; + __u32 fsavf : 1; + __u32 cons : 1; + __u32 scavf : 1; + __u32 fsaf : 1; + __u32 scnt : 6; + __u32 res16 : 16; } __attribute__ ((packed)); -/* - * subchannel logout area +/** + * struct sublog - subchannel logout area + * @res0: reserved + * @esf: extended status flags + * @lpum: last path used mask + * @arep: ancillary report + * @fvf: field-validity flags + * @sacc: storage access code + * @termc: termination code + * @devsc: device-status check + * @serr: secondary error + * @ioerr: i/o-error alert + * @seqc: sequence code */ struct sublog { - __u32 res0 : 1; /* reserved */ - __u32 esf : 7; /* extended status flags */ - __u32 lpum : 8; /* last path used mask */ - __u32 arep : 1; /* ancillary report */ - __u32 fvf : 5; /* field-validity flags */ - __u32 sacc : 2; /* storage access code */ - __u32 termc : 2; /* termination code */ - __u32 devsc : 1; /* device-status check */ - __u32 serr : 1; /* secondary error */ - __u32 ioerr : 1; /* i/o-error alert */ - __u32 seqc : 3; /* sequence code */ + __u32 res0 : 1; + __u32 esf : 7; + __u32 lpum : 8; + __u32 arep : 1; + __u32 fvf : 5; + __u32 sacc : 2; + __u32 termc : 2; + __u32 devsc : 1; + __u32 serr : 1; + __u32 ioerr : 1; + __u32 seqc : 3; } __attribute__ ((packed)); -/* - * Format 0 Extended Status Word (ESW) +/** + * struct esw0 - Format 0 Extended Status Word (ESW) + * @sublog: subchannel logout + * @erw: extended report word + * @faddr: failing storage address + * @saddr: secondary ccw address */ struct esw0 { - struct sublog sublog; /* subchannel logout */ - struct erw erw; /* extended report word */ - __u32 faddr[2]; /* failing storage address */ - __u32 saddr; /* secondary ccw address */ + struct sublog sublog; + struct erw erw; + __u32 faddr[2]; + __u32 saddr; } __attribute__ ((packed)); -/* - * Format 1 Extended Status Word (ESW) +/** + * struct esw1 - Format 1 Extended Status Word (ESW) + * @zero0: reserved zeros + * @lpum: last path used mask + * @zero16: reserved zeros + * @erw: extended report word + * @zeros: three fullwords of zeros */ struct esw1 { - __u8 zero0; /* reserved zeros */ - __u8 lpum; /* last path used mask */ - __u16 zero16; /* reserved zeros */ - struct erw erw; /* extended report word */ - __u32 zeros[3]; /* 2 fullwords of zeros */ + __u8 zero0; + __u8 lpum; + __u16 zero16; + struct erw erw; + __u32 zeros[3]; } __attribute__ ((packed)); -/* - * Format 2 Extended Status Word (ESW) +/** + * struct esw2 - Format 2 Extended Status Word (ESW) + * @zero0: reserved zeros + * @lpum: last path used mask + * @dcti: device-connect-time interval + * @erw: extended report word + * @zeros: three fullwords of zeros */ struct esw2 { - __u8 zero0; /* reserved zeros */ - __u8 lpum; /* last path used mask */ - __u16 dcti; /* device-connect-time interval */ - struct erw erw; /* extended report word */ - __u32 zeros[3]; /* 2 fullwords of zeros */ + __u8 zero0; + __u8 lpum; + __u16 dcti; + struct erw erw; + __u32 zeros[3]; } __attribute__ ((packed)); -/* - * Format 3 Extended Status Word (ESW) +/** + * struct esw3 - Format 3 Extended Status Word (ESW) + * @zero0: reserved zeros + * @lpum: last path used mask + * @res: reserved + * @erw: extended report word + * @zeros: three fullwords of zeros */ struct esw3 { - __u8 zero0; /* reserved zeros */ - __u8 lpum; /* last path used mask */ - __u16 res; /* reserved */ - struct erw erw; /* extended report word */ - __u32 zeros[3]; /* 2 fullwords of zeros */ + __u8 zero0; + __u8 lpum; + __u16 res; + struct erw erw; + __u32 zeros[3]; } __attribute__ ((packed)); -/* - * interruption response block +/** + * struct irb - interruption response block + * @scsw: subchannel status word + * @esw: extened status word, 4 formats + * @ecw: extended control word + * + * The irb that is handed to the device driver when an interrupt occurs. For + * solicited interrupts, the common I/O layer already performs checks whether + * a field is valid; a field not being valid is always passed as %0. + * If a unit check occured, @ecw may contain sense data; this is retrieved + * by the common I/O layer itself if the device doesn't support concurrent + * sense (so that the device driver never needs to perform basic sene itself). + * For unsolicited interrupts, the irb is passed as-is (expect for sense data, + * if applicable). */ struct irb { - struct scsw scsw; /* subchannel status word */ - union { /* extended status word, 4 formats */ + struct scsw scsw; + union { struct esw0 esw0; struct esw1 esw1; struct esw2 esw2; struct esw3 esw3; } esw; - __u8 ecw[32]; /* extended control word */ + __u8 ecw[32]; } __attribute__ ((packed,aligned(4))); -/* - * command information word (CIW) layout +/** + * struct ciw - command information word (CIW) layout + * @et: entry type + * @reserved: reserved bits + * @ct: command type + * @cmd: command code + * @count: command count */ struct ciw { - __u32 et : 2; /* entry type */ - __u32 reserved : 2; /* reserved */ - __u32 ct : 4; /* command type */ - __u32 cmd : 8; /* command */ - __u32 count : 16; /* coun */ + __u32 et : 2; + __u32 reserved : 2; + __u32 ct : 4; + __u32 cmd : 8; + __u32 count : 16; } __attribute__ ((packed)); #define CIW_TYPE_RCD 0x0 /* read configuration data */ @@ -258,11 +349,32 @@ struct ciw { /* Sick revalidation of device. */ #define CIO_REVALIDATE 0x0008 +/** + * struct ccw_dev_id - unique identifier for ccw devices + * @ssid: subchannel set id + * @devno: device number + * + * This structure is not directly based on any hardware structure. The + * hardware identifies a device by its device number and its subchannel, + * which is in turn identified by its id. In order to get a unique identifier + * for ccw devices across subchannel sets, @struct ccw_dev_id has been + * introduced. + */ struct ccw_dev_id { u8 ssid; u16 devno; }; +/** + * ccw_device_id_is_equal() - compare two ccw_dev_ids + * @dev_id1: a ccw_dev_id + * @dev_id2: another ccw_dev_id + * Returns: + * %1 if the two structures are equal field-by-field, + * %0 if not. + * Context: + * any + */ static inline int ccw_dev_id_is_equal(struct ccw_dev_id *dev_id1, struct ccw_dev_id *dev_id2) { diff --git a/include/asm-s390/cmb.h b/include/asm-s390/cmb.h index 021e7c3223e..50196857d27 100644 --- a/include/asm-s390/cmb.h +++ b/include/asm-s390/cmb.h @@ -1,29 +1,29 @@ #ifndef S390_CMB_H #define S390_CMB_H /** - * struct cmbdata -- channel measurement block data for user space + * struct cmbdata - channel measurement block data for user space + * @size: size of the stored data + * @elapsed_time: time since last sampling + * @ssch_rsch_count: number of ssch and rsch + * @sample_count: number of samples + * @device_connect_time: time of device connect + * @function_pending_time: time of function pending + * @device_disconnect_time: time of device disconnect + * @control_unit_queuing_time: time of control unit queuing + * @device_active_only_time: time of device active only + * @device_busy_time: time of device busy (ext. format) + * @initial_command_response_time: initial command response time (ext. format) * - * @size: size of the stored data - * @ssch_rsch_count: XXX - * @sample_count: - * @device_connect_time: - * @function_pending_time: - * @device_disconnect_time: - * @control_unit_queuing_time: - * @device_active_only_time: - * @device_busy_time: - * @initial_command_response_time: - * - * all values are stored as 64 bit for simplicity, especially + * All values are stored as 64 bit for simplicity, especially * in 32 bit emulation mode. All time values are normalized to * nanoseconds. * Currently, two formats are known, which differ by the size of * this structure, i.e. the last two members are only set when * the extended channel measurement facility (first shipped in * z990 machines) is activated. - * Potentially, more fields could be added, which results in a + * Potentially, more fields could be added, which would result in a * new ioctl number. - **/ + */ struct cmbdata { __u64 size; __u64 elapsed_time; @@ -41,53 +41,18 @@ struct cmbdata { }; /* enable channel measurement */ -#define BIODASDCMFENABLE _IO(DASD_IOCTL_LETTER,32) +#define BIODASDCMFENABLE _IO(DASD_IOCTL_LETTER, 32) /* enable channel measurement */ -#define BIODASDCMFDISABLE _IO(DASD_IOCTL_LETTER,33) +#define BIODASDCMFDISABLE _IO(DASD_IOCTL_LETTER, 33) /* read channel measurement data */ -#define BIODASDREADALLCMB _IOWR(DASD_IOCTL_LETTER,33,struct cmbdata) +#define BIODASDREADALLCMB _IOWR(DASD_IOCTL_LETTER, 33, struct cmbdata) #ifdef __KERNEL__ struct ccw_device; -/** - * enable_cmf() - switch on the channel measurement for a specific device - * @cdev: The ccw device to be enabled - * returns 0 for success or a negative error value. - * - * Context: - * non-atomic - **/ extern int enable_cmf(struct ccw_device *cdev); - -/** - * disable_cmf() - switch off the channel measurement for a specific device - * @cdev: The ccw device to be disabled - * returns 0 for success or a negative error value. - * - * Context: - * non-atomic - **/ extern int disable_cmf(struct ccw_device *cdev); - -/** - * cmf_read() - read one value from the current channel measurement block - * @cmf: the channel to be read - * @index: the name of the value that is read - * - * Context: - * any - **/ - extern u64 cmf_read(struct ccw_device *cdev, int index); -/** - * cmf_readall() - read one value from the current channel measurement block - * @cmf: the channel to be read - * @data: a pointer to a data block that will be filled - * - * Context: - * any - **/ -extern int cmf_readall(struct ccw_device *cdev, struct cmbdata*data); +extern int cmf_readall(struct ccw_device *cdev, struct cmbdata *data); #endif /* __KERNEL__ */ #endif /* S390_CMB_H */ diff --git a/include/asm-s390/cpu.h b/include/asm-s390/cpu.h new file mode 100644 index 00000000000..352dde194f3 --- /dev/null +++ b/include/asm-s390/cpu.h @@ -0,0 +1,25 @@ +/* + * include/asm-s390/cpu.h + * + * Copyright IBM Corp. 2007 + * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> + */ + +#ifndef _ASM_S390_CPU_H_ +#define _ASM_S390_CPU_H_ + +#include <linux/types.h> +#include <linux/percpu.h> +#include <linux/spinlock.h> + +struct s390_idle_data { + spinlock_t lock; + unsigned int in_idle; + unsigned long long idle_count; + unsigned long long idle_enter; + unsigned long long idle_time; +}; + +DECLARE_PER_CPU(struct s390_idle_data, s390_idle); + +#endif /* _ASM_S390_CPU_H_ */ diff --git a/include/asm-s390/ipc.h b/include/asm-s390/ipc.h deleted file mode 100644 index a46e3d9c2a3..00000000000 --- a/include/asm-s390/ipc.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/ipc.h> diff --git a/include/asm-s390/kdebug.h b/include/asm-s390/kdebug.h index 04418af08f8..40db27cd6e6 100644 --- a/include/asm-s390/kdebug.h +++ b/include/asm-s390/kdebug.h @@ -4,24 +4,9 @@ /* * Feb 2006 Ported to s390 <grundym@us.ibm.com> */ -#include <linux/notifier.h> struct pt_regs; -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} - enum die_val { DIE_OOPS = 1, DIE_BPT, diff --git a/include/asm-s390/kprobes.h b/include/asm-s390/kprobes.h index 340ba10446e..948db3d0d05 100644 --- a/include/asm-s390/kprobes.h +++ b/include/asm-s390/kprobes.h @@ -47,7 +47,7 @@ typedef u16 kprobe_opcode_t; : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 0 +#define kretprobe_blacklist_size 0 #define KPROBE_SWAP_INST 0x10 diff --git a/include/asm-s390/mmu_context.h b/include/asm-s390/mmu_context.h index 501cb9b0631..05b842126b9 100644 --- a/include/asm-s390/mmu_context.h +++ b/include/asm-s390/mmu_context.h @@ -21,45 +21,43 @@ #ifndef __s390x__ #define LCTL_OPCODE "lctl" -#define PGTABLE_BITS (_SEGMENT_TABLE|USER_STD_MASK) #else #define LCTL_OPCODE "lctlg" -#define PGTABLE_BITS (_REGION_TABLE|USER_STD_MASK) #endif -static inline void enter_lazy_tlb(struct mm_struct *mm, - struct task_struct *tsk) +static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk) { + pgd_t *pgd = mm->pgd; + unsigned long asce_bits; + + /* Calculate asce bits from the first pgd table entry. */ + asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS; +#ifdef CONFIG_64BIT + asce_bits |= _ASCE_TYPE_REGION3; +#endif + S390_lowcore.user_asce = asce_bits | __pa(pgd); + if (switch_amode) { + /* Load primary space page table origin. */ + pgd_t *shadow_pgd = get_shadow_table(pgd) ? : pgd; + S390_lowcore.user_exec_asce = asce_bits | __pa(shadow_pgd); + asm volatile(LCTL_OPCODE" 1,1,%0\n" + : : "m" (S390_lowcore.user_exec_asce) ); + } else + /* Load home space page table origin. */ + asm volatile(LCTL_OPCODE" 13,13,%0" + : : "m" (S390_lowcore.user_asce) ); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { - pgd_t *shadow_pgd = get_shadow_pgd(next->pgd); - - if (prev != next) { - S390_lowcore.user_asce = (__pa(next->pgd) & PAGE_MASK) | - PGTABLE_BITS; - if (shadow_pgd) { - /* Load primary/secondary space page table origin. */ - S390_lowcore.user_exec_asce = - (__pa(shadow_pgd) & PAGE_MASK) | PGTABLE_BITS; - asm volatile(LCTL_OPCODE" 1,1,%0\n" - LCTL_OPCODE" 7,7,%1" - : : "m" (S390_lowcore.user_exec_asce), - "m" (S390_lowcore.user_asce) ); - } else if (switch_amode) { - /* Load primary space page table origin. */ - asm volatile(LCTL_OPCODE" 1,1,%0" - : : "m" (S390_lowcore.user_asce) ); - } else - /* Load home space page table origin. */ - asm volatile(LCTL_OPCODE" 13,13,%0" - : : "m" (S390_lowcore.user_asce) ); - } + if (unlikely(prev == next)) + return; cpu_set(smp_processor_id(), next->cpu_vm_mask); + update_mm(next, tsk); } +#define enter_lazy_tlb(mm,tsk) do { } while (0) #define deactivate_mm(tsk,mm) do { } while (0) static inline void activate_mm(struct mm_struct *prev, diff --git a/include/asm-s390/page.h b/include/asm-s390/page.h index f326451ed6e..584d0ee3c7f 100644 --- a/include/asm-s390/page.h +++ b/include/asm-s390/page.h @@ -9,11 +9,12 @@ #ifndef _S390_PAGE_H #define _S390_PAGE_H +#include <linux/const.h> #include <asm/types.h> /* PAGE_SHIFT determines the page size */ #define PAGE_SHIFT 12 -#define PAGE_SIZE (1UL << PAGE_SHIFT) +#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) #define PAGE_DEFAULT_ACC 0 #define PAGE_DEFAULT_KEY (PAGE_DEFAULT_ACC << 4) @@ -81,6 +82,7 @@ typedef struct { unsigned long pte; } pte_t; #ifndef __s390x__ typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pud; } pud_t; typedef struct { unsigned long pgd0; unsigned long pgd1; @@ -89,6 +91,7 @@ typedef struct { } pgd_t; #define pmd_val(x) ((x).pmd) +#define pud_val(x) ((x).pud) #define pgd_val(x) ((x).pgd0) #else /* __s390x__ */ @@ -97,10 +100,12 @@ typedef struct { unsigned long pmd0; unsigned long pmd1; } pmd_t; +typedef struct { unsigned long pud; } pud_t; typedef struct { unsigned long pgd; } pgd_t; #define pmd_val(x) ((x).pmd0) #define pmd_val1(x) ((x).pmd1) +#define pud_val(x) ((x).pud) #define pgd_val(x) ((x).pgd) #endif /* __s390x__ */ diff --git a/include/asm-s390/pgalloc.h b/include/asm-s390/pgalloc.h index e45d3c9a4b7..709dd174095 100644 --- a/include/asm-s390/pgalloc.h +++ b/include/asm-s390/pgalloc.h @@ -19,140 +19,115 @@ #define check_pgt_cache() do {} while (0) -/* - * Page allocation orders. - */ -#ifndef __s390x__ -# define PTE_ALLOC_ORDER 0 -# define PMD_ALLOC_ORDER 0 -# define PGD_ALLOC_ORDER 1 -#else /* __s390x__ */ -# define PTE_ALLOC_ORDER 0 -# define PMD_ALLOC_ORDER 2 -# define PGD_ALLOC_ORDER 2 -#endif /* __s390x__ */ +unsigned long *crst_table_alloc(struct mm_struct *, int); +void crst_table_free(unsigned long *); -/* - * Allocate and free page tables. The xxx_kernel() versions are - * used to allocate a kernel page table - this turns on ASN bits - * if any. - */ +unsigned long *page_table_alloc(int); +void page_table_free(unsigned long *); -static inline pgd_t *pgd_alloc(struct mm_struct *mm) +static inline void clear_table(unsigned long *s, unsigned long val, size_t n) { - pgd_t *pgd = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ALLOC_ORDER); - int i; - - if (!pgd) - return NULL; - if (s390_noexec) { - pgd_t *shadow_pgd = (pgd_t *) - __get_free_pages(GFP_KERNEL, PGD_ALLOC_ORDER); - struct page *page = virt_to_page(pgd); - - if (!shadow_pgd) { - free_pages((unsigned long) pgd, PGD_ALLOC_ORDER); - return NULL; - } - page->lru.next = (void *) shadow_pgd; - } - for (i = 0; i < PTRS_PER_PGD; i++) -#ifndef __s390x__ - pmd_clear(pmd_offset(pgd + i, i*PGDIR_SIZE)); + *s = val; + n = (n / 256) - 1; + asm volatile( +#ifdef CONFIG_64BIT + " mvc 8(248,%0),0(%0)\n" #else - pgd_clear(pgd + i); + " mvc 4(252,%0),0(%0)\n" #endif - return pgd; + "0: mvc 256(256,%0),0(%0)\n" + " la %0,256(%0)\n" + " brct %1,0b\n" + : "+a" (s), "+d" (n)); } -static inline void pgd_free(pgd_t *pgd) +static inline void crst_table_init(unsigned long *crst, unsigned long entry) { - pgd_t *shadow_pgd = get_shadow_pgd(pgd); - - if (shadow_pgd) - free_pages((unsigned long) shadow_pgd, PGD_ALLOC_ORDER); - free_pages((unsigned long) pgd, PGD_ALLOC_ORDER); + clear_table(crst, entry, sizeof(unsigned long)*2048); + crst = get_shadow_table(crst); + if (crst) + clear_table(crst, entry, sizeof(unsigned long)*2048); } #ifndef __s390x__ -/* - * page middle directory allocation/free routines. - * We use pmd cache only on s390x, so these are dummy routines. This - * code never triggers because the pgd will always be present. - */ -#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(x) do { } while (0) -#define __pmd_free_tlb(tlb,x) do { } while (0) -#define pgd_populate(mm, pmd, pte) BUG() -#define pgd_populate_kernel(mm, pmd, pte) BUG() -#else /* __s390x__ */ -static inline pmd_t * pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr) + +static inline unsigned long pgd_entry_type(struct mm_struct *mm) { - pmd_t *pmd = (pmd_t *) __get_free_pages(GFP_KERNEL, PMD_ALLOC_ORDER); - int i; - - if (!pmd) - return NULL; - if (s390_noexec) { - pmd_t *shadow_pmd = (pmd_t *) - __get_free_pages(GFP_KERNEL, PMD_ALLOC_ORDER); - struct page *page = virt_to_page(pmd); - - if (!shadow_pmd) { - free_pages((unsigned long) pmd, PMD_ALLOC_ORDER); - return NULL; - } - page->lru.next = (void *) shadow_pmd; - } - for (i=0; i < PTRS_PER_PMD; i++) - pmd_clear(pmd + i); - return pmd; + return _SEGMENT_ENTRY_EMPTY; } -static inline void pmd_free (pmd_t *pmd) +#define pud_alloc_one(mm,address) ({ BUG(); ((pud_t *)2); }) +#define pud_free(x) do { } while (0) + +#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) +#define pmd_free(x) do { } while (0) + +#define pgd_populate(mm, pgd, pud) BUG() +#define pgd_populate_kernel(mm, pgd, pud) BUG() + +#define pud_populate(mm, pud, pmd) BUG() +#define pud_populate_kernel(mm, pud, pmd) BUG() + +#else /* __s390x__ */ + +static inline unsigned long pgd_entry_type(struct mm_struct *mm) { - pmd_t *shadow_pmd = get_shadow_pmd(pmd); + return _REGION3_ENTRY_EMPTY; +} + +#define pud_alloc_one(mm,address) ({ BUG(); ((pud_t *)2); }) +#define pud_free(x) do { } while (0) - if (shadow_pmd) - free_pages((unsigned long) shadow_pmd, PMD_ALLOC_ORDER); - free_pages((unsigned long) pmd, PMD_ALLOC_ORDER); +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr) +{ + unsigned long *crst = crst_table_alloc(mm, s390_noexec); + if (crst) + crst_table_init(crst, _SEGMENT_ENTRY_EMPTY); + return (pmd_t *) crst; } +#define pmd_free(pmd) crst_table_free((unsigned long *) pmd) -#define __pmd_free_tlb(tlb,pmd) \ - do { \ - tlb_flush_mmu(tlb, 0, 0); \ - pmd_free(pmd); \ - } while (0) +#define pgd_populate(mm, pgd, pud) BUG() +#define pgd_populate_kernel(mm, pgd, pud) BUG() -static inline void -pgd_populate_kernel(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) +static inline void pud_populate_kernel(struct mm_struct *mm, + pud_t *pud, pmd_t *pmd) { - pgd_val(*pgd) = _PGD_ENTRY | __pa(pmd); + pud_val(*pud) = _REGION3_ENTRY | __pa(pmd); } -static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { - pgd_t *shadow_pgd = get_shadow_pgd(pgd); - pmd_t *shadow_pmd = get_shadow_pmd(pmd); + pud_t *shadow_pud = get_shadow_table(pud); + pmd_t *shadow_pmd = get_shadow_table(pmd); - if (shadow_pgd && shadow_pmd) - pgd_populate_kernel(mm, shadow_pgd, shadow_pmd); - pgd_populate_kernel(mm, pgd, pmd); + if (shadow_pud && shadow_pmd) + pud_populate_kernel(mm, shadow_pud, shadow_pmd); + pud_populate_kernel(mm, pud, pmd); } #endif /* __s390x__ */ +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + unsigned long *crst = crst_table_alloc(mm, s390_noexec); + if (crst) + crst_table_init(crst, pgd_entry_type(mm)); + return (pgd_t *) crst; +} +#define pgd_free(pgd) crst_table_free((unsigned long *) pgd) + static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) { #ifndef __s390x__ - pmd_val(pmd[0]) = _PAGE_TABLE + __pa(pte); - pmd_val(pmd[1]) = _PAGE_TABLE + __pa(pte+256); - pmd_val(pmd[2]) = _PAGE_TABLE + __pa(pte+512); - pmd_val(pmd[3]) = _PAGE_TABLE + __pa(pte+768); + pmd_val(pmd[0]) = _SEGMENT_ENTRY + __pa(pte); + pmd_val(pmd[1]) = _SEGMENT_ENTRY + __pa(pte+256); + pmd_val(pmd[2]) = _SEGMENT_ENTRY + __pa(pte+512); + pmd_val(pmd[3]) = _SEGMENT_ENTRY + __pa(pte+768); #else /* __s390x__ */ - pmd_val(*pmd) = _PMD_ENTRY + __pa(pte); - pmd_val1(*pmd) = _PMD_ENTRY + __pa(pte+256); + pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte); + pmd_val1(*pmd) = _SEGMENT_ENTRY + __pa(pte+256); #endif /* __s390x__ */ } @@ -160,7 +135,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) { pte_t *pte = (pte_t *)page_to_phys(page); - pmd_t *shadow_pmd = get_shadow_pmd(pmd); + pmd_t *shadow_pmd = get_shadow_table(pmd); pte_t *shadow_pte = get_shadow_pte(pte); pmd_populate_kernel(mm, pmd, pte); @@ -171,67 +146,14 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) /* * page table entry allocation/free routines. */ -static inline pte_t * -pte_alloc_one_kernel(struct mm_struct *mm, unsigned long vmaddr) -{ - pte_t *pte = (pte_t *) __get_free_page(GFP_KERNEL|__GFP_REPEAT); - int i; - - if (!pte) - return NULL; - if (s390_noexec) { - pte_t *shadow_pte = (pte_t *) - __get_free_page(GFP_KERNEL|__GFP_REPEAT); - struct page *page = virt_to_page(pte); - - if (!shadow_pte) { - free_page((unsigned long) pte); - return NULL; - } - page->lru.next = (void *) shadow_pte; - } - for (i=0; i < PTRS_PER_PTE; i++) { - pte_clear(mm, vmaddr, pte + i); - vmaddr += PAGE_SIZE; - } - return pte; -} - -static inline struct page * -pte_alloc_one(struct mm_struct *mm, unsigned long vmaddr) -{ - pte_t *pte = pte_alloc_one_kernel(mm, vmaddr); - if (pte) - return virt_to_page(pte); - return NULL; -} - -static inline void pte_free_kernel(pte_t *pte) -{ - pte_t *shadow_pte = get_shadow_pte(pte); - - if (shadow_pte) - free_page((unsigned long) shadow_pte); - free_page((unsigned long) pte); -} - -static inline void pte_free(struct page *pte) -{ - struct page *shadow_page = get_shadow_page(pte); - - if (shadow_page) - __free_page(shadow_page); - __free_page(pte); -} - -#define __pte_free_tlb(tlb, pte) \ -({ \ - struct mmu_gather *__tlb = (tlb); \ - struct page *__pte = (pte); \ - struct page *shadow_page = get_shadow_page(__pte); \ - if (shadow_page) \ - tlb_remove_page(__tlb, shadow_page); \ - tlb_remove_page(__tlb, __pte); \ -}) +#define pte_alloc_one_kernel(mm, vmaddr) \ + ((pte_t *) page_table_alloc(s390_noexec)) +#define pte_alloc_one(mm, vmaddr) \ + virt_to_page(page_table_alloc(s390_noexec)) + +#define pte_free_kernel(pte) \ + page_table_free((unsigned long *) pte) +#define pte_free(pte) \ + page_table_free((unsigned long *) page_to_phys((struct page *) pte)) #endif /* _S390_PGALLOC_H */ diff --git a/include/asm-s390/pgtable.h b/include/asm-s390/pgtable.h index 3208dc6c412..f2cc25b74ad 100644 --- a/include/asm-s390/pgtable.h +++ b/include/asm-s390/pgtable.h @@ -13,8 +13,6 @@ #ifndef _ASM_S390_PGTABLE_H #define _ASM_S390_PGTABLE_H -#include <asm-generic/4level-fixup.h> - /* * The Linux memory management assumes a three-level page table setup. For * s390 31 bit we "fold" the mid level into the top-level page table, so @@ -35,9 +33,6 @@ #include <asm/bug.h> #include <asm/processor.h> -struct vm_area_struct; /* forward declaration (include/linux/mm.h) */ -struct mm_struct; - extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096))); extern void paging_init(void); extern void vmem_map_init(void); @@ -63,14 +58,18 @@ extern char empty_zero_page[PAGE_SIZE]; */ #ifndef __s390x__ # define PMD_SHIFT 22 +# define PUD_SHIFT 22 # define PGDIR_SHIFT 22 #else /* __s390x__ */ # define PMD_SHIFT 21 +# define PUD_SHIFT 31 # define PGDIR_SHIFT 31 #endif /* __s390x__ */ #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) @@ -83,10 +82,12 @@ extern char empty_zero_page[PAGE_SIZE]; #ifndef __s390x__ # define PTRS_PER_PTE 1024 # define PTRS_PER_PMD 1 +# define PTRS_PER_PUD 1 # define PTRS_PER_PGD 512 #else /* __s390x__ */ # define PTRS_PER_PTE 512 # define PTRS_PER_PMD 1024 +# define PTRS_PER_PUD 1 # define PTRS_PER_PGD 2048 #endif /* __s390x__ */ @@ -96,6 +97,8 @@ extern char empty_zero_page[PAGE_SIZE]; printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e)) #define pmd_ERROR(e) \ printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e)) +#define pud_ERROR(e) \ + printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e)) #define pgd_ERROR(e) \ printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e)) @@ -107,11 +110,18 @@ extern char empty_zero_page[PAGE_SIZE]; * any out-of-bounds memory accesses will hopefully be caught. * The vmalloc() routines leaves a hole of 4kB between each vmalloced * area for the same reason. ;) + * vmalloc area starts at 4GB to prevent syscall table entry exchanging + * from modules. */ extern unsigned long vmalloc_end; -#define VMALLOC_OFFSET (8*1024*1024) -#define VMALLOC_START (((unsigned long) high_memory + VMALLOC_OFFSET) \ - & ~(VMALLOC_OFFSET-1)) + +#ifdef CONFIG_64BIT +#define VMALLOC_ADDR (max(0x100000000UL, (unsigned long) high_memory)) +#else +#define VMALLOC_ADDR ((unsigned long) high_memory) +#endif +#define VMALLOC_OFFSET (8*1024*1024) +#define VMALLOC_START ((VMALLOC_ADDR + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)) #define VMALLOC_END vmalloc_end /* @@ -188,7 +198,7 @@ extern unsigned long vmalloc_end; * I Segment-Invalid Bit: Segment is not available for address-translation * TT Type 01 * TF - * TL Table lenght + * TL Table length * * The 64 bit regiontable origin of S390 has following format: * | region table origon | DTTL @@ -214,6 +224,8 @@ extern unsigned long vmalloc_end; /* Hardware bits in the page table entry */ #define _PAGE_RO 0x200 /* HW read-only bit */ #define _PAGE_INVALID 0x400 /* HW invalid bit */ + +/* Software bits in the page table entry */ #define _PAGE_SWT 0x001 /* SW pte type bit t */ #define _PAGE_SWX 0x002 /* SW pte type bit x */ @@ -257,60 +269,75 @@ extern unsigned long vmalloc_end; #ifndef __s390x__ -/* Bits in the segment table entry */ -#define _PAGE_TABLE_LEN 0xf /* only full page-tables */ -#define _PAGE_TABLE_COM 0x10 /* common page-table */ -#define _PAGE_TABLE_INV 0x20 /* invalid page-table */ -#define _SEG_PRESENT 0x001 /* Software (overlap with PTL) */ +/* Bits in the segment table address-space-control-element */ +#define _ASCE_SPACE_SWITCH 0x80000000UL /* space switch event */ +#define _ASCE_ORIGIN_MASK 0x7ffff000UL /* segment table origin */ +#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */ +#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */ +#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */ -/* Bits int the storage key */ -#define _PAGE_CHANGED 0x02 /* HW changed bit */ -#define _PAGE_REFERENCED 0x04 /* HW referenced bit */ - -#define _USER_SEG_TABLE_LEN 0x7f /* user-segment-table up to 2 GB */ -#define _KERNEL_SEG_TABLE_LEN 0x7f /* kernel-segment-table up to 2 GB */ - -/* - * User and Kernel pagetables are identical - */ -#define _PAGE_TABLE _PAGE_TABLE_LEN -#define _KERNPG_TABLE _PAGE_TABLE_LEN - -/* - * The Kernel segment-tables includes the User segment-table - */ - -#define _SEGMENT_TABLE (_USER_SEG_TABLE_LEN|0x80000000|0x100) -#define _KERNSEG_TABLE _KERNEL_SEG_TABLE_LEN +/* Bits in the segment table entry */ +#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */ +#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ +#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */ +#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */ -#define USER_STD_MASK 0x00000080UL +#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL) +#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) #else /* __s390x__ */ +/* Bits in the segment/region table address-space-control-element */ +#define _ASCE_ORIGIN ~0xfffUL/* segment table origin */ +#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */ +#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */ +#define _ASCE_SPACE_SWITCH 0x40 /* space switch event */ +#define _ASCE_REAL_SPACE 0x20 /* real space control */ +#define _ASCE_TYPE_MASK 0x0c /* asce table type mask */ +#define _ASCE_TYPE_REGION1 0x0c /* region first table type */ +#define _ASCE_TYPE_REGION2 0x08 /* region second table type */ +#define _ASCE_TYPE_REGION3 0x04 /* region third table type */ +#define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */ +#define _ASCE_TABLE_LENGTH 0x03 /* region table length */ + +/* Bits in the region table entry */ +#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */ +#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */ +#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ +#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */ +#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */ +#define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */ +#define _REGION_ENTRY_LENGTH 0x03 /* region third length */ + +#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH) +#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV) +#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH) +#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV) +#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH) +#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV) + /* Bits in the segment table entry */ -#define _PMD_ENTRY_INV 0x20 /* invalid segment table entry */ -#define _PMD_ENTRY 0x00 +#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */ +#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */ +#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ + +#define _SEGMENT_ENTRY (0) +#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) -/* Bits in the region third table entry */ -#define _PGD_ENTRY_INV 0x20 /* invalid region table entry */ -#define _PGD_ENTRY 0x07 +#endif /* __s390x__ */ /* - * User and kernel page directory + * A user page table pointer has the space-switch-event bit, the + * private-space-control bit and the storage-alteration-event-control + * bit set. A kernel page table pointer doesn't need them. */ -#define _REGION_THIRD 0x4 -#define _REGION_THIRD_LEN 0x3 -#define _REGION_TABLE (_REGION_THIRD|_REGION_THIRD_LEN|0x40|0x100) -#define _KERN_REGION_TABLE (_REGION_THIRD|_REGION_THIRD_LEN) - -#define USER_STD_MASK 0x0000000000000080UL +#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \ + _ASCE_ALT_EVENT) -/* Bits in the storage key */ +/* Bits int the storage key */ #define _PAGE_CHANGED 0x02 /* HW changed bit */ #define _PAGE_REFERENCED 0x04 /* HW referenced bit */ -#endif /* __s390x__ */ - /* * Page protection definitions. */ @@ -351,65 +378,38 @@ extern unsigned long vmalloc_end; #define __S111 PAGE_EX_RW #ifndef __s390x__ -# define PMD_SHADOW_SHIFT 1 -# define PGD_SHADOW_SHIFT 1 +# define PxD_SHADOW_SHIFT 1 #else /* __s390x__ */ -# define PMD_SHADOW_SHIFT 2 -# define PGD_SHADOW_SHIFT 2 +# define PxD_SHADOW_SHIFT 2 #endif /* __s390x__ */ static inline struct page *get_shadow_page(struct page *page) { - if (s390_noexec && !list_empty(&page->lru)) - return virt_to_page(page->lru.next); - return NULL; -} - -static inline pte_t *get_shadow_pte(pte_t *ptep) -{ - unsigned long pteptr = (unsigned long) (ptep); - - if (s390_noexec) { - unsigned long offset = pteptr & (PAGE_SIZE - 1); - void *addr = (void *) (pteptr ^ offset); - struct page *page = virt_to_page(addr); - if (!list_empty(&page->lru)) - return (pte_t *) ((unsigned long) page->lru.next | - offset); - } + if (s390_noexec && page->index) + return virt_to_page((void *)(addr_t) page->index); return NULL; } -static inline pmd_t *get_shadow_pmd(pmd_t *pmdp) +static inline void *get_shadow_pte(void *table) { - unsigned long pmdptr = (unsigned long) (pmdp); + unsigned long addr, offset; + struct page *page; - if (s390_noexec) { - unsigned long offset = pmdptr & - ((PAGE_SIZE << PMD_SHADOW_SHIFT) - 1); - void *addr = (void *) (pmdptr ^ offset); - struct page *page = virt_to_page(addr); - if (!list_empty(&page->lru)) - return (pmd_t *) ((unsigned long) page->lru.next | - offset); - } - return NULL; + addr = (unsigned long) table; + offset = addr & (PAGE_SIZE - 1); + page = virt_to_page((void *)(addr ^ offset)); + return (void *)(addr_t)(page->index ? (page->index | offset) : 0UL); } -static inline pgd_t *get_shadow_pgd(pgd_t *pgdp) +static inline void *get_shadow_table(void *table) { - unsigned long pgdptr = (unsigned long) (pgdp); + unsigned long addr, offset; + struct page *page; - if (s390_noexec) { - unsigned long offset = pgdptr & - ((PAGE_SIZE << PGD_SHADOW_SHIFT) - 1); - void *addr = (void *) (pgdptr ^ offset); - struct page *page = virt_to_page(addr); - if (!list_empty(&page->lru)) - return (pgd_t *) ((unsigned long) page->lru.next | - offset); - } - return NULL; + addr = (unsigned long) table; + offset = addr & ((PAGE_SIZE << PxD_SHADOW_SHIFT) - 1); + page = virt_to_page((void *)(addr ^ offset)); + return (void *)(addr_t)(page->index ? (page->index | offset) : 0UL); } /* @@ -417,7 +417,8 @@ static inline pgd_t *get_shadow_pgd(pgd_t *pgdp) * within a page table are directly modified. Thus, the following * hook is made available. */ -static inline void set_pte(pte_t *pteptr, pte_t pteval) +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *pteptr, pte_t pteval) { pte_t *shadow_pte = get_shadow_pte(pteptr); @@ -430,7 +431,6 @@ static inline void set_pte(pte_t *pteptr, pte_t pteval) pte_val(*shadow_pte) = _PAGE_TYPE_EMPTY; } } -#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) /* * pgd/pmd/pte query functions @@ -441,47 +441,50 @@ static inline int pgd_present(pgd_t pgd) { return 1; } static inline int pgd_none(pgd_t pgd) { return 0; } static inline int pgd_bad(pgd_t pgd) { return 0; } -static inline int pmd_present(pmd_t pmd) { return pmd_val(pmd) & _SEG_PRESENT; } -static inline int pmd_none(pmd_t pmd) { return pmd_val(pmd) & _PAGE_TABLE_INV; } -static inline int pmd_bad(pmd_t pmd) -{ - return (pmd_val(pmd) & (~PAGE_MASK & ~_PAGE_TABLE_INV)) != _PAGE_TABLE; -} +static inline int pud_present(pud_t pud) { return 1; } +static inline int pud_none(pud_t pud) { return 0; } +static inline int pud_bad(pud_t pud) { return 0; } #else /* __s390x__ */ -static inline int pgd_present(pgd_t pgd) +static inline int pgd_present(pgd_t pgd) { return 1; } +static inline int pgd_none(pgd_t pgd) { return 0; } +static inline int pgd_bad(pgd_t pgd) { return 0; } + +static inline int pud_present(pud_t pud) { - return (pgd_val(pgd) & ~PAGE_MASK) == _PGD_ENTRY; + return pud_val(pud) & _REGION_ENTRY_ORIGIN; } -static inline int pgd_none(pgd_t pgd) +static inline int pud_none(pud_t pud) { - return pgd_val(pgd) & _PGD_ENTRY_INV; + return pud_val(pud) & _REGION_ENTRY_INV; } -static inline int pgd_bad(pgd_t pgd) +static inline int pud_bad(pud_t pud) { - return (pgd_val(pgd) & (~PAGE_MASK & ~_PGD_ENTRY_INV)) != _PGD_ENTRY; + unsigned long mask = ~_REGION_ENTRY_ORIGIN & ~_REGION_ENTRY_INV; + return (pud_val(pud) & mask) != _REGION3_ENTRY; } +#endif /* __s390x__ */ + static inline int pmd_present(pmd_t pmd) { - return (pmd_val(pmd) & ~PAGE_MASK) == _PMD_ENTRY; + return pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN; } static inline int pmd_none(pmd_t pmd) { - return pmd_val(pmd) & _PMD_ENTRY_INV; + return pmd_val(pmd) & _SEGMENT_ENTRY_INV; } static inline int pmd_bad(pmd_t pmd) { - return (pmd_val(pmd) & (~PAGE_MASK & ~_PMD_ENTRY_INV)) != _PMD_ENTRY; + unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV; + return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY; } -#endif /* __s390x__ */ - static inline int pte_none(pte_t pte) { return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT); @@ -501,7 +504,8 @@ static inline int pte_file(pte_t pte) return (pte_val(pte) & mask) == _PAGE_TYPE_FILE; } -#define pte_same(a,b) (pte_val(a) == pte_val(b)) +#define __HAVE_ARCH_PTE_SAME +#define pte_same(a,b) (pte_val(a) == pte_val(b)) /* * query functions pte_write/pte_dirty/pte_young only work if @@ -536,58 +540,52 @@ static inline int pte_young(pte_t pte) #ifndef __s390x__ -static inline void pgd_clear(pgd_t * pgdp) { } +#define pgd_clear(pgd) do { } while (0) +#define pud_clear(pud) do { } while (0) static inline void pmd_clear_kernel(pmd_t * pmdp) { - pmd_val(pmdp[0]) = _PAGE_TABLE_INV; - pmd_val(pmdp[1]) = _PAGE_TABLE_INV; - pmd_val(pmdp[2]) = _PAGE_TABLE_INV; - pmd_val(pmdp[3]) = _PAGE_TABLE_INV; -} - -static inline void pmd_clear(pmd_t * pmdp) -{ - pmd_t *shadow_pmd = get_shadow_pmd(pmdp); - - pmd_clear_kernel(pmdp); - if (shadow_pmd) - pmd_clear_kernel(shadow_pmd); + pmd_val(pmdp[0]) = _SEGMENT_ENTRY_EMPTY; + pmd_val(pmdp[1]) = _SEGMENT_ENTRY_EMPTY; + pmd_val(pmdp[2]) = _SEGMENT_ENTRY_EMPTY; + pmd_val(pmdp[3]) = _SEGMENT_ENTRY_EMPTY; } #else /* __s390x__ */ -static inline void pgd_clear_kernel(pgd_t * pgdp) +#define pgd_clear(pgd) do { } while (0) + +static inline void pud_clear_kernel(pud_t *pud) { - pgd_val(*pgdp) = _PGD_ENTRY_INV | _PGD_ENTRY; + pud_val(*pud) = _REGION3_ENTRY_EMPTY; } -static inline void pgd_clear(pgd_t * pgdp) +static inline void pud_clear(pud_t * pud) { - pgd_t *shadow_pgd = get_shadow_pgd(pgdp); + pud_t *shadow = get_shadow_table(pud); - pgd_clear_kernel(pgdp); - if (shadow_pgd) - pgd_clear_kernel(shadow_pgd); + pud_clear_kernel(pud); + if (shadow) + pud_clear_kernel(shadow); } static inline void pmd_clear_kernel(pmd_t * pmdp) { - pmd_val(*pmdp) = _PMD_ENTRY_INV | _PMD_ENTRY; - pmd_val1(*pmdp) = _PMD_ENTRY_INV | _PMD_ENTRY; + pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; + pmd_val1(*pmdp) = _SEGMENT_ENTRY_EMPTY; } +#endif /* __s390x__ */ + static inline void pmd_clear(pmd_t * pmdp) { - pmd_t *shadow_pmd = get_shadow_pmd(pmdp); + pmd_t *shadow_pmd = get_shadow_table(pmdp); pmd_clear_kernel(pmdp); if (shadow_pmd) pmd_clear_kernel(shadow_pmd); } -#endif /* __s390x__ */ - static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t *shadow_pte = get_shadow_pte(ptep); @@ -656,24 +654,19 @@ static inline pte_t pte_mkyoung(pte_t pte) return pte; } -static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { return 0; } -static inline int -ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +static inline int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { /* No need to flush TLB; bits are in storage key */ - return ptep_test_and_clear_young(vma, address, ptep); -} - -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) -{ - pte_t pte = *ptep; - pte_clear(mm, addr, ptep); - return pte; + return 0; } static inline void __ptep_ipte(unsigned long address, pte_t *ptep) @@ -702,6 +695,32 @@ static inline void ptep_invalidate(unsigned long address, pte_t *ptep) __ptep_ipte(address, ptep); } +/* + * This is hard to understand. ptep_get_and_clear and ptep_clear_flush + * both clear the TLB for the unmapped pte. The reason is that + * ptep_get_and_clear is used in common code (e.g. change_pte_range) + * to modify an active pte. The sequence is + * 1) ptep_get_and_clear + * 2) set_pte_at + * 3) flush_tlb_range + * On s390 the tlb needs to get flushed with the modification of the pte + * if the pte is active. The only way how this can be implemented is to + * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range + * is a nop. + */ +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +#define ptep_get_and_clear(__mm, __address, __ptep) \ +({ \ + pte_t __pte = *(__ptep); \ + if (atomic_read(&(__mm)->mm_users) > 1 || \ + (__mm) != current->active_mm) \ + ptep_invalidate(__address, __ptep); \ + else \ + pte_clear((__mm), (__address), (__ptep)); \ + __pte; \ +}) + +#define __HAVE_ARCH_PTEP_CLEAR_FLUSH static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { @@ -710,12 +729,40 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, return pte; } -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +/* + * The batched pte unmap code uses ptep_get_and_clear_full to clear the + * ptes. Here an optimization is possible. tlb_gather_mmu flushes all + * tlbs of an mm if it can guarantee that the ptes of the mm_struct + * cannot be accessed while the batched unmap is running. In this case + * full==1 and a simple pte_clear is enough. See tlb.h. + */ +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL +static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, int full) { - pte_t old_pte = *ptep; - set_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); + pte_t pte = *ptep; + + if (full) + pte_clear(mm, addr, ptep); + else + ptep_invalidate(addr, ptep); + return pte; } +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +#define ptep_set_wrprotect(__mm, __addr, __ptep) \ +({ \ + pte_t __pte = *(__ptep); \ + if (pte_write(__pte)) { \ + if (atomic_read(&(__mm)->mm_users) > 1 || \ + (__mm) != current->active_mm) \ + ptep_invalidate(__addr, __ptep); \ + set_pte_at(__mm, __addr, __ptep, pte_wrprotect(__pte)); \ + } \ +}) + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ ({ \ int __changed = !pte_same(*(__ptep), __entry); \ @@ -733,11 +780,13 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, * should therefore only be called if it is not mapped in any * address space. */ +#define __HAVE_ARCH_PAGE_TEST_DIRTY static inline int page_test_dirty(struct page *page) { return (page_get_storage_key(page_to_phys(page)) & _PAGE_CHANGED) != 0; } +#define __HAVE_ARCH_PAGE_CLEAR_DIRTY static inline void page_clear_dirty(struct page *page) { page_set_storage_key(page_to_phys(page), PAGE_DEFAULT_KEY); @@ -746,6 +795,7 @@ static inline void page_clear_dirty(struct page *page) /* * Test and clear referenced bit in storage key. */ +#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG static inline int page_test_and_clear_young(struct page *page) { unsigned long physpage = page_to_phys(page); @@ -777,63 +827,48 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) return mk_pte_phys(physpage, pgprot); } -static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) -{ - unsigned long physpage = __pa((pfn) << PAGE_SHIFT); - - return mk_pte_phys(physpage, pgprot); -} - -#ifdef __s390x__ - -static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot) -{ - unsigned long physpage = __pa((pfn) << PAGE_SHIFT); - - return __pmd(physpage + pgprot_val(pgprot)); -} - -#endif /* __s390x__ */ - -#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) -#define pte_page(x) pfn_to_page(pte_pfn(x)) +#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) +#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) +#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) +#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) -#define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK) +#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) +#define pgd_offset_k(address) pgd_offset(&init_mm, address) -#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) +#ifndef __s390x__ -#define pgd_page_vaddr(pgd) (pgd_val(pgd) & PAGE_MASK) +#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) +#define pud_deref(pmd) ({ BUG(); 0UL; }) +#define pgd_deref(pmd) ({ BUG(); 0UL; }) -#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT) +#define pud_offset(pgd, address) ((pud_t *) pgd) +#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address)) -/* to find an entry in a page-table-directory */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address)) +#else /* __s390x__ */ -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) +#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) +#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN) +#define pgd_deref(pgd) ({ BUG(); 0UL; }) -#ifndef __s390x__ +#define pud_offset(pgd, address) ((pud_t *) pgd) -/* Find an entry in the second-level page table.. */ -static inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address) +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) { - return (pmd_t *) dir; + pmd_t *pmd = (pmd_t *) pud_deref(*pud); + return pmd + pmd_index(address); } -#else /* __s390x__ */ +#endif /* __s390x__ */ -/* Find an entry in the second-level page table.. */ -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -#define pmd_offset(dir,addr) \ - ((pmd_t *) pgd_page_vaddr(*(dir)) + pmd_index(addr)) +#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) +#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) +#define pte_page(x) pfn_to_page(pte_pfn(x)) -#endif /* __s390x__ */ +#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) -/* Find an entry in the third-level page table.. */ -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) -#define pte_offset_kernel(pmd, address) \ - ((pte_t *) pmd_page_vaddr(*(pmd)) + pte_index(address)) +/* Find an entry in the lowest level page table.. */ +#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr)) +#define pte_offset_kernel(pmd, address) pte_offset(pmd,address) #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) #define pte_offset_map_nested(pmd, address) pte_offset_kernel(pmd, address) #define pte_unmap(pte) do { } while (0) @@ -923,17 +958,6 @@ extern int remove_shared_memory(unsigned long start, unsigned long size); #define __HAVE_ARCH_MEMMAP_INIT extern void memmap_init(unsigned long, int, unsigned long, unsigned long); -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -#define __HAVE_ARCH_PTEP_GET_AND_CLEAR -#define __HAVE_ARCH_PTEP_CLEAR_FLUSH -#define __HAVE_ARCH_PTEP_SET_WRPROTECT -#define __HAVE_ARCH_PTE_SAME -#define __HAVE_ARCH_PAGE_TEST_DIRTY -#define __HAVE_ARCH_PAGE_CLEAR_DIRTY -#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG #include <asm-generic/pgtable.h> #endif /* _S390_PAGE_H */ - diff --git a/include/asm-s390/processor.h b/include/asm-s390/processor.h index 3b972d4c6b2..21d40a19355 100644 --- a/include/asm-s390/processor.h +++ b/include/asm-s390/processor.h @@ -93,7 +93,6 @@ struct thread_struct { s390_fp_regs fp_regs; unsigned int acrs[NUM_ACRS]; unsigned long ksp; /* kernel stack pointer */ - unsigned long user_seg; /* HSTD */ mm_segment_t mm_segment; unsigned long prot_addr; /* address of protection-excep. */ unsigned int error_code; /* error-code of last prog-excep. */ @@ -128,22 +127,9 @@ struct stack_frame { #define ARCH_MIN_TASKALIGN 8 -#ifndef __s390x__ -# define __SWAPPER_PG_DIR __pa(&swapper_pg_dir[0]) + _SEGMENT_TABLE -#else /* __s390x__ */ -# define __SWAPPER_PG_DIR __pa(&swapper_pg_dir[0]) + _REGION_TABLE -#endif /* __s390x__ */ - -#define INIT_THREAD {{0,{{0},{0},{0},{0},{0},{0},{0},{0},{0},{0}, \ - {0},{0},{0},{0},{0},{0}}}, \ - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \ - sizeof(init_stack) + (unsigned long) &init_stack, \ - __SWAPPER_PG_DIR, \ - {0}, \ - 0,0,0, \ - (per_struct) {{{{0,}}},0,0,0,0,{{0,}}}, \ - 0, 0 \ -} +#define INIT_THREAD { \ + .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ +} /* * Do necessary setup to start up a new thread. diff --git a/include/asm-s390/s390_ext.h b/include/asm-s390/s390_ext.h index 1e72362cad7..2afc060266a 100644 --- a/include/asm-s390/s390_ext.h +++ b/include/asm-s390/s390_ext.h @@ -5,7 +5,7 @@ * include/asm-s390/s390_ext.h * * S390 version - * Copyright (C) 1999,2000 IBM Deutschland Entwicklung GmbH, IBM Corporation + * Copyright IBM Corp. 1999,2007 * Author(s): Holger Smolinski (Holger.Smolinski@de.ibm.com), * Martin Schwidefsky (schwidefsky@de.ibm.com) */ @@ -14,15 +14,11 @@ typedef void (*ext_int_handler_t)(__u16 code); -/* - * Warning: if you change ext_int_info_t you have to change the - * external interrupt handler in entry.S too. - */ typedef struct ext_int_info_t { struct ext_int_info_t *next; ext_int_handler_t handler; __u16 code; -} __attribute__ ((packed)) ext_int_info_t; +} ext_int_info_t; extern ext_int_info_t *ext_int_hash[]; diff --git a/include/asm-s390/scatterlist.h b/include/asm-s390/scatterlist.h index a43b3afc5e2..29ec8e28c8d 100644 --- a/include/asm-s390/scatterlist.h +++ b/include/asm-s390/scatterlist.h @@ -2,7 +2,10 @@ #define _ASMS390_SCATTERLIST_H struct scatterlist { - struct page *page; +#ifdef CONFIG_DEBUG_SG + unsigned long sg_magic; +#endif + unsigned long page_link; unsigned int offset; unsigned int length; }; diff --git a/include/asm-s390/semaphore.h b/include/asm-s390/semaphore.h index dbce058aefa..0e7001ad839 100644 --- a/include/asm-s390/semaphore.h +++ b/include/asm-s390/semaphore.h @@ -33,7 +33,6 @@ struct semaphore { struct semaphore name = __SEMAPHORE_INITIALIZER(name,count) #define DECLARE_MUTEX(name) __DECLARE_SEMAPHORE_GENERIC(name,1) -#define DECLARE_MUTEX_LOCKED(name) __DECLARE_SEMAPHORE_GENERIC(name,0) static inline void sema_init (struct semaphore *sem, int val) { diff --git a/include/asm-s390/system.h b/include/asm-s390/system.h index 64a3cd05cae..d866d338555 100644 --- a/include/asm-s390/system.h +++ b/include/asm-s390/system.h @@ -130,6 +130,8 @@ extern void pfault_fini(void); __ret; \ }) +extern void __xchg_called_with_bad_pointer(void); + static inline unsigned long __xchg(unsigned long x, void * ptr, int size) { unsigned long addr, old; @@ -150,8 +152,7 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size) : "=&d" (old), "=m" (*(int *) addr) : "d" (x << shift), "d" (~(255 << shift)), "a" (addr), "m" (*(int *) addr) : "memory", "cc", "0"); - x = old >> shift; - break; + return old >> shift; case 2: addr = (unsigned long) ptr; shift = (2 ^ (addr & 2)) << 3; @@ -166,8 +167,7 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size) : "=&d" (old), "=m" (*(int *) addr) : "d" (x << shift), "d" (~(65535 << shift)), "a" (addr), "m" (*(int *) addr) : "memory", "cc", "0"); - x = old >> shift; - break; + return old >> shift; case 4: asm volatile( " l %0,0(%3)\n" @@ -176,8 +176,7 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size) : "=&d" (old), "=m" (*(int *) ptr) : "d" (x), "a" (ptr), "m" (*(int *) ptr) : "memory", "cc"); - x = old; - break; + return old; #ifdef __s390x__ case 8: asm volatile( @@ -187,11 +186,11 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size) : "=&d" (old), "=m" (*(long *) ptr) : "d" (x), "a" (ptr), "m" (*(long *) ptr) : "memory", "cc"); - x = old; - break; + return old; #endif /* __s390x__ */ - } - return x; + } + __xchg_called_with_bad_pointer(); + return x; } /* @@ -206,6 +205,8 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size) ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +extern void __cmpxchg_called_with_bad_pointer(void); + static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { @@ -270,7 +271,8 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) return prev; #endif /* __s390x__ */ } - return old; + __cmpxchg_called_with_bad_pointer(); + return old; } /* diff --git a/include/asm-s390/tlb.h b/include/asm-s390/tlb.h index 51bd957b85b..618693cfc10 100644 --- a/include/asm-s390/tlb.h +++ b/include/asm-s390/tlb.h @@ -2,19 +2,130 @@ #define _S390_TLB_H /* - * s390 doesn't need any special per-pte or - * per-vma handling.. + * TLB flushing on s390 is complicated. The following requirement + * from the principles of operation is the most arduous: + * + * "A valid table entry must not be changed while it is attached + * to any CPU and may be used for translation by that CPU except to + * (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY, + * or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page + * table entry, or (3) make a change by means of a COMPARE AND SWAP + * AND PURGE instruction that purges the TLB." + * + * The modification of a pte of an active mm struct therefore is + * a two step process: i) invalidate the pte, ii) store the new pte. + * This is true for the page protection bit as well. + * The only possible optimization is to flush at the beginning of + * a tlb_gather_mmu cycle if the mm_struct is currently not in use. + * + * Pages used for the page tables is a different story. FIXME: more */ -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) -#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) + +#include <linux/mm.h> +#include <linux/swap.h> +#include <asm/processor.h> +#include <asm/pgalloc.h> +#include <asm/smp.h> +#include <asm/tlbflush.h> + +#ifndef CONFIG_SMP +#define TLB_NR_PTRS 1 +#else +#define TLB_NR_PTRS 508 +#endif + +struct mmu_gather { + struct mm_struct *mm; + unsigned int fullmm; + unsigned int nr_ptes; + unsigned int nr_pmds; + void *array[TLB_NR_PTRS]; +}; + +DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); + +static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, + unsigned int full_mm_flush) +{ + struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); + + tlb->mm = mm; + tlb->fullmm = full_mm_flush || (num_online_cpus() == 1) || + (atomic_read(&mm->mm_users) <= 1 && mm == current->active_mm); + tlb->nr_ptes = 0; + tlb->nr_pmds = TLB_NR_PTRS; + if (tlb->fullmm) + __tlb_flush_mm(mm); + return tlb; +} + +static inline void tlb_flush_mmu(struct mmu_gather *tlb, + unsigned long start, unsigned long end) +{ + if (!tlb->fullmm && (tlb->nr_ptes > 0 || tlb->nr_pmds < TLB_NR_PTRS)) + __tlb_flush_mm(tlb->mm); + while (tlb->nr_ptes > 0) + pte_free(tlb->array[--tlb->nr_ptes]); + while (tlb->nr_pmds < TLB_NR_PTRS) + pmd_free((pmd_t *) tlb->array[tlb->nr_pmds++]); +} + +static inline void tlb_finish_mmu(struct mmu_gather *tlb, + unsigned long start, unsigned long end) +{ + tlb_flush_mmu(tlb, start, end); + + /* keep the page table cache within bounds */ + check_pgt_cache(); + + put_cpu_var(mmu_gathers); +} /* - * .. because we flush the whole mm when it - * fills up. + * Release the page cache reference for a pte removed by + * tlb_ptep_clear_flush. In both flush modes the tlb fo a page cache page + * has already been freed, so just do free_page_and_swap_cache. */ -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + free_page_and_swap_cache(page); +} -#include <asm-generic/tlb.h> +/* + * pte_free_tlb frees a pte table and clears the CRSTE for the + * page table from the tlb. + */ +static inline void pte_free_tlb(struct mmu_gather *tlb, struct page *page) +{ + if (!tlb->fullmm) { + tlb->array[tlb->nr_ptes++] = page; + if (tlb->nr_ptes >= tlb->nr_pmds) + tlb_flush_mmu(tlb, 0, 0); + } else + pte_free(page); +} +/* + * pmd_free_tlb frees a pmd table and clears the CRSTE for the + * segment table entry from the tlb. + */ +static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +{ +#ifdef __s390x__ + if (!tlb->fullmm) { + tlb->array[--tlb->nr_pmds] = (struct page *) pmd; + if (tlb->nr_ptes >= tlb->nr_pmds) + tlb_flush_mmu(tlb, 0, 0); + } else + pmd_free(pmd); #endif +} + +#define pud_free_tlb(tlb, pud) do { } while (0) + +#define tlb_start_vma(tlb, vma) do { } while (0) +#define tlb_end_vma(tlb, vma) do { } while (0) +#define tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0) +#define tlb_migrate_finish(mm) do { } while (0) + +#endif /* _S390_TLB_H */ diff --git a/include/asm-s390/tlbflush.h b/include/asm-s390/tlbflush.h index 66793f55c8b..a69bd2490d5 100644 --- a/include/asm-s390/tlbflush.h +++ b/include/asm-s390/tlbflush.h @@ -6,69 +6,19 @@ #include <asm/pgalloc.h> /* - * TLB flushing: - * - * - flush_tlb() flushes the current mm struct TLBs - * - flush_tlb_all() flushes all processes TLBs - * - flush_tlb_mm(mm) flushes the specified mm context TLB's - * - flush_tlb_page(vma, vmaddr) flushes one page - * - flush_tlb_range(vma, start, end) flushes a range of pages - * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages - * - flush_tlb_pgtables(mm, start, end) flushes a range of page tables - */ - -/* - * S/390 has three ways of flushing TLBs - * 'ptlb' does a flush of the local processor - * 'csp' flushes the TLBs on all PUs of a SMP - * 'ipte' invalidates a pte in a page table and flushes that out of - * the TLBs of all PUs of a SMP + * Flush all tlb entries on the local cpu. */ - -#define local_flush_tlb() \ -do { asm volatile("ptlb": : :"memory"); } while (0) - -#ifndef CONFIG_SMP - -/* - * We always need to flush, since s390 does not flush tlb - * on each context switch - */ - -static inline void flush_tlb(void) -{ - local_flush_tlb(); -} -static inline void flush_tlb_all(void) -{ - local_flush_tlb(); -} -static inline void flush_tlb_mm(struct mm_struct *mm) +static inline void __tlb_flush_local(void) { - local_flush_tlb(); + asm volatile("ptlb" : : : "memory"); } -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long addr) -{ - local_flush_tlb(); -} -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - local_flush_tlb(); -} - -#define flush_tlb_kernel_range(start, end) \ - local_flush_tlb(); - -#else - -#include <asm/smp.h> - -extern void smp_ptlb_all(void); -static inline void global_flush_tlb(void) +/* + * Flush all tlb entries on all cpus. + */ +static inline void __tlb_flush_global(void) { + extern void smp_ptlb_all(void); register unsigned long reg2 asm("2"); register unsigned long reg3 asm("3"); register unsigned long reg4 asm("4"); @@ -90,72 +40,75 @@ static inline void global_flush_tlb(void) } /* - * We only have to do global flush of tlb if process run since last - * flush on any other pu than current. - * If we have threads (mm->count > 1) we always do a global flush, - * since the process runs on more than one processor at the same time. + * Flush all tlb entries of a page table on all cpus. */ +static inline void __tlb_flush_idte(pgd_t *pgd) +{ + asm volatile( + " .insn rrf,0xb98e0000,0,%0,%1,0" + : : "a" (2048), "a" (__pa(pgd) & PAGE_MASK) : "cc" ); +} -static inline void __flush_tlb_mm(struct mm_struct * mm) +static inline void __tlb_flush_mm(struct mm_struct * mm) { cpumask_t local_cpumask; if (unlikely(cpus_empty(mm->cpu_vm_mask))) return; + /* + * If the machine has IDTE we prefer to do a per mm flush + * on all cpus instead of doing a local flush if the mm + * only ran on the local cpu. + */ if (MACHINE_HAS_IDTE) { - pgd_t *shadow_pgd = get_shadow_pgd(mm->pgd); + pgd_t *shadow_pgd = get_shadow_table(mm->pgd); - if (shadow_pgd) { - asm volatile( - " .insn rrf,0xb98e0000,0,%0,%1,0" - : : "a" (2048), - "a" (__pa(shadow_pgd) & PAGE_MASK) : "cc" ); - } - asm volatile( - " .insn rrf,0xb98e0000,0,%0,%1,0" - : : "a" (2048), "a" (__pa(mm->pgd)&PAGE_MASK) : "cc"); + if (shadow_pgd) + __tlb_flush_idte(shadow_pgd); + __tlb_flush_idte(mm->pgd); return; } preempt_disable(); + /* + * If the process only ran on the local cpu, do a local flush. + */ local_cpumask = cpumask_of_cpu(smp_processor_id()); if (cpus_equal(mm->cpu_vm_mask, local_cpumask)) - local_flush_tlb(); + __tlb_flush_local(); else - global_flush_tlb(); + __tlb_flush_global(); preempt_enable(); } -static inline void flush_tlb(void) -{ - __flush_tlb_mm(current->mm); -} -static inline void flush_tlb_all(void) -{ - global_flush_tlb(); -} -static inline void flush_tlb_mm(struct mm_struct *mm) -{ - __flush_tlb_mm(mm); -} -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long addr) +static inline void __tlb_flush_mm_cond(struct mm_struct * mm) { - __flush_tlb_mm(vma->vm_mm); + if (atomic_read(&mm->mm_users) <= 1 && mm == current->active_mm) + __tlb_flush_mm(mm); } -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - __flush_tlb_mm(vma->vm_mm); -} - -#define flush_tlb_kernel_range(start, end) global_flush_tlb() -#endif +/* + * TLB flushing: + * flush_tlb() - flushes the current mm struct TLBs + * flush_tlb_all() - flushes all processes TLBs + * flush_tlb_mm(mm) - flushes the specified mm context TLB's + * flush_tlb_page(vma, vmaddr) - flushes one page + * flush_tlb_range(vma, start, end) - flushes a range of pages + * flush_tlb_kernel_range(start, end) - flushes a range of kernel pages + */ -static inline void flush_tlb_pgtables(struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - /* S/390 does not keep any page table caches in TLB */ -} +/* + * flush_tlb_mm goes together with ptep_set_wrprotect for the + * copy_page_range operation and flush_tlb_range is related to + * ptep_get_and_clear for change_protection. ptep_set_wrprotect and + * ptep_get_and_clear do not flush the TLBs directly if the mm has + * only one user. At the end of the update the flush_tlb_mm and + * flush_tlb_range functions need to do the flush. + */ +#define flush_tlb() do { } while (0) +#define flush_tlb_all() do { } while (0) +#define flush_tlb_mm(mm) __tlb_flush_mm_cond(mm) +#define flush_tlb_page(vma, addr) do { } while (0) +#define flush_tlb_range(vma, start, end) __tlb_flush_mm_cond(mm) +#define flush_tlb_kernel_range(start, end) __tlb_flush_mm(&init_mm) #endif /* _S390_TLBFLUSH_H */ diff --git a/include/asm-s390/types.h b/include/asm-s390/types.h index fc5d7cf1932..2c5879ae90c 100644 --- a/include/asm-s390/types.h +++ b/include/asm-s390/types.h @@ -28,9 +28,9 @@ typedef __signed__ int __s32; typedef unsigned int __u32; #ifndef __s390x__ -#if defined(__GNUC__) && !defined(__STRICT_ANSI__) -typedef __signed__ long long __s64; -typedef unsigned long long __u64; +#if defined(__GNUC__) +__extension__ typedef __signed__ long long __s64; +__extension__ typedef unsigned long long __u64; #endif #else /* __s390x__ */ typedef __signed__ long __s64; diff --git a/include/asm-s390/zcrypt.h b/include/asm-s390/zcrypt.h index b90e55888a5..a5dada61775 100644 --- a/include/asm-s390/zcrypt.h +++ b/include/asm-s390/zcrypt.h @@ -91,7 +91,7 @@ struct ica_rsa_modexpo_crt { * VUD block * key block */ -struct ica_CPRBX { +struct CPRBX { unsigned short cprb_len; /* CPRB length 220 */ unsigned char cprb_ver_id; /* CPRB version id. 0x02 */ unsigned char pad_000[3]; /* Alignment pad bytes */ @@ -130,7 +130,7 @@ struct ica_CPRBX { unsigned char cntrl_domain[4];/* Control domain */ unsigned char S390enf_mask[4];/* S/390 enforcement mask */ unsigned char pad_004[36]; /* reserved */ -}; +} __attribute__((packed)); /** * xcRB |