From 95492e4646e5de8b43d9a7908d6177fb737b61f0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 16 Feb 2007 01:27:34 -0800 Subject: [PATCH] x86: rewrite SMP TSC sync code make the TSC synchronization code more robust, and unify it between x86_64 and i386. The biggest change is the removal of the 'fix up TSCs' code on x86_64 and i386, in some rare cases it was /causing/ time-warps on SMP systems. The new code only checks for TSC asynchronity - and if it can prove a time-warp (if it can observe the TSC going backwards when going from one CPU to another within a critical section), then the TSC clock-source is turned off. The TSC synchronization-checking code also got moved into a separate file. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Cc: john stultz Cc: Roman Zippel Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/asm-i386/tsc.h | 49 +--------------------------------- include/asm-x86_64/proto.h | 2 -- include/asm-x86_64/timex.h | 26 +----------------- include/asm-x86_64/tsc.h | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 75 deletions(-) create mode 100644 include/asm-x86_64/tsc.h (limited to 'include') diff --git a/include/asm-i386/tsc.h b/include/asm-i386/tsc.h index c13933185c1..e997891cc7c 100644 --- a/include/asm-i386/tsc.h +++ b/include/asm-i386/tsc.h @@ -1,48 +1 @@ -/* - * linux/include/asm-i386/tsc.h - * - * i386 TSC related functions - */ -#ifndef _ASM_i386_TSC_H -#define _ASM_i386_TSC_H - -#include - -/* - * Standard way to access the cycle counter on i586+ CPUs. - * Currently only used on SMP. - * - * If you really have a SMP machine with i486 chips or older, - * compile for that, and this will just always return zero. - * That's ok, it just means that the nicer scheduling heuristics - * won't work for you. - * - * We only use the low 32 bits, and we'd simply better make sure - * that we reschedule before that wraps. Scheduling at least every - * four billion cycles just basically sounds like a good idea, - * regardless of how fast the machine is. - */ -typedef unsigned long long cycles_t; - -extern unsigned int cpu_khz; -extern unsigned int tsc_khz; - -static inline cycles_t get_cycles(void) -{ - unsigned long long ret = 0; - -#ifndef CONFIG_X86_TSC - if (!cpu_has_tsc) - return 0; -#endif - -#if defined(CONFIG_X86_GENERIC) || defined(CONFIG_X86_TSC) - rdtscll(ret); -#endif - return ret; -} - -extern void tsc_init(void); -extern void mark_tsc_unstable(void); - -#endif +#include diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index a6d2ff5c69b..2ce3adf7bfd 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -91,8 +91,6 @@ extern void check_efer(void); extern int unhandled_signal(struct task_struct *tsk, int sig); -extern int unsynchronized_tsc(void); - extern void select_idle_routine(const struct cpuinfo_x86 *c); extern unsigned long table_start, table_end; diff --git a/include/asm-x86_64/timex.h b/include/asm-x86_64/timex.h index b9e5320b762..a4493a77d64 100644 --- a/include/asm-x86_64/timex.h +++ b/include/asm-x86_64/timex.h @@ -12,35 +12,11 @@ #include #include #include +#include #include #define CLOCK_TICK_RATE PIT_TICK_RATE /* Underlying HZ */ -typedef unsigned long long cycles_t; - -static inline cycles_t get_cycles (void) -{ - unsigned long long ret; - - rdtscll(ret); - return ret; -} - -/* Like get_cycles, but make sure the CPU is synchronized. */ -static __always_inline cycles_t get_cycles_sync(void) -{ - unsigned long long ret; - unsigned eax; - /* Don't do an additional sync on CPUs where we know - RDTSC is already synchronous. */ - alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC, - "=a" (eax), "0" (1) : "ebx","ecx","edx","memory"); - rdtscll(ret); - return ret; -} - -extern unsigned int cpu_khz; - extern int read_current_timer(unsigned long *timer_value); #define ARCH_HAS_READ_CURRENT_TIMER 1 diff --git a/include/asm-x86_64/tsc.h b/include/asm-x86_64/tsc.h new file mode 100644 index 00000000000..9a0a368852c --- /dev/null +++ b/include/asm-x86_64/tsc.h @@ -0,0 +1,66 @@ +/* + * linux/include/asm-x86_64/tsc.h + * + * x86_64 TSC related functions + */ +#ifndef _ASM_x86_64_TSC_H +#define _ASM_x86_64_TSC_H + +#include + +/* + * Standard way to access the cycle counter. + */ +typedef unsigned long long cycles_t; + +extern unsigned int cpu_khz; +extern unsigned int tsc_khz; + +static inline cycles_t get_cycles(void) +{ + unsigned long long ret = 0; + +#ifndef CONFIG_X86_TSC + if (!cpu_has_tsc) + return 0; +#endif + +#if defined(CONFIG_X86_GENERIC) || defined(CONFIG_X86_TSC) + rdtscll(ret); +#endif + return ret; +} + +/* Like get_cycles, but make sure the CPU is synchronized. */ +static __always_inline cycles_t get_cycles_sync(void) +{ + unsigned long long ret; +#ifdef X86_FEATURE_SYNC_RDTSC + unsigned eax; + + /* + * Don't do an additional sync on CPUs where we know + * RDTSC is already synchronous: + */ + alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC, + "=a" (eax), "0" (1) : "ebx","ecx","edx","memory"); +#else + sync_core(); +#endif + rdtscll(ret); + + return ret; +} + +extern void tsc_init(void); +extern void mark_tsc_unstable(void); +extern int unsynchronized_tsc(void); + +/* + * Boot-time check whether the TSCs are synchronized across + * all CPUs/cores: + */ +extern void check_tsc_sync_source(int cpu); +extern void check_tsc_sync_target(void); + +#endif -- cgit v1.2.3