diff options
Diffstat (limited to 'include/linux/rcupdate.h')
| -rw-r--r-- | include/linux/rcupdate.h | 243 |
1 files changed, 128 insertions, 115 deletions
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 39cbb889e20d..00a7fd61b3c6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -12,8 +12,8 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * * Copyright IBM Corporation, 2001 * @@ -44,19 +44,21 @@ #include <linux/debugobjects.h> #include <linux/bug.h> #include <linux/compiler.h> +#include <asm/barrier.h> +extern int rcu_expedited; /* for sysctl */ #ifdef CONFIG_RCU_TORTURE_TEST extern int rcutorture_runnable; /* for sysctl */ #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */ #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) -extern void rcutorture_record_test_transition(void); -extern void rcutorture_record_progress(unsigned long vernum); -extern void do_trace_rcu_torture_read(const char *rcutorturename, - struct rcu_head *rhp, - unsigned long secs, - unsigned long c_old, - unsigned long c); +void rcutorture_record_test_transition(void); +void rcutorture_record_progress(unsigned long vernum); +void do_trace_rcu_torture_read(const char *rcutorturename, + struct rcu_head *rhp, + unsigned long secs, + unsigned long c_old, + unsigned long c); #else static inline void rcutorture_record_test_transition(void) { @@ -65,11 +67,11 @@ static inline void rcutorture_record_progress(unsigned long vernum) { } #ifdef CONFIG_RCU_TRACE -extern void do_trace_rcu_torture_read(const char *rcutorturename, - struct rcu_head *rhp, - unsigned long secs, - unsigned long c_old, - unsigned long c); +void do_trace_rcu_torture_read(const char *rcutorturename, + struct rcu_head *rhp, + unsigned long secs, + unsigned long c_old, + unsigned long c); #else #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ do { } while (0) @@ -118,8 +120,8 @@ extern void do_trace_rcu_torture_read(const char *rcutorturename, * if CPU A and CPU B are the same CPU (but again only if the system has * more than one CPU). */ -extern void call_rcu(struct rcu_head *head, - void (*func)(struct rcu_head *head)); +void call_rcu(struct rcu_head *head, + void (*func)(struct rcu_head *head)); #else /* #ifdef CONFIG_PREEMPT_RCU */ @@ -149,8 +151,8 @@ extern void call_rcu(struct rcu_head *head, * See the description of call_rcu() for more detailed information on * memory ordering guarantees. */ -extern void call_rcu_bh(struct rcu_head *head, - void (*func)(struct rcu_head *head)); +void call_rcu_bh(struct rcu_head *head, + void (*func)(struct rcu_head *head)); /** * call_rcu_sched() - Queue an RCU for invocation after sched grace period. @@ -171,16 +173,16 @@ extern void call_rcu_bh(struct rcu_head *head, * See the description of call_rcu() for more detailed information on * memory ordering guarantees. */ -extern void call_rcu_sched(struct rcu_head *head, - void (*func)(struct rcu_head *rcu)); +void call_rcu_sched(struct rcu_head *head, + void (*func)(struct rcu_head *rcu)); -extern void synchronize_sched(void); +void synchronize_sched(void); #ifdef CONFIG_PREEMPT_RCU -extern void __rcu_read_lock(void); -extern void __rcu_read_unlock(void); -extern void rcu_read_unlock_special(struct task_struct *t); +void __rcu_read_lock(void); +void __rcu_read_unlock(void); +void rcu_read_unlock_special(struct task_struct *t); void synchronize_rcu(void); /* @@ -216,19 +218,19 @@ static inline int rcu_preempt_depth(void) #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* Internal to kernel */ -extern void rcu_init(void); -extern void rcu_sched_qs(int cpu); -extern void rcu_bh_qs(int cpu); -extern void rcu_check_callbacks(int cpu, int user); +void rcu_init(void); +void rcu_sched_qs(int cpu); +void rcu_bh_qs(int cpu); +void rcu_check_callbacks(int cpu, int user); struct notifier_block; -extern void rcu_idle_enter(void); -extern void rcu_idle_exit(void); -extern void rcu_irq_enter(void); -extern void rcu_irq_exit(void); +void rcu_idle_enter(void); +void rcu_idle_exit(void); +void rcu_irq_enter(void); +void rcu_irq_exit(void); #ifdef CONFIG_RCU_USER_QS -extern void rcu_user_enter(void); -extern void rcu_user_exit(void); +void rcu_user_enter(void); +void rcu_user_exit(void); #else static inline void rcu_user_enter(void) { } static inline void rcu_user_exit(void) { } @@ -262,7 +264,7 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev, } while (0) #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) -extern bool __rcu_is_watching(void); +bool __rcu_is_watching(void); #endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */ /* @@ -289,8 +291,8 @@ void wait_rcu_gp(call_rcu_func_t crf); * initialization. */ #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD -extern void init_rcu_head_on_stack(struct rcu_head *head); -extern void destroy_rcu_head_on_stack(struct rcu_head *head); +void init_rcu_head_on_stack(struct rcu_head *head); +void destroy_rcu_head_on_stack(struct rcu_head *head); #else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ static inline void init_rcu_head_on_stack(struct rcu_head *head) { @@ -314,7 +316,7 @@ static inline bool rcu_lockdep_current_cpu_online(void) static inline void rcu_lock_acquire(struct lockdep_map *map) { - lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_); + lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_); } static inline void rcu_lock_release(struct lockdep_map *map) @@ -325,6 +327,7 @@ static inline void rcu_lock_release(struct lockdep_map *map) extern struct lockdep_map rcu_lock_map; extern struct lockdep_map rcu_bh_lock_map; extern struct lockdep_map rcu_sched_lock_map; +extern struct lockdep_map rcu_callback_map; extern int debug_lockdep_rcu_enabled(void); /** @@ -362,7 +365,7 @@ static inline int rcu_read_lock_held(void) * rcu_read_lock_bh_held() is defined out of line to avoid #include-file * hell. */ -extern int rcu_read_lock_bh_held(void); +int rcu_read_lock_bh_held(void); /** * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? @@ -448,8 +451,6 @@ static inline int rcu_read_lock_sched_held(void) #ifdef CONFIG_PROVE_RCU -extern int rcu_my_thread_group_empty(void); - /** * rcu_lockdep_assert - emit lockdep splat if specified condition not met * @c: condition to check @@ -480,11 +481,9 @@ static inline void rcu_preempt_sleep_check(void) do { \ rcu_preempt_sleep_check(); \ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \ - "Illegal context switch in RCU-bh" \ - " read-side critical section"); \ + "Illegal context switch in RCU-bh read-side critical section"); \ rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \ - "Illegal context switch in RCU-sched"\ - " read-side critical section"); \ + "Illegal context switch in RCU-sched read-side critical section"); \ } while (0) #else /* #ifdef CONFIG_PROVE_RCU */ @@ -511,49 +510,79 @@ static inline void rcu_preempt_sleep_check(void) #endif /* #else #ifdef __CHECKER__ */ #define __rcu_access_pointer(p, space) \ - ({ \ - typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ - rcu_dereference_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(_________p1)); \ - }) +({ \ + typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \ + rcu_dereference_sparse(p, space); \ + ((typeof(*p) __force __kernel *)(_________p1)); \ +}) #define __rcu_dereference_check(p, c, space) \ - ({ \ - typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \ - rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \ - " usage"); \ - rcu_dereference_sparse(p, space); \ - smp_read_barrier_depends(); \ - ((typeof(*p) __force __kernel *)(_________p1)); \ - }) +({ \ + typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \ + rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \ + rcu_dereference_sparse(p, space); \ + smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ + ((typeof(*p) __force __kernel *)(_________p1)); \ +}) #define __rcu_dereference_protected(p, c, space) \ - ({ \ - rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \ - " usage"); \ - rcu_dereference_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(p)); \ - }) +({ \ + rcu_lockdep_assert(c, "suspicious rcu_dereference_protected() usage"); \ + rcu_dereference_sparse(p, space); \ + ((typeof(*p) __force __kernel *)(p)); \ +}) #define __rcu_access_index(p, space) \ - ({ \ - typeof(p) _________p1 = ACCESS_ONCE(p); \ - rcu_dereference_sparse(p, space); \ - (_________p1); \ - }) +({ \ + typeof(p) _________p1 = ACCESS_ONCE(p); \ + rcu_dereference_sparse(p, space); \ + (_________p1); \ +}) #define __rcu_dereference_index_check(p, c) \ - ({ \ - typeof(p) _________p1 = ACCESS_ONCE(p); \ - rcu_lockdep_assert(c, \ - "suspicious rcu_dereference_index_check()" \ - " usage"); \ - smp_read_barrier_depends(); \ - (_________p1); \ - }) -#define __rcu_assign_pointer(p, v, space) \ - do { \ - smp_wmb(); \ - (p) = (typeof(*v) __force space *)(v); \ - } while (0) +({ \ + typeof(p) _________p1 = ACCESS_ONCE(p); \ + rcu_lockdep_assert(c, \ + "suspicious rcu_dereference_index_check() usage"); \ + smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ + (_________p1); \ +}) + +/** + * RCU_INITIALIZER() - statically initialize an RCU-protected global variable + * @v: The value to statically initialize with. + */ +#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v) +/** + * rcu_assign_pointer() - assign to RCU-protected pointer + * @p: pointer to assign to + * @v: value to assign (publish) + * + * Assigns the specified value to the specified RCU-protected + * pointer, ensuring that any concurrent RCU readers will see + * any prior initialization. + * + * Inserts memory barriers on architectures that require them + * (which is most of them), and also prevents the compiler from + * reordering the code that initializes the structure after the pointer + * assignment. More importantly, this call documents which pointers + * will be dereferenced by RCU read-side code. + * + * In some special cases, you may use RCU_INIT_POINTER() instead + * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due + * to the fact that it does not constrain either the CPU or the compiler. + * That said, using RCU_INIT_POINTER() when you should have used + * rcu_assign_pointer() is a very bad thing that results in + * impossible-to-diagnose memory corruption. So please be careful. + * See the RCU_INIT_POINTER() comment header for details. + * + * Note that rcu_assign_pointer() evaluates each of its arguments only + * once, appearances notwithstanding. One of the "extra" evaluations + * is in typeof() and the other visible only to sparse (__CHECKER__), + * neither of which actually execute the argument. As with most cpp + * macros, this execute-arguments-only-once property is important, so + * please be careful when making changes to rcu_assign_pointer() and the + * other macros that it invokes. + */ +#define rcu_assign_pointer(p, v) smp_store_release(&p, RCU_INITIALIZER(v)) /** * rcu_access_pointer() - fetch RCU pointer with no dereferencing @@ -890,32 +919,6 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) } /** - * rcu_assign_pointer() - assign to RCU-protected pointer - * @p: pointer to assign to - * @v: value to assign (publish) - * - * Assigns the specified value to the specified RCU-protected - * pointer, ensuring that any concurrent RCU readers will see - * any prior initialization. - * - * Inserts memory barriers on architectures that require them - * (which is most of them), and also prevents the compiler from - * reordering the code that initializes the structure after the pointer - * assignment. More importantly, this call documents which pointers - * will be dereferenced by RCU read-side code. - * - * In some special cases, you may use RCU_INIT_POINTER() instead - * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due - * to the fact that it does not constrain either the CPU or the compiler. - * That said, using RCU_INIT_POINTER() when you should have used - * rcu_assign_pointer() is a very bad thing that results in - * impossible-to-diagnose memory corruption. So please be careful. - * See the RCU_INIT_POINTER() comment header for details. - */ -#define rcu_assign_pointer(p, v) \ - __rcu_assign_pointer((p), (v), __rcu) - -/** * RCU_INIT_POINTER() - initialize an RCU protected pointer * * Initialize an RCU-protected pointer in special cases where readers @@ -949,7 +952,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) */ #define RCU_INIT_POINTER(p, v) \ do { \ - p = (typeof(*v) __force __rcu *)(v); \ + p = RCU_INITIALIZER(v); \ } while (0) /** @@ -958,7 +961,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * GCC-style initialization for an RCU-protected pointer in a structure field. */ #define RCU_POINTER_INITIALIZER(p, v) \ - .p = (typeof(*v) __force __rcu *)(v) + .p = RCU_INITIALIZER(v) /* * Does the specified offset indicate that the corresponding rcu_head @@ -1004,17 +1007,27 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) #define kfree_rcu(ptr, rcu_head) \ __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) -#ifdef CONFIG_RCU_NOCB_CPU -extern bool rcu_is_nocb_cpu(int cpu); +#if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) +static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) +{ + *delta_jiffies = ULONG_MAX; + return 0; +} +#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */ + +#if defined(CONFIG_RCU_NOCB_CPU_ALL) +static inline bool rcu_is_nocb_cpu(int cpu) { return true; } +#elif defined(CONFIG_RCU_NOCB_CPU) +bool rcu_is_nocb_cpu(int cpu); #else static inline bool rcu_is_nocb_cpu(int cpu) { return false; } -#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ +#endif /* Only for use by adaptive-ticks code. */ #ifdef CONFIG_NO_HZ_FULL_SYSIDLE -extern bool rcu_sys_is_idle(void); -extern void rcu_sysidle_force_exit(void); +bool rcu_sys_is_idle(void); +void rcu_sysidle_force_exit(void); #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ static inline bool rcu_sys_is_idle(void) |