summaryrefslogtreecommitdiffstats
path: root/include/linux/mutex.h
blob: 3793ed7feeeb1de4f0eec721da5bf008b92f786d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
/*
 * Mutexes: blocking mutual exclusion locks
 *
 * started by Ingo Molnar:
 *
 *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *
 * This file contains the main data structure and API definitions.
 */
#ifndef __LINUX_MUTEX_H
#define __LINUX_MUTEX_H

#include <asm/current.h>
#include <linux/list.h>
#include <linux/spinlock_types.h>
#include <linux/linkage.h>
#include <linux/lockdep.h>

#include <linux/atomic.h>

/*
 * Simple, straightforward mutexes with strict semantics:
 *
 * - only one task can hold the mutex at a time
 * - only the owner can unlock the mutex
 * - multiple unlocks are not permitted
 * - recursive locking is not permitted
 * - a mutex object must be initialized via the API
 * - a mutex object must not be initialized via memset or copying
 * - task may not exit with mutex held
 * - memory areas where held locks reside must not be freed
 * - held mutexes must not be reinitialized
 * - mutexes may not be used in hardware or software interrupt
 *   contexts such as tasklets and timers
 *
 * These semantics are fully enforced when DEBUG_MUTEXES is
 * enabled. Furthermore, besides enforcing the above rules, the mutex
 * debugging code also implements a number of additional features
 * that make lock debugging easier and faster:
 *
 * - uses symbolic names of mutexes, whenever they are printed in debug output
 * - point-of-acquire tracking, symbolic lookup of function names
 * - list of all locks held in the system, printout of them
 * - owner tracking
 * - detects self-recursing locks and prints out all relevant info
 * - detects multi-task circular deadlocks and prints out all affected
 *   locks and tasks (and only those tasks)
 */
struct mutex {
	/* 1: unlocked, 0: locked, negative: locked, possible waiters */
	atomic_t		count;
	spinlock_t		wait_lock;
	struct list_head	wait_list;
#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_SMP)
	struct task_struct	*owner;
#endif
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
	void			*spin_mlock;	/* Spinner MCS lock */
#endif
#ifdef CONFIG_DEBUG_MUTEXES
	const char 		*name;
	void			*magic;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	struct lockdep_map	dep_map;
#endif
};

/*
 * This is the control structure for tasks blocked on mutex,
 * which resides on the blocked task's kernel stack:
 */
struct mutex_waiter {
	struct list_head	list;
	struct task_struct	*task;
#ifdef CONFIG_DEBUG_MUTEXES
	void			*magic;
#endif
};

struct ww_class {
	atomic_long_t stamp;
	struct lock_class_key acquire_key;
	struct lock_class_key mutex_key;
	const char *acquire_name;
	const char *mutex_name;
};

struct ww_acquire_ctx {
	struct task_struct *task;
	unsigned long stamp;
	unsigned acquired;
#ifdef CONFIG_DEBUG_MUTEXES
	unsigned done_acquire;
	struct ww_class *ww_class;
	struct ww_mutex *contending_lock;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	struct lockdep_map dep_map;
#endif
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
	unsigned deadlock_inject_interval;
	unsigned deadlock_inject_countdown;
#endif
};

struct ww_mutex {
	struct mutex base;
	struct ww_acquire_ctx *ctx;
#ifdef CONFIG_DEBUG_MUTEXES
	struct ww_class *ww_class;
#endif
};

#ifdef CONFIG_DEBUG_MUTEXES
# include <linux/mutex-debug.h>
#else
# define __DEBUG_MUTEX_INITIALIZER(lockname)
/**
 * mutex_init - initialize the mutex
 * @mutex: the mutex to be initialized
 *
 * Initialize the mutex to unlocked state.
 *
 * It is not allowed to initialize an already locked mutex.
 */
# define mutex_init(mutex) \
do {							\
	static struct lock_class_key __key;		\
							\
	__mutex_init((mutex), #mutex, &__key);		\
} while (0)
static inline void mutex_destroy(struct mutex *lock) {}
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
		, .dep_map = { .name = #lockname }
# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \
		, .ww_class = &ww_class
#else
# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class)
#endif

#define __MUTEX_INITIALIZER(lockname) \
		{ .count = ATOMIC_INIT(1) \
		, .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
		, .wait_list = LIST_HEAD_INIT(lockname.wait_list) \
		__DEBUG_MUTEX_INITIALIZER(lockname) \
		__DEP_MAP_MUTEX_INITIALIZER(lockname) }

#define __WW_CLASS_INITIALIZER(ww_class) \
		{ .stamp = ATOMIC_LONG_INIT(0) \
		, .acquire_name = #ww_class "_acquire" \
		, .mutex_name = #ww_class "_mutex" }

#define __WW_MUTEX_INITIALIZER(lockname, class) \
		{ .base = { \__MUTEX_INITIALIZER(lockname) } \
		__WW_CLASS_MUTEX_INITIALIZER(lockname, class) }

#define DEFINE_MUTEX(mutexname) \
	struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)

#define DEFINE_WW_CLASS(classname) \
	struct ww_class classname = __WW_CLASS_INITIALIZER(classname)

#define DEFINE_WW_MUTEX(mutexname, ww_class) \
	struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class)


extern void __mutex_init(struct mutex *lock, const char *name,
			 struct lock_class_key *key);

/**
 * ww_mutex_init - initialize the w/w mutex
 * @lock: the mutex to be initialized
 * @ww_class: the w/w class the mutex should belong to
 *
 * Initialize the w/w mutex to unlocked state and associate it with the given
 * class.
 *
 * It is not allowed to initialize an already locked mutex.
 */
static inline void ww_mutex_init(struct ww_mutex *lock,
				 struct ww_class *ww_class)
{
	__mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key);
	lock->ctx = NULL;
#ifdef CONFIG_DEBUG_MUTEXES
	lock->ww_class = ww_class;
#endif
}

/**
 * mutex_is_locked - is the mutex locked
 * @lock: the mutex to be queried
 *
 * Returns 1 if the mutex is locked, 0 if unlocked.
 */
static inline int mutex_is_locked(struct mutex *lock)
{
	return atomic_read(&lock->count) != 1;
}

/*
 * See kernel/mutex.c for detailed documentation of these APIs.
 * Also see Documentation/mutex-design.txt.
 */
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass);
extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);

extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock,
					unsigned int subclass);
extern int __must_check mutex_lock_killable_nested(struct mutex *lock,
					unsigned int subclass);

#define mutex_lock(lock) mutex_lock_nested(lock, 0)
#define mutex_lock_interruptible(lock) mutex_lock_interruptible_nested(lock, 0)
#define mutex_lock_killable(lock) mutex_lock_killable_nested(lock, 0)

#define mutex_lock_nest_lock(lock, nest_lock)				\
do {									\
	typecheck(struct lockdep_map *, &(nest_lock)->dep_map);	\
	_mutex_lock_nest_lock(lock, &(nest_lock)->dep_map);		\
} while (0)

#else
extern void mutex_lock(struct mutex *lock);
extern int __must_check mutex_lock_interruptible(struct mutex *lock);
extern int __must_check mutex_lock_killable(struct mutex *lock);

# define mutex_lock_nested(lock, subclass) mutex_lock(lock)
# define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
# define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock)
#endif

/*
 * NOTE: mutex_trylock() follows the spin_trylock() convention,
 *       not the down_trylock() convention!
 *
 * Returns 1 if the mutex has been acquired successfully, and 0 on contention.
 */
extern int mutex_trylock(struct mutex *lock);
extern void mutex_unlock(struct mutex *lock);

/**
 * ww_acquire_init - initialize a w/w acquire context
 * @ctx: w/w acquire context to initialize
 * @ww_class: w/w class of the context
 *
 * Initializes an context to acquire multiple mutexes of the given w/w class.
 *
 * Context-based w/w mutex acquiring can be done in any order whatsoever within
 * a given lock class. Deadlocks will be detected and handled with the
 * wait/wound logic.
 *
 * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can
 * result in undetected deadlocks and is so forbidden. Mixing different contexts
 * for the same w/w class when acquiring mutexes can also result in undetected
 * deadlocks, and is hence also forbidden. Both types of abuse will be caught by
 * enabling CONFIG_PROVE_LOCKING.
 *
 * Nesting of acquire contexts for _different_ w/w classes is possible, subject
 * to the usual locking rules between different lock classes.
 *
 * An acquire context must be released with ww_acquire_fini by the same task
 * before the memory is freed. It is recommended to allocate the context itself
 * on the stack.
 */
static inline void ww_acquire_init(struct ww_acquire_ctx *ctx,
				   struct ww_class *ww_class)
{
	ctx->task = current;
	ctx->stamp = atomic_long_inc_return(&ww_class->stamp);
	ctx->acquired = 0;
#ifdef CONFIG_DEBUG_MUTEXES
	ctx->ww_class = ww_class;
	ctx->done_acquire = 0;
	ctx->contending_lock = NULL;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	debug_check_no_locks_freed((void *)ctx, sizeof(*ctx));
	lockdep_init_map(&ctx->dep_map, ww_class->acquire_name,
			 &ww_class->acquire_key, 0);
	mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_);
#endif
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
	ctx->deadlock_inject_interval = 1;
	ctx->deadlock_inject_countdown = ctx->stamp & 0xf;
#endif
}

/**
 * ww_acquire_done - marks the end of the acquire phase
 * @ctx: the acquire context
 *
 * Marks the end of the acquire phase, any further w/w mutex lock calls using
 * this context are forbidden.
 *
 * Calling this function is optional, it is just useful to document w/w mutex
 * code and clearly designated the acquire phase from actually using the locked
 * data structures.
 */
static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
{
#ifdef CONFIG_DEBUG_MUTEXES
	lockdep_assert_held(ctx);

	DEBUG_LOCKS_WARN_ON(ctx->done_acquire);
	ctx->done_acquire = 1;
#endif
}

/**
 * ww_acquire_fini - releases a w/w acquire context
 * @ctx: the acquire context to free
 *
 * Releases a w/w acquire context. This must be called _after_ all acquired w/w
 * mutexes have been released with ww_mutex_unlock.
 */
static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx)
{
#ifdef CONFIG_DEBUG_MUTEXES
	mutex_release(&ctx->dep_map, 0, _THIS_IP_);

	DEBUG_LOCKS_WARN_ON(ctx->acquired);
	if (!config_enabled(CONFIG_PROVE_LOCKING))
		/*
		 * lockdep will normally handle this,
		 * but fail without anyway
		 */
		ctx->done_acquire = 1;

	if (!config_enabled(CONFIG_DEBUG_LOCK_ALLOC))
		/* ensure ww_acquire_fini will still fail if called twice */
		ctx->acquired = ~0U;
#endif
}

extern int __must_check __ww_mutex_lock(struct ww_mutex *lock,
					struct ww_acquire_ctx *ctx);
extern int __must_check __ww_mutex_lock_interruptible(struct ww_mutex *lock,
						      struct ww_acquire_ctx *ctx);

/**
 * ww_mutex_lock - acquire the w/w mutex
 * @lock: the mutex to be acquired
 * @ctx: w/w acquire context, or NULL to acquire only a single lock.
 *
 * Lock the w/w mutex exclusively for this task.
 *
 * Deadlocks within a given w/w class of locks are detected and handled with the
 * wait/wound algorithm. If the lock isn't immediately avaiable this function
 * will either sleep until it is (wait case). Or it selects the current context
 * for backing off by returning -EDEADLK (wound case). Trying to acquire the
 * same lock with the same context twice is also detected and signalled by
 * returning -EALREADY. Returns 0 if the mutex was successfully acquired.
 *
 * In the wound case the caller must release all currently held w/w mutexes for
 * the given context and then wait for this contending lock to be available by
 * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this
 * lock and proceed with trying to acquire further w/w mutexes (e.g. when
 * scanning through lru lists trying to free resources).
 *
 * The mutex must later on be released by the same task that
 * acquired it. The task may not exit without first unlocking the mutex. Also,
 * kernel memory where the mutex resides must not be freed with the mutex still
 * locked. The mutex must first be initialized (or statically defined) before it
 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be
 * of the same w/w lock class as was used to initialize the acquire context.
 *
 * A mutex acquired with this function must be released with ww_mutex_unlock.
 */
static inline int ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
	if (ctx)
		return __ww_mutex_lock(lock, ctx);
	else {
		mutex_lock(&lock->base);
		return 0;
	}
}

/**
 * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible
 * @lock: the mutex to be acquired
 * @ctx: w/w acquire context
 *
 * Lock the w/w mutex exclusively for this task.
 *
 * Deadlocks within a given w/w class of locks are detected and handled with the
 * wait/wound algorithm. If the lock isn't immediately avaiable this function
 * will either sleep until it is (wait case). Or it selects the current context
 * for backing off by returning -EDEADLK (wound case). Trying to acquire the
 * same lock with the same context twice is also detected and signalled by
 * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a
 * signal arrives while waiting for the lock then this function returns -EINTR.
 *
 * In the wound case the caller must release all currently held w/w mutexes for
 * the given context and then wait for this contending lock to be available by
 * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to
 * not acquire this lock and proceed with trying to acquire further w/w mutexes
 * (e.g. when scanning through lru lists trying to free resources).
 *
 * The mutex must later on be released by the same task that
 * acquired it. The task may not exit without first unlocking the mutex. Also,
 * kernel memory where the mutex resides must not be freed with the mutex still
 * locked. The mutex must first be initialized (or statically defined) before it
 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be
 * of the same w/w lock class as was used to initialize the acquire context.
 *
 * A mutex acquired with this function must be released with ww_mutex_unlock.
 */
static inline int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock,
							   struct ww_acquire_ctx *ctx)
{
	if (ctx)
		return __ww_mutex_lock_interruptible(lock, ctx);
	else
		return mutex_lock_interruptible(&lock->base);
}

/**
 * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex
 * @lock: the mutex to be acquired
 * @ctx: w/w acquire context
 *
 * Acquires a w/w mutex with the given context after a wound case. This function
 * will sleep until the lock becomes available.
 *
 * The caller must have released all w/w mutexes already acquired with the
 * context and then call this function on the contended lock.
 *
 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it
 * needs with ww_mutex_lock. Note that the -EALREADY return code from
 * ww_mutex_lock can be used to avoid locking this contended mutex twice.
 *
 * It is forbidden to call this function with any other w/w mutexes associated
 * with the context held. It is forbidden to call this on anything else than the
 * contending mutex.
 *
 * Note that the slowpath lock acquiring can also be done by calling
 * ww_mutex_lock directly. This function here is simply to help w/w mutex
 * locking code readability by clearly denoting the slowpath.
 */
static inline void
ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
	int ret;
#ifdef CONFIG_DEBUG_MUTEXES
	DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
#endif
	ret = ww_mutex_lock(lock, ctx);
	(void)ret;
}

/**
 * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex,
 * 				      interruptible
 * @lock: the mutex to be acquired
 * @ctx: w/w acquire context
 *
 * Acquires a w/w mutex with the given context after a wound case. This function
 * will sleep until the lock becomes available and returns 0 when the lock has
 * been acquired. If a signal arrives while waiting for the lock then this
 * function returns -EINTR.
 *
 * The caller must have released all w/w mutexes already acquired with the
 * context and then call this function on the contended lock.
 *
 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it
 * needs with ww_mutex_lock. Note that the -EALREADY return code from
 * ww_mutex_lock can be used to avoid locking this contended mutex twice.
 *
 * It is forbidden to call this function with any other w/w mutexes associated
 * with the given context held. It is forbidden to call this on anything else
 * than the contending mutex.
 *
 * Note that the slowpath lock acquiring can also be done by calling
 * ww_mutex_lock_interruptible directly. This function here is simply to help
 * w/w mutex locking code readability by clearly denoting the slowpath.
 */
static inline int __must_check
ww_mutex_lock_slow_interruptible(struct ww_mutex *lock,
				 struct ww_acquire_ctx *ctx)
{
#ifdef CONFIG_DEBUG_MUTEXES
	DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
#endif
	return ww_mutex_lock_interruptible(lock, ctx);
}

extern void ww_mutex_unlock(struct ww_mutex *lock);

/**
 * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context
 * @lock: mutex to lock
 *
 * Trylocks a mutex without acquire context, so no deadlock detection is
 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise.
 */
static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock)
{
	return mutex_trylock(&lock->base);
}

/***
 * ww_mutex_destroy - mark a w/w mutex unusable
 * @lock: the mutex to be destroyed
 *
 * This function marks the mutex uninitialized, and any subsequent
 * use of the mutex is forbidden. The mutex must not be locked when
 * this function is called.
 */
static inline void ww_mutex_destroy(struct ww_mutex *lock)
{
	mutex_destroy(&lock->base);
}

/**
 * ww_mutex_is_locked - is the w/w mutex locked
 * @lock: the mutex to be queried
 *
 * Returns 1 if the mutex is locked, 0 if unlocked.
 */
static inline bool ww_mutex_is_locked(struct ww_mutex *lock)
{
	return mutex_is_locked(&lock->base);
}

extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);

#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX
#define arch_mutex_cpu_relax()	cpu_relax()
#endif

#endif
OpenPOWER on IntegriCloud