2006-01-10 11:52:32 +07:00
|
|
|
/*
|
|
|
|
* linux/kernel/hrtimer.c
|
|
|
|
*
|
2007-02-16 16:27:50 +07:00
|
|
|
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
|
2007-02-16 16:28:03 +07:00
|
|
|
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
|
2007-02-16 16:28:11 +07:00
|
|
|
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
|
2006-01-10 11:52:32 +07:00
|
|
|
*
|
|
|
|
* High-resolution kernel timers
|
|
|
|
*
|
|
|
|
* In contrast to the low-resolution timeout API implemented in
|
|
|
|
* kernel/timer.c, hrtimers provide finer resolution and accuracy
|
|
|
|
* depending on system configuration and capabilities.
|
|
|
|
*
|
|
|
|
* These timers are currently used for:
|
|
|
|
* - itimers
|
|
|
|
* - POSIX timers
|
|
|
|
* - nanosleep
|
|
|
|
* - precise in-kernel timing
|
|
|
|
*
|
|
|
|
* Started by: Thomas Gleixner and Ingo Molnar
|
|
|
|
*
|
|
|
|
* Credits:
|
|
|
|
* based on kernel/timer.c
|
|
|
|
*
|
2006-02-01 18:05:13 +07:00
|
|
|
* Help, testing, suggestions, bugfixes, improvements were
|
|
|
|
* provided by:
|
|
|
|
*
|
|
|
|
* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
|
|
|
|
* et. al.
|
|
|
|
*
|
2006-01-10 11:52:32 +07:00
|
|
|
* For licencing details see kernel-base/COPYING
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/cpu.h>
|
2011-05-24 01:51:41 +07:00
|
|
|
#include <linux/export.h>
|
2006-01-10 11:52:32 +07:00
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/hrtimer.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/syscalls.h>
|
2007-02-16 16:28:11 +07:00
|
|
|
#include <linux/kallsyms.h>
|
2006-01-10 11:52:32 +07:00
|
|
|
#include <linux/interrupt.h>
|
2007-02-16 16:28:03 +07:00
|
|
|
#include <linux/tick.h>
|
2007-02-16 16:28:11 +07:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/err.h>
|
2008-04-30 14:55:04 +07:00
|
|
|
#include <linux/debugobjects.h>
|
2017-02-03 01:15:33 +07:00
|
|
|
#include <linux/sched/signal.h>
|
2013-02-07 22:46:59 +07:00
|
|
|
#include <linux/sched/sysctl.h>
|
2013-02-07 22:47:07 +07:00
|
|
|
#include <linux/sched/rt.h>
|
sched/deadline: Add SCHED_DEADLINE structures & implementation
Introduces the data structures, constants and symbols needed for
SCHED_DEADLINE implementation.
Core data structure of SCHED_DEADLINE are defined, along with their
initializers. Hooks for checking if a task belong to the new policy
are also added where they are needed.
Adds a scheduling class, in sched/dl.c and a new policy called
SCHED_DEADLINE. It is an implementation of the Earliest Deadline
First (EDF) scheduling algorithm, augmented with a mechanism (called
Constant Bandwidth Server, CBS) that makes it possible to isolate
the behaviour of tasks between each other.
The typical -deadline task will be made up of a computation phase
(instance) which is activated on a periodic or sporadic fashion. The
expected (maximum) duration of such computation is called the task's
runtime; the time interval by which each instance need to be completed
is called the task's relative deadline. The task's absolute deadline
is dynamically calculated as the time instant a task (better, an
instance) activates plus the relative deadline.
The EDF algorithms selects the task with the smallest absolute
deadline as the one to be executed first, while the CBS ensures each
task to run for at most its runtime every (relative) deadline
length time interval, avoiding any interference between different
tasks (bandwidth isolation).
Thanks to this feature, also tasks that do not strictly comply with
the computational model sketched above can effectively use the new
policy.
To summarize, this patch:
- introduces the data structures, constants and symbols needed;
- implements the core logic of the scheduling algorithm in the new
scheduling class file;
- provides all the glue code between the new scheduling class and
the core scheduler and refines the interactions between sched/dl
and the other existing scheduling classes.
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Michael Trimarchi <michael@amarulasolutions.com>
Signed-off-by: Fabio Checconi <fchecconi@gmail.com>
Signed-off-by: Juri Lelli <juri.lelli@gmail.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1383831828-15501-4-git-send-email-juri.lelli@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-11-28 17:14:43 +07:00
|
|
|
#include <linux/sched/deadline.h>
|
2017-02-09 00:51:35 +07:00
|
|
|
#include <linux/sched/nohz.h>
|
2017-02-09 00:51:35 +07:00
|
|
|
#include <linux/sched/debug.h>
|
2009-04-16 13:46:41 +07:00
|
|
|
#include <linux/timer.h>
|
2013-05-07 06:50:19 +07:00
|
|
|
#include <linux/freezer.h>
|
2017-06-07 15:42:31 +07:00
|
|
|
#include <linux/compat.h>
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2016-12-25 02:46:01 +07:00
|
|
|
#include <linux/uaccess.h>
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2009-08-10 09:51:23 +07:00
|
|
|
#include <trace/events/timer.h>
|
|
|
|
|
2015-03-25 19:07:37 +07:00
|
|
|
#include "tick-internal.h"
|
2014-07-17 04:04:02 +07:00
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
|
|
|
* The timer bases:
|
2006-02-01 18:05:11 +07:00
|
|
|
*
|
2015-08-25 13:42:53 +07:00
|
|
|
* There are more clockids than hrtimer bases. Thus, we index
|
2010-12-15 10:37:07 +07:00
|
|
|
* into the timer bases by the hrtimer_base_type enum. When trying
|
|
|
|
* to reach a base using a clockid, hrtimer_clockid_to_base()
|
|
|
|
* is used to convert from clockid to the proper hrtimer_base_type.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2007-02-16 16:28:11 +07:00
|
|
|
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
hrtimer: Don't reinitialize a cpu_base lock on CPU_UP
The current code makes the assumption that a cpu_base lock won't be
held if the CPU corresponding to that cpu_base is offline, which isn't
always true.
If a hrtimer is not queued, then it will not be migrated by
migrate_hrtimers() when a CPU is offlined. Therefore, the hrtimer's
cpu_base may still point to a CPU which has subsequently gone offline
if the timer wasn't enqueued at the time the CPU went down.
Normally this wouldn't be a problem, but a cpu_base's lock is blindly
reinitialized each time a CPU is brought up. If a CPU is brought
online during the period that another thread is performing a hrtimer
operation on a stale hrtimer, then the lock will be reinitialized
under its feet, and a SPIN_BUG() like the following will be observed:
<0>[ 28.082085] BUG: spinlock already unlocked on CPU#0, swapper/0/0
<0>[ 28.087078] lock: 0xc4780b40, value 0x0 .magic: dead4ead, .owner: <none>/-1, .owner_cpu: -1
<4>[ 42.451150] [<c0014398>] (unwind_backtrace+0x0/0x120) from [<c0269220>] (do_raw_spin_unlock+0x44/0xdc)
<4>[ 42.460430] [<c0269220>] (do_raw_spin_unlock+0x44/0xdc) from [<c071b5bc>] (_raw_spin_unlock+0x8/0x30)
<4>[ 42.469632] [<c071b5bc>] (_raw_spin_unlock+0x8/0x30) from [<c00a9ce0>] (__hrtimer_start_range_ns+0x1e4/0x4f8)
<4>[ 42.479521] [<c00a9ce0>] (__hrtimer_start_range_ns+0x1e4/0x4f8) from [<c00aa014>] (hrtimer_start+0x20/0x28)
<4>[ 42.489247] [<c00aa014>] (hrtimer_start+0x20/0x28) from [<c00e6190>] (rcu_idle_enter_common+0x1ac/0x320)
<4>[ 42.498709] [<c00e6190>] (rcu_idle_enter_common+0x1ac/0x320) from [<c00e6440>] (rcu_idle_enter+0xa0/0xb8)
<4>[ 42.508259] [<c00e6440>] (rcu_idle_enter+0xa0/0xb8) from [<c000f268>] (cpu_idle+0x24/0xf0)
<4>[ 42.516503] [<c000f268>] (cpu_idle+0x24/0xf0) from [<c06ed3c0>] (rest_init+0x88/0xa0)
<4>[ 42.524319] [<c06ed3c0>] (rest_init+0x88/0xa0) from [<c0c00978>] (start_kernel+0x3d0/0x434)
As an example, this particular crash occurred when hrtimer_start() was
executed on CPU #0. The code locked the hrtimer's current cpu_base
corresponding to CPU #1. CPU #0 then tried to switch the hrtimer's
cpu_base to an optimal CPU which was online. In this case, it selected
the cpu_base corresponding to CPU #3.
Before it could proceed, CPU #1 came online and reinitialized the
spinlock corresponding to its cpu_base. Thus now CPU #0 held a lock
which was reinitialized. When CPU #0 finally ended up unlocking the
old cpu_base corresponding to CPU #1 so that it could switch to CPU
#3, we hit this SPIN_BUG() above while in switch_hrtimer_base().
CPU #0 CPU #1
---- ----
... <offline>
hrtimer_start()
lock_hrtimer_base(base #1)
... init_hrtimers_cpu()
switch_hrtimer_base() ...
... raw_spin_lock_init(&cpu_base->lock)
raw_spin_unlock(&cpu_base->lock) ...
<spin_bug>
Solve this by statically initializing the lock.
Signed-off-by: Michael Bohan <mbohan@codeaurora.org>
Link: http://lkml.kernel.org/r/1363745965-23475-1-git-send-email-mbohan@codeaurora.org
Cc: stable@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2013-03-20 09:19:25 +07:00
|
|
|
.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
|
2015-06-11 19:46:48 +07:00
|
|
|
.seq = SEQCNT_ZERO(hrtimer_bases.seq),
|
2007-02-16 16:27:50 +07:00
|
|
|
.clock_base =
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
{
|
2011-05-20 18:05:15 +07:00
|
|
|
.index = HRTIMER_BASE_MONOTONIC,
|
|
|
|
.clockid = CLOCK_MONOTONIC,
|
2007-02-16 16:27:50 +07:00
|
|
|
.get_time = &ktime_get,
|
|
|
|
},
|
2011-05-21 04:14:04 +07:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_REALTIME,
|
|
|
|
.clockid = CLOCK_REALTIME,
|
|
|
|
.get_time = &ktime_get_real,
|
|
|
|
},
|
2011-02-16 01:45:16 +07:00
|
|
|
{
|
2011-05-20 18:05:15 +07:00
|
|
|
.index = HRTIMER_BASE_BOOTTIME,
|
|
|
|
.clockid = CLOCK_BOOTTIME,
|
2011-02-16 01:45:16 +07:00
|
|
|
.get_time = &ktime_get_boottime,
|
|
|
|
},
|
2013-01-22 08:00:11 +07:00
|
|
|
{
|
|
|
|
.index = HRTIMER_BASE_TAI,
|
|
|
|
.clockid = CLOCK_TAI,
|
|
|
|
.get_time = &ktime_get_clocktai,
|
|
|
|
},
|
2007-02-16 16:27:50 +07:00
|
|
|
}
|
2006-01-10 11:52:32 +07:00
|
|
|
};
|
|
|
|
|
2011-05-03 02:24:27 +07:00
|
|
|
static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
|
2016-01-16 00:41:09 +07:00
|
|
|
/* Make sure we catch unsupported clockids */
|
|
|
|
[0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,
|
|
|
|
|
2011-04-29 05:02:00 +07:00
|
|
|
[CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
|
|
|
|
[CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
|
|
|
|
[CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
|
2013-01-22 08:00:11 +07:00
|
|
|
[CLOCK_TAI] = HRTIMER_BASE_TAI,
|
2011-04-29 05:02:00 +07:00
|
|
|
};
|
2010-12-15 10:37:07 +07:00
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
|
|
|
* Functions and macros which are different for UP/SMP systems are kept in a
|
|
|
|
* single place
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
/*
|
|
|
|
* We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
|
|
|
|
* such that hrtimer_callback_running() can unconditionally dereference
|
|
|
|
* timer->base->cpu_base
|
|
|
|
*/
|
|
|
|
static struct hrtimer_cpu_base migration_cpu_base = {
|
|
|
|
.seq = SEQCNT_ZERO(migration_cpu_base),
|
|
|
|
.clock_base = { { .cpu_base = &migration_cpu_base, }, },
|
|
|
|
};
|
|
|
|
|
|
|
|
#define migration_base migration_cpu_base.clock_base[0]
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
|
|
|
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
|
|
|
|
* means that all timers which are tied to this base via timer->base are
|
|
|
|
* locked, and the base itself is locked too.
|
|
|
|
*
|
|
|
|
* So __run_timers/migrate_timers can safely modify all timers which could
|
|
|
|
* be found on the lists/queues.
|
|
|
|
*
|
|
|
|
* When the timer's base is locked, and the timer removed from list, it is
|
2015-06-11 19:46:48 +07:00
|
|
|
* possible to set timer->base = &migration_base and drop the lock: the timer
|
|
|
|
* remains locked.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2007-02-16 16:27:50 +07:00
|
|
|
static
|
|
|
|
struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
|
|
|
|
unsigned long *flags)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
base = timer->base;
|
2015-06-11 19:46:48 +07:00
|
|
|
if (likely(base != &migration_base)) {
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-10 11:52:32 +07:00
|
|
|
if (likely(base == timer->base))
|
|
|
|
return base;
|
|
|
|
/* The timer has migrated to another CPU: */
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-07-10 19:57:05 +07:00
|
|
|
/*
|
|
|
|
* With HIGHRES=y we do not migrate the timer when it is expiring
|
|
|
|
* before the next event on the target cpu because we cannot reprogram
|
|
|
|
* the target cpu hardware and we would cause it to fire late.
|
|
|
|
*
|
|
|
|
* Called with cpu_base->lock of target cpu held.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
ktime_t expires;
|
|
|
|
|
|
|
|
if (!new_base->cpu_base->hres_active)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
|
2016-12-25 17:38:40 +07:00
|
|
|
return expires <= new_base->cpu_base->expires_next;
|
2009-07-10 19:57:05 +07:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-03-02 00:28:49 +07:00
|
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
2015-05-27 05:50:33 +07:00
|
|
|
static inline
|
|
|
|
struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
|
|
|
|
int pinned)
|
|
|
|
{
|
|
|
|
if (pinned || !base->migration_enabled)
|
2015-08-18 21:18:28 +07:00
|
|
|
return base;
|
2015-05-27 05:50:33 +07:00
|
|
|
return &per_cpu(hrtimer_bases, get_nohz_timer_target());
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline
|
|
|
|
struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
|
|
|
|
int pinned)
|
|
|
|
{
|
2015-08-18 21:18:28 +07:00
|
|
|
return base;
|
2015-05-27 05:50:33 +07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
2015-08-18 21:18:29 +07:00
|
|
|
* We switch the timer base to a power-optimized selected CPU target,
|
|
|
|
* if:
|
|
|
|
* - NO_HZ_COMMON is enabled
|
|
|
|
* - timer migration is enabled
|
|
|
|
* - the timer callback is not running
|
|
|
|
* - the timer is not the first expiring timer on the new target
|
|
|
|
*
|
|
|
|
* If one of the above requirements is not fulfilled we move the timer
|
|
|
|
* to the current CPU or leave it on the previously assigned CPU if
|
|
|
|
* the timer callback is currently running.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2007-02-16 16:27:50 +07:00
|
|
|
static inline struct hrtimer_clock_base *
|
2009-04-16 13:43:26 +07:00
|
|
|
switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
|
|
|
|
int pinned)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2015-08-18 21:18:29 +07:00
|
|
|
struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_clock_base *new_base;
|
2011-05-20 18:05:15 +07:00
|
|
|
int basenum = base->index;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-08-18 21:18:29 +07:00
|
|
|
this_cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
new_cpu_base = get_target_base(this_cpu_base, pinned);
|
2009-04-16 13:46:41 +07:00
|
|
|
again:
|
2010-12-15 10:37:07 +07:00
|
|
|
new_base = &new_cpu_base->clock_base[basenum];
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
if (base != new_base) {
|
|
|
|
/*
|
2009-07-10 19:57:05 +07:00
|
|
|
* We are trying to move timer to new_base.
|
2006-01-10 11:52:32 +07:00
|
|
|
* However we can't change timer's base while it is running,
|
|
|
|
* so we keep it on the same CPU. No hassle vs. reprogramming
|
|
|
|
* the event source in the high resolution case. The softirq
|
|
|
|
* code will take care of this when the timer function has
|
|
|
|
* completed. There is no conflict as we hold the lock until
|
|
|
|
* the timer is enqueued.
|
|
|
|
*/
|
2007-02-16 16:28:11 +07:00
|
|
|
if (unlikely(hrtimer_callback_running(timer)))
|
2006-01-10 11:52:32 +07:00
|
|
|
return base;
|
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
/* See the comment in lock_hrtimer_base() */
|
|
|
|
timer->base = &migration_base;
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock(&base->cpu_base->lock);
|
|
|
|
raw_spin_lock(&new_base->cpu_base->lock);
|
2009-04-16 13:46:41 +07:00
|
|
|
|
2015-08-18 21:18:29 +07:00
|
|
|
if (new_cpu_base != this_cpu_base &&
|
2015-05-27 05:50:33 +07:00
|
|
|
hrtimer_check_target(timer, new_base)) {
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock(&new_base->cpu_base->lock);
|
|
|
|
raw_spin_lock(&base->cpu_base->lock);
|
2015-08-18 21:18:29 +07:00
|
|
|
new_cpu_base = this_cpu_base;
|
2009-07-10 19:57:05 +07:00
|
|
|
timer->base = base;
|
|
|
|
goto again;
|
2009-04-16 13:46:41 +07:00
|
|
|
}
|
2006-01-10 11:52:32 +07:00
|
|
|
timer->base = new_base;
|
2014-04-30 15:43:10 +07:00
|
|
|
} else {
|
2015-08-18 21:18:29 +07:00
|
|
|
if (new_cpu_base != this_cpu_base &&
|
2015-05-27 05:50:33 +07:00
|
|
|
hrtimer_check_target(timer, new_base)) {
|
2015-08-18 21:18:29 +07:00
|
|
|
new_cpu_base = this_cpu_base;
|
2014-04-30 15:43:10 +07:00
|
|
|
goto again;
|
|
|
|
}
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
return new_base;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
2007-02-16 16:27:50 +07:00
|
|
|
static inline struct hrtimer_clock_base *
|
2006-01-10 11:52:32 +07:00
|
|
|
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_clock_base *base = timer->base;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
|
2009-04-16 13:46:41 +07:00
|
|
|
# define switch_hrtimer_base(t, b, p) (b)
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Functions for the union type storage format of ktime_t which are
|
|
|
|
* too large for inlining:
|
|
|
|
*/
|
|
|
|
#if BITS_PER_LONG < 64
|
|
|
|
/*
|
|
|
|
* Divide a ktime value by a nanosecond value
|
|
|
|
*/
|
2015-05-09 03:47:23 +07:00
|
|
|
s64 __ktime_divns(const ktime_t kt, s64 div)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
|
|
|
int sft = 0;
|
2015-05-09 03:47:23 +07:00
|
|
|
s64 dclc;
|
|
|
|
u64 tmp;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2008-05-23 05:25:11 +07:00
|
|
|
dclc = ktime_to_ns(kt);
|
2015-05-09 03:47:23 +07:00
|
|
|
tmp = dclc < 0 ? -dclc : dclc;
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/* Make sure the divisor is less than 2^32: */
|
|
|
|
while (div >> 32) {
|
|
|
|
sft++;
|
|
|
|
div >>= 1;
|
|
|
|
}
|
2015-05-09 03:47:23 +07:00
|
|
|
tmp >>= sft;
|
|
|
|
do_div(tmp, (unsigned long) div);
|
|
|
|
return dclc < 0 ? -tmp : tmp;
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
2014-12-04 02:43:06 +07:00
|
|
|
EXPORT_SYMBOL_GPL(__ktime_divns);
|
2006-01-10 11:52:32 +07:00
|
|
|
#endif /* BITS_PER_LONG >= 64 */
|
|
|
|
|
2008-02-13 15:20:43 +07:00
|
|
|
/*
|
|
|
|
* Add two ktime values and do a safety check for overflow:
|
|
|
|
*/
|
|
|
|
ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
|
|
|
|
{
|
2016-08-13 06:37:04 +07:00
|
|
|
ktime_t res = ktime_add_unsafe(lhs, rhs);
|
2008-02-13 15:20:43 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We use KTIME_SEC_MAX here, the maximum timeout which we can
|
|
|
|
* return to user space in a timespec:
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (res < 0 || res < lhs || res < rhs)
|
2008-02-13 15:20:43 +07:00
|
|
|
res = ktime_set(KTIME_SEC_MAX, 0);
|
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2009-05-28 20:21:24 +07:00
|
|
|
EXPORT_SYMBOL_GPL(ktime_add_safe);
|
|
|
|
|
2008-04-30 14:55:04 +07:00
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
|
|
|
|
|
|
|
|
static struct debug_obj_descr hrtimer_debug_descr;
|
|
|
|
|
2011-03-07 15:58:33 +07:00
|
|
|
static void *hrtimer_debug_hint(void *addr)
|
|
|
|
{
|
|
|
|
return ((struct hrtimer *) addr)->function;
|
|
|
|
}
|
|
|
|
|
2008-04-30 14:55:04 +07:00
|
|
|
/*
|
|
|
|
* fixup_init is called when:
|
|
|
|
* - an active object is initialized
|
|
|
|
*/
|
2016-05-20 07:09:29 +07:00
|
|
|
static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
|
2008-04-30 14:55:04 +07:00
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
2016-05-20 07:09:29 +07:00
|
|
|
return true;
|
2008-04-30 14:55:04 +07:00
|
|
|
default:
|
2016-05-20 07:09:29 +07:00
|
|
|
return false;
|
2008-04-30 14:55:04 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_activate is called when:
|
|
|
|
* - an active object is activated
|
2016-05-20 07:09:41 +07:00
|
|
|
* - an unknown non-static object is activated
|
2008-04-30 14:55:04 +07:00
|
|
|
*/
|
2016-05-20 07:09:29 +07:00
|
|
|
static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
|
2008-04-30 14:55:04 +07:00
|
|
|
{
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
WARN_ON(1);
|
|
|
|
|
|
|
|
default:
|
2016-05-20 07:09:29 +07:00
|
|
|
return false;
|
2008-04-30 14:55:04 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_free is called when:
|
|
|
|
* - an active object is freed
|
|
|
|
*/
|
2016-05-20 07:09:29 +07:00
|
|
|
static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
|
2008-04-30 14:55:04 +07:00
|
|
|
{
|
|
|
|
struct hrtimer *timer = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
hrtimer_cancel(timer);
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
2016-05-20 07:09:29 +07:00
|
|
|
return true;
|
2008-04-30 14:55:04 +07:00
|
|
|
default:
|
2016-05-20 07:09:29 +07:00
|
|
|
return false;
|
2008-04-30 14:55:04 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct debug_obj_descr hrtimer_debug_descr = {
|
|
|
|
.name = "hrtimer",
|
2011-03-07 15:58:33 +07:00
|
|
|
.debug_hint = hrtimer_debug_hint,
|
2008-04-30 14:55:04 +07:00
|
|
|
.fixup_init = hrtimer_fixup_init,
|
|
|
|
.fixup_activate = hrtimer_fixup_activate,
|
|
|
|
.fixup_free = hrtimer_fixup_free,
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_init(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_activate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_deactivate(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_hrtimer_free(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode);
|
|
|
|
|
|
|
|
void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_object_init_on_stack(timer, &hrtimer_debug_descr);
|
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
2009-08-29 13:41:29 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
|
2008-04-30 14:55:04 +07:00
|
|
|
|
|
|
|
void destroy_hrtimer_on_stack(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_object_free(timer, &hrtimer_debug_descr);
|
|
|
|
}
|
2016-05-27 07:21:05 +07:00
|
|
|
EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
|
2008-04-30 14:55:04 +07:00
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void debug_hrtimer_init(struct hrtimer *timer) { }
|
|
|
|
static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
|
|
|
|
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
|
|
|
|
#endif
|
|
|
|
|
2009-08-10 09:51:23 +07:00
|
|
|
static inline void
|
|
|
|
debug_init(struct hrtimer *timer, clockid_t clockid,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
debug_hrtimer_init(timer);
|
|
|
|
trace_hrtimer_init(timer, clockid, mode);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_activate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_hrtimer_activate(timer);
|
|
|
|
trace_hrtimer_start(timer);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_deactivate(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
debug_hrtimer_deactivate(timer);
|
|
|
|
trace_hrtimer_cancel(timer);
|
|
|
|
}
|
|
|
|
|
2015-01-21 03:24:10 +07:00
|
|
|
#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
|
2015-04-15 04:08:49 +07:00
|
|
|
static inline void hrtimer_update_next_timer(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
cpu_base->next_timer = timer;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2015-01-23 19:12:06 +07:00
|
|
|
static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base)
|
2015-01-21 03:24:10 +07:00
|
|
|
{
|
|
|
|
struct hrtimer_clock_base *base = cpu_base->clock_base;
|
2015-04-15 04:08:41 +07:00
|
|
|
unsigned int active = cpu_base->active_bases;
|
2016-12-25 17:38:40 +07:00
|
|
|
ktime_t expires, expires_next = KTIME_MAX;
|
2015-01-21 03:24:10 +07:00
|
|
|
|
2015-04-15 04:08:49 +07:00
|
|
|
hrtimer_update_next_timer(cpu_base, NULL);
|
2015-04-15 04:08:41 +07:00
|
|
|
for (; active; base++, active >>= 1) {
|
2015-01-21 03:24:10 +07:00
|
|
|
struct timerqueue_node *next;
|
|
|
|
struct hrtimer *timer;
|
|
|
|
|
2015-04-15 04:08:41 +07:00
|
|
|
if (!(active & 0x01))
|
2015-01-21 03:24:10 +07:00
|
|
|
continue;
|
|
|
|
|
2015-04-15 04:08:41 +07:00
|
|
|
next = timerqueue_getnext(&base->active);
|
2015-01-21 03:24:10 +07:00
|
|
|
timer = container_of(next, struct hrtimer, node);
|
|
|
|
expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
2016-12-25 17:38:40 +07:00
|
|
|
if (expires < expires_next) {
|
2015-01-21 03:24:10 +07:00
|
|
|
expires_next = expires;
|
2015-04-15 04:08:49 +07:00
|
|
|
hrtimer_update_next_timer(cpu_base, timer);
|
|
|
|
}
|
2015-01-21 03:24:10 +07:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* clock_was_set() might have changed base->offset of any of
|
|
|
|
* the clock bases so the result might be negative. Fix it up
|
|
|
|
* to prevent a false positive in clockevents_program_event().
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (expires_next < 0)
|
|
|
|
expires_next = 0;
|
2015-01-21 03:24:10 +07:00
|
|
|
return expires_next;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2015-04-15 04:08:35 +07:00
|
|
|
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
|
|
|
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
|
|
|
|
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
|
|
|
|
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
|
|
|
|
|
2015-04-15 04:08:37 +07:00
|
|
|
return ktime_get_update_offsets_now(&base->clock_was_set_seq,
|
|
|
|
offs_real, offs_boot, offs_tai);
|
2015-04-15 04:08:35 +07:00
|
|
|
}
|
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
/* High resolution timer related functions */
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer enabled ?
|
|
|
|
*/
|
2016-03-18 04:23:00 +07:00
|
|
|
static bool hrtimer_hres_enabled __read_mostly = true;
|
2015-04-15 04:08:27 +07:00
|
|
|
unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
|
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_resolution);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable / Disable high resolution mode
|
|
|
|
*/
|
|
|
|
static int __init setup_hrtimer_hres(char *str)
|
|
|
|
{
|
2016-03-18 04:23:00 +07:00
|
|
|
return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
__setup("highres=", setup_hrtimer_hres);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* hrtimer_high_res_enabled - query, if the highres mode is enabled
|
|
|
|
*/
|
|
|
|
static inline int hrtimer_is_hres_enabled(void)
|
|
|
|
{
|
|
|
|
return hrtimer_hres_enabled;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Is the high resolution mode active ?
|
|
|
|
*/
|
2015-04-15 04:08:39 +07:00
|
|
|
static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
|
|
|
|
{
|
|
|
|
return cpu_base->hres_active;
|
|
|
|
}
|
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
static inline int hrtimer_hres_active(void)
|
|
|
|
{
|
2015-04-15 04:08:39 +07:00
|
|
|
return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reprogram the event source with checking both queues for the
|
|
|
|
* next event
|
|
|
|
* Called with interrupts disabled and base->lock held
|
|
|
|
*/
|
2009-09-02 10:03:33 +07:00
|
|
|
static void
|
|
|
|
hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
|
2007-02-16 16:28:11 +07:00
|
|
|
{
|
2015-04-15 04:08:35 +07:00
|
|
|
ktime_t expires_next;
|
|
|
|
|
|
|
|
if (!cpu_base->hres_active)
|
|
|
|
return;
|
|
|
|
|
|
|
|
expires_next = __hrtimer_get_next_event(cpu_base);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
if (skip_equal && expires_next == cpu_base->expires_next)
|
2009-09-02 10:03:33 +07:00
|
|
|
return;
|
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
cpu_base->expires_next = expires_next;
|
2009-09-02 10:03:33 +07:00
|
|
|
|
2014-04-30 05:55:02 +07:00
|
|
|
/*
|
|
|
|
* If a hang was detected in the last timer interrupt then we
|
|
|
|
* leave the hang delay active in the hardware. We want the
|
|
|
|
* system to make progress. That also prevents the following
|
|
|
|
* scenario:
|
|
|
|
* T1 expires 50ms from now
|
|
|
|
* T2 expires 5s from now
|
|
|
|
*
|
|
|
|
* T1 is removed, so this code is called and would reprogram
|
|
|
|
* the hardware to 5s from now. Any hrtimer_start after that
|
|
|
|
* will not reprogram the hardware due to hang_detected being
|
|
|
|
* set. So we'd effectivly block all timers until the T2 event
|
|
|
|
* fires.
|
|
|
|
*/
|
|
|
|
if (cpu_base->hang_detected)
|
|
|
|
return;
|
|
|
|
|
2015-04-03 10:34:05 +07:00
|
|
|
tick_program_event(cpu_base->expires_next, 1);
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When a timer is enqueued and expires earlier than the already enqueued
|
|
|
|
* timers, we have to check, whether it expires earlier than the timer for
|
|
|
|
* which the clock event device was armed.
|
|
|
|
*
|
|
|
|
* Called with interrupts disabled and base->cpu_base.lock held
|
|
|
|
*/
|
2015-04-15 04:08:51 +07:00
|
|
|
static void hrtimer_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
2007-02-16 16:28:11 +07:00
|
|
|
{
|
2014-08-18 00:30:26 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2008-09-02 05:02:30 +07:00
|
|
|
ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2008-09-02 05:02:30 +07:00
|
|
|
WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
|
2008-02-14 06:58:36 +07:00
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
/*
|
2015-04-15 04:08:51 +07:00
|
|
|
* If the timer is not on the current cpu, we cannot reprogram
|
|
|
|
* the other cpus clock event device.
|
2007-02-16 16:28:11 +07:00
|
|
|
*/
|
2015-04-15 04:08:51 +07:00
|
|
|
if (base->cpu_base != cpu_base)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the hrtimer interrupt is running, then it will
|
|
|
|
* reevaluate the clock bases and reprogram the clock event
|
|
|
|
* device. The callbacks are always executed in hard interrupt
|
|
|
|
* context so we don't need an extra check for a running
|
|
|
|
* callback.
|
|
|
|
*/
|
|
|
|
if (cpu_base->in_hrtirq)
|
|
|
|
return;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2008-02-14 06:58:36 +07:00
|
|
|
/*
|
|
|
|
* CLOCK_REALTIME timer might be requested with an absolute
|
2015-04-15 04:08:51 +07:00
|
|
|
* expiry time which is less than base->offset. Set it to 0.
|
2008-02-14 06:58:36 +07:00
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (expires < 0)
|
|
|
|
expires = 0;
|
2008-02-14 06:58:36 +07:00
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
if (expires >= cpu_base->expires_next)
|
2015-04-15 04:08:51 +07:00
|
|
|
return;
|
2009-11-13 23:05:44 +07:00
|
|
|
|
2015-04-15 04:08:51 +07:00
|
|
|
/* Update the pointer to the next expiring timer */
|
2015-04-15 04:08:49 +07:00
|
|
|
cpu_base->next_timer = timer;
|
2015-01-21 03:24:10 +07:00
|
|
|
|
2009-11-13 23:05:44 +07:00
|
|
|
/*
|
|
|
|
* If a hang was detected in the last timer interrupt then we
|
|
|
|
* do not schedule a timer which is earlier than the expiry
|
|
|
|
* which we enforced in the hang detection. We want the system
|
|
|
|
* to make progress.
|
|
|
|
*/
|
|
|
|
if (cpu_base->hang_detected)
|
2015-04-15 04:08:51 +07:00
|
|
|
return;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
/*
|
2015-04-15 04:08:51 +07:00
|
|
|
* Program the timer hardware. We enforce the expiry for
|
|
|
|
* events which are already in the past.
|
2007-02-16 16:28:11 +07:00
|
|
|
*/
|
2015-04-15 04:08:51 +07:00
|
|
|
cpu_base->expires_next = expires;
|
|
|
|
tick_program_event(expires, 1);
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Initialize the high resolution related parts of cpu_base
|
|
|
|
*/
|
|
|
|
static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
|
|
|
|
{
|
2016-12-25 17:38:40 +07:00
|
|
|
base->expires_next = KTIME_MAX;
|
2007-02-16 16:28:11 +07:00
|
|
|
base->hres_active = 0;
|
|
|
|
}
|
|
|
|
|
2011-05-20 21:18:50 +07:00
|
|
|
/*
|
|
|
|
* Retrigger next event is called after clock was set
|
|
|
|
*
|
|
|
|
* Called with interrupts disabled via on_each_cpu()
|
|
|
|
*/
|
|
|
|
static void retrigger_next_event(void *arg)
|
|
|
|
{
|
2014-08-18 00:30:26 +07:00
|
|
|
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
|
2011-05-20 21:18:50 +07:00
|
|
|
|
2015-04-15 04:08:39 +07:00
|
|
|
if (!base->hres_active)
|
2011-05-20 21:18:50 +07:00
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock(&base->lock);
|
2012-07-11 05:43:25 +07:00
|
|
|
hrtimer_update_base(base);
|
2011-05-20 21:18:50 +07:00
|
|
|
hrtimer_force_reprogram(base, 0);
|
|
|
|
raw_spin_unlock(&base->lock);
|
|
|
|
}
|
2011-05-02 21:48:57 +07:00
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
/*
|
|
|
|
* Switch to high resolution mode
|
|
|
|
*/
|
2015-08-12 03:40:43 +07:00
|
|
|
static void hrtimer_switch_to_hres(void)
|
2007-02-16 16:28:11 +07:00
|
|
|
{
|
2015-04-15 04:08:51 +07:00
|
|
|
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
if (tick_init_highres()) {
|
2007-07-21 18:37:36 +07:00
|
|
|
printk(KERN_WARNING "Could not switch to high resolution "
|
2015-04-15 04:08:51 +07:00
|
|
|
"mode on CPU %d\n", base->cpu);
|
2015-08-22 15:10:47 +07:00
|
|
|
return;
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
base->hres_active = 1;
|
2015-04-15 04:08:27 +07:00
|
|
|
hrtimer_resolution = HIGH_RES_NSEC;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
tick_setup_sched_timer();
|
|
|
|
/* "Retrigger" the interrupt to get things going */
|
|
|
|
retrigger_next_event(NULL);
|
|
|
|
}
|
|
|
|
|
2013-07-05 17:09:18 +07:00
|
|
|
static void clock_was_set_work(struct work_struct *work)
|
|
|
|
{
|
|
|
|
clock_was_set();
|
|
|
|
}
|
|
|
|
|
|
|
|
static DECLARE_WORK(hrtimer_work, clock_was_set_work);
|
|
|
|
|
2012-07-11 05:43:19 +07:00
|
|
|
/*
|
2016-06-24 01:50:37 +07:00
|
|
|
* Called from timekeeping and resume code to reprogram the hrtimer
|
2013-07-05 17:09:18 +07:00
|
|
|
* interrupt device on all cpus.
|
2012-07-11 05:43:19 +07:00
|
|
|
*/
|
|
|
|
void clock_was_set_delayed(void)
|
|
|
|
{
|
2013-07-05 17:09:18 +07:00
|
|
|
schedule_work(&hrtimer_work);
|
2012-07-11 05:43:19 +07:00
|
|
|
}
|
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
#else
|
|
|
|
|
2015-04-15 04:08:39 +07:00
|
|
|
static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *b) { return 0; }
|
2007-02-16 16:28:11 +07:00
|
|
|
static inline int hrtimer_hres_active(void) { return 0; }
|
|
|
|
static inline int hrtimer_is_hres_enabled(void) { return 0; }
|
2015-08-12 03:40:43 +07:00
|
|
|
static inline void hrtimer_switch_to_hres(void) { }
|
2009-09-02 10:03:33 +07:00
|
|
|
static inline void
|
|
|
|
hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
|
2014-06-22 06:29:17 +07:00
|
|
|
static inline int hrtimer_reprogram(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
2007-02-16 16:28:11 +07:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
|
2011-05-20 21:18:50 +07:00
|
|
|
static inline void retrigger_next_event(void *arg) { }
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
#endif /* CONFIG_HIGH_RES_TIMERS */
|
|
|
|
|
2011-05-02 21:48:57 +07:00
|
|
|
/*
|
|
|
|
* Clock realtime was set
|
|
|
|
*
|
|
|
|
* Change the offset of the realtime clock vs. the monotonic
|
|
|
|
* clock.
|
|
|
|
*
|
|
|
|
* We might have to reprogram the high resolution timer interrupt. On
|
|
|
|
* SMP we call the architecture specific code to retrigger _all_ high
|
|
|
|
* resolution timer interrupts. On UP we just disable interrupts and
|
|
|
|
* call the high resolution interrupt code.
|
|
|
|
*/
|
|
|
|
void clock_was_set(void)
|
|
|
|
{
|
2011-05-26 04:08:17 +07:00
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
2011-05-02 21:48:57 +07:00
|
|
|
/* Retrigger the CPU local events everywhere */
|
|
|
|
on_each_cpu(retrigger_next_event, NULL, 1);
|
2011-05-20 21:18:50 +07:00
|
|
|
#endif
|
|
|
|
timerfd_clock_was_set();
|
2011-05-02 21:48:57 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* During resume we might have to reprogram the high resolution timer
|
2013-06-27 17:35:44 +07:00
|
|
|
* interrupt on all online CPUs. However, all other CPUs will be
|
|
|
|
* stopped with IRQs interrupts disabled so the clock_was_set() call
|
2013-07-05 17:09:18 +07:00
|
|
|
* must be deferred.
|
2011-05-02 21:48:57 +07:00
|
|
|
*/
|
|
|
|
void hrtimers_resume(void)
|
|
|
|
{
|
|
|
|
WARN_ONCE(!irqs_disabled(),
|
|
|
|
KERN_INFO "hrtimers_resume() called with IRQs enabled!");
|
|
|
|
|
2013-07-05 17:09:18 +07:00
|
|
|
/* Retrigger on the local CPU */
|
2011-05-02 21:48:57 +07:00
|
|
|
retrigger_next_event(NULL);
|
2013-07-05 17:09:18 +07:00
|
|
|
/* And schedule a retrigger for all others */
|
|
|
|
clock_was_set_delayed();
|
2011-05-02 21:48:57 +07:00
|
|
|
}
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
2007-10-20 06:56:53 +07:00
|
|
|
* Counterpart to lock_hrtimer_base above:
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
|
|
|
static inline
|
|
|
|
void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
|
|
|
{
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_forward - forward the timer expiry
|
|
|
|
* @timer: hrtimer to forward
|
2006-03-26 16:38:06 +07:00
|
|
|
* @now: forward past this time
|
2006-01-10 11:52:32 +07:00
|
|
|
* @interval: the interval to forward
|
|
|
|
*
|
|
|
|
* Forward the timer expiry so it will expire in the future.
|
2006-01-17 05:58:55 +07:00
|
|
|
* Returns the number of overruns.
|
2015-04-14 04:02:22 +07:00
|
|
|
*
|
|
|
|
* Can be safely called from the callback function of @timer. If
|
|
|
|
* called from other contexts @timer must neither be enqueued nor
|
|
|
|
* running the callback and the caller needs to take care of
|
|
|
|
* serialization.
|
|
|
|
*
|
|
|
|
* Note: This only updates the timer expiry value and does not requeue
|
|
|
|
* the timer.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 13:27:26 +07:00
|
|
|
u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
timerfd: new timerfd API
This is the new timerfd API as it is implemented by the following patch:
int timerfd_create(int clockid, int flags);
int timerfd_settime(int ufd, int flags,
const struct itimerspec *utmr,
struct itimerspec *otmr);
int timerfd_gettime(int ufd, struct itimerspec *otmr);
The timerfd_create() API creates an un-programmed timerfd fd. The "clockid"
parameter can be either CLOCK_MONOTONIC or CLOCK_REALTIME.
The timerfd_settime() API give new settings by the timerfd fd, by optionally
retrieving the previous expiration time (in case the "otmr" parameter is not
NULL).
The time value specified in "utmr" is absolute, if the TFD_TIMER_ABSTIME bit
is set in the "flags" parameter. Otherwise it's a relative time.
The timerfd_gettime() API returns the next expiration time of the timer, or
{0, 0} if the timerfd has not been set yet.
Like the previous timerfd API implementation, read(2) and poll(2) are
supported (with the same interface). Here's a simple test program I used to
exercise the new timerfd APIs:
http://www.xmailserver.org/timerfd-test2.c
[akpm@linux-foundation.org: coding-style cleanups]
[akpm@linux-foundation.org: fix ia64 build]
[akpm@linux-foundation.org: fix m68k build]
[akpm@linux-foundation.org: fix mips build]
[akpm@linux-foundation.org: fix alpha, arm, blackfin, cris, m68k, s390, sparc and sparc64 builds]
[heiko.carstens@de.ibm.com: fix s390]
[akpm@linux-foundation.org: fix powerpc build]
[akpm@linux-foundation.org: fix sparc64 more]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Davide Libenzi <davidel@xmailserver.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 13:27:26 +07:00
|
|
|
u64 orun = 1;
|
2006-03-26 16:38:06 +07:00
|
|
|
ktime_t delta;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2008-09-02 05:02:30 +07:00
|
|
|
delta = ktime_sub(now, hrtimer_get_expires(timer));
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
if (delta < 0)
|
2006-01-10 11:52:32 +07:00
|
|
|
return 0;
|
|
|
|
|
2014-05-20 20:49:48 +07:00
|
|
|
if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
|
|
|
|
return 0;
|
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
if (interval < hrtimer_resolution)
|
|
|
|
interval = hrtimer_resolution;
|
2006-01-12 17:47:34 +07:00
|
|
|
|
2016-12-25 17:38:40 +07:00
|
|
|
if (unlikely(delta >= interval)) {
|
2006-03-26 16:38:11 +07:00
|
|
|
s64 incr = ktime_to_ns(interval);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
orun = ktime_divns(delta, incr);
|
2008-09-02 05:02:30 +07:00
|
|
|
hrtimer_add_expires_ns(timer, incr * orun);
|
2016-12-25 17:38:40 +07:00
|
|
|
if (hrtimer_get_expires_tv64(timer) > now)
|
2006-01-10 11:52:32 +07:00
|
|
|
return orun;
|
|
|
|
/*
|
|
|
|
* This (and the ktime_add() below) is the
|
|
|
|
* correction for exact:
|
|
|
|
*/
|
|
|
|
orun++;
|
|
|
|
}
|
2008-09-02 05:02:30 +07:00
|
|
|
hrtimer_add_expires(timer, interval);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
return orun;
|
|
|
|
}
|
2007-05-08 14:31:58 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_forward);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* enqueue_hrtimer - internal function to (re)start a timer
|
|
|
|
*
|
|
|
|
* The timer is inserted in expiry order. Insertion into the
|
|
|
|
* red black tree is O(log(n)). Must hold the base lock.
|
2009-01-05 17:28:22 +07:00
|
|
|
*
|
|
|
|
* Returns 1 when the new timer is the leftmost timer in the tree.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2009-01-05 17:28:22 +07:00
|
|
|
static int enqueue_hrtimer(struct hrtimer *timer,
|
|
|
|
struct hrtimer_clock_base *base)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2009-08-10 09:51:23 +07:00
|
|
|
debug_activate(timer);
|
2008-04-30 14:55:04 +07:00
|
|
|
|
2011-05-20 18:05:15 +07:00
|
|
|
base->cpu_base->active_bases |= 1 << base->index;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
timer->state = HRTIMER_STATE_ENQUEUED;
|
2009-01-05 17:28:22 +07:00
|
|
|
|
2015-04-15 04:08:47 +07:00
|
|
|
return timerqueue_add(&base->active, &timer->node);
|
2006-01-12 17:25:54 +07:00
|
|
|
}
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* __remove_hrtimer - internal function to remove a timer
|
|
|
|
*
|
|
|
|
* Caller must hold the base lock.
|
2007-02-16 16:28:11 +07:00
|
|
|
*
|
|
|
|
* High resolution timer mode reprograms the clock event device when the
|
|
|
|
* timer is the one which expires next. The caller can disable this by setting
|
|
|
|
* reprogram to zero. This is useful, when the context does a reprogramming
|
|
|
|
* anyway (e.g. timer interrupt)
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2007-02-16 16:27:50 +07:00
|
|
|
static void __remove_hrtimer(struct hrtimer *timer,
|
2007-02-16 16:27:51 +07:00
|
|
|
struct hrtimer_clock_base *base,
|
2016-01-14 23:54:46 +07:00
|
|
|
u8 newstate, int reprogram)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2015-04-15 04:08:39 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
|
2016-01-14 23:54:46 +07:00
|
|
|
u8 state = timer->state;
|
2015-04-15 04:08:39 +07:00
|
|
|
|
2015-04-15 04:08:49 +07:00
|
|
|
timer->state = newstate;
|
|
|
|
if (!(state & HRTIMER_STATE_ENQUEUED))
|
|
|
|
return;
|
2009-09-02 10:03:33 +07:00
|
|
|
|
2015-04-15 04:08:47 +07:00
|
|
|
if (!timerqueue_del(&base->active, &timer->node))
|
2015-04-15 04:08:39 +07:00
|
|
|
cpu_base->active_bases &= ~(1 << base->index);
|
2009-09-02 10:03:33 +07:00
|
|
|
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
2015-04-15 04:08:49 +07:00
|
|
|
/*
|
|
|
|
* Note: If reprogram is false we do not update
|
|
|
|
* cpu_base->next_timer. This happens when we remove the first
|
|
|
|
* timer on a remote cpu. No harm as we never dereference
|
|
|
|
* cpu_base->next_timer. So the worst thing what can happen is
|
|
|
|
* an superflous call to hrtimer_force_reprogram() on the
|
|
|
|
* remote cpu later on if the same timer gets enqueued again.
|
|
|
|
*/
|
|
|
|
if (reprogram && timer == cpu_base->next_timer)
|
|
|
|
hrtimer_force_reprogram(cpu_base, 1);
|
2009-09-02 10:03:33 +07:00
|
|
|
#endif
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* remove hrtimer, called with base lock held
|
|
|
|
*/
|
|
|
|
static inline int
|
2015-06-11 19:46:45 +07:00
|
|
|
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:51 +07:00
|
|
|
if (hrtimer_is_queued(timer)) {
|
2016-01-14 23:54:46 +07:00
|
|
|
u8 state = timer->state;
|
2007-02-16 16:28:11 +07:00
|
|
|
int reprogram;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove the timer and force reprogramming when high
|
|
|
|
* resolution mode is active and the timer is on the current
|
|
|
|
* CPU. If we remove a timer on another CPU, reprogramming is
|
|
|
|
* skipped. The interrupt event on this CPU is fired and
|
|
|
|
* reprogramming happens in the interrupt handler. This is a
|
|
|
|
* rare case and less expensive than a smp call.
|
|
|
|
*/
|
2009-08-10 09:51:23 +07:00
|
|
|
debug_deactivate(timer);
|
2014-08-18 00:30:26 +07:00
|
|
|
reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
|
2015-06-11 19:46:45 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
if (!restart)
|
|
|
|
state = HRTIMER_STATE_INACTIVE;
|
|
|
|
|
2010-10-12 21:25:19 +07:00
|
|
|
__remove_hrtimer(timer, base, state, reprogram);
|
2006-01-10 11:52:32 +07:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-01-14 23:54:46 +07:00
|
|
|
static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
|
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_TIME_LOW_RES
|
|
|
|
/*
|
|
|
|
* CONFIG_TIME_LOW_RES indicates that the system has no way to return
|
|
|
|
* granular time values. For relative timers we add hrtimer_resolution
|
|
|
|
* (i.e. one jiffie) to prevent short timeouts.
|
|
|
|
*/
|
|
|
|
timer->is_rel = mode & HRTIMER_MODE_REL;
|
|
|
|
if (timer->is_rel)
|
2016-12-25 18:30:41 +07:00
|
|
|
tim = ktime_add_safe(tim, hrtimer_resolution);
|
2016-01-14 23:54:46 +07:00
|
|
|
#endif
|
|
|
|
return tim;
|
|
|
|
}
|
|
|
|
|
2015-04-15 04:09:08 +07:00
|
|
|
/**
|
|
|
|
* hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
|
|
|
|
* @timer: the timer to be added
|
|
|
|
* @tim: expiry time
|
|
|
|
* @delta_ns: "slack" range for the timer
|
|
|
|
* @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or
|
|
|
|
* relative (HRTIMER_MODE_REL)
|
|
|
|
*/
|
2015-04-15 04:09:23 +07:00
|
|
|
void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 04:20:51 +07:00
|
|
|
u64 delta_ns, const enum hrtimer_mode mode)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_clock_base *base, *new_base;
|
2006-01-10 11:52:32 +07:00
|
|
|
unsigned long flags;
|
2015-04-15 04:09:23 +07:00
|
|
|
int leftmost;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
/* Remove an active timer from the queue: */
|
2015-06-11 19:46:45 +07:00
|
|
|
remove_hrtimer(timer, base, true);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2016-01-14 23:54:46 +07:00
|
|
|
if (mode & HRTIMER_MODE_REL)
|
2014-05-12 15:12:29 +07:00
|
|
|
tim = ktime_add_safe(tim, base->get_time());
|
2016-01-14 23:54:46 +07:00
|
|
|
|
|
|
|
tim = hrtimer_update_lowres(timer, tim, mode);
|
2008-04-30 14:55:04 +07:00
|
|
|
|
2008-09-08 00:47:46 +07:00
|
|
|
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2014-05-12 15:12:29 +07:00
|
|
|
/* Switch the timer base, if necessary: */
|
|
|
|
new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
|
|
|
|
|
2009-01-05 17:28:22 +07:00
|
|
|
leftmost = enqueue_hrtimer(timer, new_base);
|
2015-04-15 04:09:23 +07:00
|
|
|
if (!leftmost)
|
|
|
|
goto unlock;
|
2014-06-23 15:09:37 +07:00
|
|
|
|
|
|
|
if (!hrtimer_is_hres_active(timer)) {
|
|
|
|
/*
|
|
|
|
* Kick to reschedule the next tick to handle the new timer
|
|
|
|
* on dynticks target.
|
|
|
|
*/
|
2015-05-27 05:50:35 +07:00
|
|
|
if (new_base->cpu_base->nohz_active)
|
|
|
|
wake_up_nohz_cpu(new_base->cpu_base->cpu);
|
2015-04-15 04:08:51 +07:00
|
|
|
} else {
|
|
|
|
hrtimer_reprogram(timer, new_base);
|
2013-02-04 19:33:37 +07:00
|
|
|
}
|
2015-04-15 04:09:23 +07:00
|
|
|
unlock:
|
2006-01-10 11:52:32 +07:00
|
|
|
unlock_hrtimer_base(timer, &flags);
|
2009-03-13 18:21:27 +07:00
|
|
|
}
|
2008-09-08 00:47:46 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/**
|
|
|
|
* hrtimer_try_to_cancel - try to deactivate a timer
|
|
|
|
* @timer: hrtimer to stop
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 when the timer was not active
|
|
|
|
* 1 when the timer was active
|
2017-03-07 18:48:02 +07:00
|
|
|
* -1 when the timer is currently executing the callback function and
|
2006-06-25 19:49:15 +07:00
|
|
|
* cannot be stopped
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
|
|
|
int hrtimer_try_to_cancel(struct hrtimer *timer)
|
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_clock_base *base;
|
2006-01-10 11:52:32 +07:00
|
|
|
unsigned long flags;
|
|
|
|
int ret = -1;
|
|
|
|
|
2015-04-15 04:09:25 +07:00
|
|
|
/*
|
|
|
|
* Check lockless first. If the timer is not active (neither
|
|
|
|
* enqueued nor running the callback, nothing to do here. The
|
|
|
|
* base lock does not serialize against a concurrent enqueue,
|
|
|
|
* so we can avoid taking it.
|
|
|
|
*/
|
|
|
|
if (!hrtimer_active(timer))
|
|
|
|
return 0;
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
base = lock_hrtimer_base(timer, &flags);
|
|
|
|
|
2007-02-16 16:27:51 +07:00
|
|
|
if (!hrtimer_callback_running(timer))
|
2015-06-11 19:46:45 +07:00
|
|
|
ret = remove_hrtimer(timer, base, false);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
}
|
2006-05-31 11:26:09 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
|
|
|
|
* @timer: the timer to be cancelled
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 when the timer was not active
|
|
|
|
* 1 when the timer was active
|
|
|
|
*/
|
|
|
|
int hrtimer_cancel(struct hrtimer *timer)
|
|
|
|
{
|
|
|
|
for (;;) {
|
|
|
|
int ret = hrtimer_try_to_cancel(timer);
|
|
|
|
|
|
|
|
if (ret >= 0)
|
|
|
|
return ret;
|
2006-04-11 12:54:13 +07:00
|
|
|
cpu_relax();
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
}
|
2006-05-31 11:26:09 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_cancel);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_get_remaining - get remaining time for the timer
|
|
|
|
* @timer: the timer to read
|
2016-01-14 23:54:46 +07:00
|
|
|
* @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2016-01-14 23:54:46 +07:00
|
|
|
ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
ktime_t rem;
|
|
|
|
|
2010-08-11 04:17:51 +07:00
|
|
|
lock_hrtimer_base(timer, &flags);
|
2016-01-14 23:54:46 +07:00
|
|
|
if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
|
|
|
|
rem = hrtimer_expires_remaining_adjusted(timer);
|
|
|
|
else
|
|
|
|
rem = hrtimer_expires_remaining(timer);
|
2006-01-10 11:52:32 +07:00
|
|
|
unlock_hrtimer_base(timer, &flags);
|
|
|
|
|
|
|
|
return rem;
|
|
|
|
}
|
2016-01-14 23:54:46 +07:00
|
|
|
EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2011-08-11 04:21:01 +07:00
|
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
2006-03-07 06:42:45 +07:00
|
|
|
/**
|
|
|
|
* hrtimer_get_next_event - get the time until next expiry event
|
|
|
|
*
|
2015-04-15 04:08:58 +07:00
|
|
|
* Returns the next expiry time or KTIME_MAX if no timer is pending.
|
2006-03-07 06:42:45 +07:00
|
|
|
*/
|
2015-04-15 04:08:58 +07:00
|
|
|
u64 hrtimer_get_next_event(void)
|
2006-03-07 06:42:45 +07:00
|
|
|
{
|
2014-08-18 00:30:26 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2015-04-15 04:08:58 +07:00
|
|
|
u64 expires = KTIME_MAX;
|
2006-03-07 06:42:45 +07:00
|
|
|
unsigned long flags;
|
|
|
|
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
2007-02-16 16:27:50 +07:00
|
|
|
|
2015-04-15 04:08:39 +07:00
|
|
|
if (!__hrtimer_hres_active(cpu_base))
|
2016-12-25 17:38:40 +07:00
|
|
|
expires = __hrtimer_get_next_event(cpu_base);
|
2007-02-16 16:27:50 +07:00
|
|
|
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
2007-02-16 16:27:50 +07:00
|
|
|
|
2015-04-15 04:08:58 +07:00
|
|
|
return expires;
|
2006-03-07 06:42:45 +07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-01-16 00:41:09 +07:00
|
|
|
static inline int hrtimer_clockid_to_base(clockid_t clock_id)
|
|
|
|
{
|
|
|
|
if (likely(clock_id < MAX_CLOCKS)) {
|
|
|
|
int base = hrtimer_clock_to_base_table[clock_id];
|
|
|
|
|
|
|
|
if (likely(base != HRTIMER_MAX_CLOCK_BASES))
|
|
|
|
return base;
|
|
|
|
}
|
|
|
|
WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
|
|
|
|
return HRTIMER_BASE_MONOTONIC;
|
|
|
|
}
|
|
|
|
|
2008-04-30 14:55:04 +07:00
|
|
|
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base;
|
2010-12-15 10:37:07 +07:00
|
|
|
int base;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2006-02-01 18:05:11 +07:00
|
|
|
memset(timer, 0, sizeof(struct hrtimer));
|
|
|
|
|
2014-08-18 00:30:25 +07:00
|
|
|
cpu_base = raw_cpu_ptr(&hrtimer_bases);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2007-02-16 16:27:49 +07:00
|
|
|
if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
|
2006-02-01 18:05:11 +07:00
|
|
|
clock_id = CLOCK_MONOTONIC;
|
|
|
|
|
2010-12-15 10:37:07 +07:00
|
|
|
base = hrtimer_clockid_to_base(clock_id);
|
|
|
|
timer->base = &cpu_base->clock_base[base];
|
2010-09-21 09:19:17 +07:00
|
|
|
timerqueue_init(&timer->node);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
2008-04-30 14:55:04 +07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* hrtimer_init - initialize a timer to the given clock
|
|
|
|
* @timer: the timer to be initialized
|
|
|
|
* @clock_id: the clock to be used
|
|
|
|
* @mode: timer mode abs/rel
|
|
|
|
*/
|
|
|
|
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
|
|
|
enum hrtimer_mode mode)
|
|
|
|
{
|
2009-08-10 09:51:23 +07:00
|
|
|
debug_init(timer, clock_id, mode);
|
2008-04-30 14:55:04 +07:00
|
|
|
__hrtimer_init(timer, clock_id, mode);
|
|
|
|
}
|
2006-05-31 11:26:09 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
/*
|
|
|
|
* A timer is active, when it is enqueued into the rbtree or the
|
|
|
|
* callback function is running or it's in the state of being migrated
|
|
|
|
* to another cpu.
|
2006-01-10 11:52:32 +07:00
|
|
|
*
|
2015-06-11 19:46:48 +07:00
|
|
|
* It is important for this function to not return a false negative.
|
2006-01-10 11:52:32 +07:00
|
|
|
*/
|
2015-06-11 19:46:48 +07:00
|
|
|
bool hrtimer_active(const struct hrtimer *timer)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base;
|
2015-06-11 19:46:48 +07:00
|
|
|
unsigned int seq;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
do {
|
|
|
|
cpu_base = READ_ONCE(timer->base->cpu_base);
|
|
|
|
seq = raw_read_seqcount_begin(&cpu_base->seq);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
if (timer->state != HRTIMER_STATE_INACTIVE ||
|
|
|
|
cpu_base->running == timer)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
} while (read_seqcount_retry(&cpu_base->seq, seq) ||
|
|
|
|
cpu_base != READ_ONCE(timer->base->cpu_base));
|
|
|
|
|
|
|
|
return false;
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
2015-06-11 19:46:48 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_active);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
/*
|
|
|
|
* The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
|
|
|
|
* distinct sections:
|
|
|
|
*
|
|
|
|
* - queued: the timer is queued
|
|
|
|
* - callback: the timer is being ran
|
|
|
|
* - post: the timer is inactive or (re)queued
|
|
|
|
*
|
|
|
|
* On the read side we ensure we observe timer->state and cpu_base->running
|
|
|
|
* from the same section, if anything changed while we looked at it, we retry.
|
|
|
|
* This includes timer->base changing because sequence numbers alone are
|
|
|
|
* insufficient for that.
|
|
|
|
*
|
|
|
|
* The sequence numbers are required because otherwise we could still observe
|
|
|
|
* a false negative if the read side got smeared over multiple consequtive
|
|
|
|
* __run_hrtimer() invocations.
|
|
|
|
*/
|
|
|
|
|
2015-04-15 04:08:35 +07:00
|
|
|
static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
|
|
|
|
struct hrtimer_clock_base *base,
|
|
|
|
struct hrtimer *timer, ktime_t *now)
|
2008-01-26 03:08:31 +07:00
|
|
|
{
|
|
|
|
enum hrtimer_restart (*fn)(struct hrtimer *);
|
|
|
|
int restart;
|
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
lockdep_assert_held(&cpu_base->lock);
|
2008-11-25 18:43:51 +07:00
|
|
|
|
2009-08-10 09:51:23 +07:00
|
|
|
debug_deactivate(timer);
|
2015-06-11 19:46:48 +07:00
|
|
|
cpu_base->running = timer;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Separate the ->running assignment from the ->state assignment.
|
|
|
|
*
|
|
|
|
* As with a regular write barrier, this ensures the read side in
|
|
|
|
* hrtimer_active() cannot observe cpu_base->running == NULL &&
|
|
|
|
* timer->state == INACTIVE.
|
|
|
|
*/
|
|
|
|
raw_write_seqcount_barrier(&cpu_base->seq);
|
|
|
|
|
|
|
|
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
|
2008-01-26 03:08:31 +07:00
|
|
|
fn = timer->function;
|
2008-11-25 18:43:51 +07:00
|
|
|
|
2016-01-14 23:54:46 +07:00
|
|
|
/*
|
|
|
|
* Clear the 'is relative' flag for the TIME_LOW_RES case. If the
|
|
|
|
* timer is restarted with a period then it becomes an absolute
|
|
|
|
* timer. If its not restarted it does not matter.
|
|
|
|
*/
|
|
|
|
if (IS_ENABLED(CONFIG_TIME_LOW_RES))
|
|
|
|
timer->is_rel = false;
|
|
|
|
|
2008-11-25 18:43:51 +07:00
|
|
|
/*
|
|
|
|
* Because we run timers from hardirq context, there is no chance
|
|
|
|
* they get migrated to another cpu, therefore its safe to unlock
|
|
|
|
* the timer base.
|
|
|
|
*/
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock(&cpu_base->lock);
|
2009-08-10 09:51:23 +07:00
|
|
|
trace_hrtimer_expire_entry(timer, now);
|
2008-11-25 18:43:51 +07:00
|
|
|
restart = fn(timer);
|
2009-08-10 09:51:23 +07:00
|
|
|
trace_hrtimer_expire_exit(timer);
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_lock(&cpu_base->lock);
|
2008-01-26 03:08:31 +07:00
|
|
|
|
|
|
|
/*
|
2015-06-11 19:46:48 +07:00
|
|
|
* Note: We clear the running state after enqueue_hrtimer and
|
2016-06-24 01:50:37 +07:00
|
|
|
* we do not reprogram the event hardware. Happens either in
|
2009-01-05 17:28:23 +07:00
|
|
|
* hrtimer_start_range_ns() or in hrtimer_interrupt()
|
2014-05-20 20:49:48 +07:00
|
|
|
*
|
|
|
|
* Note: Because we dropped the cpu_base->lock above,
|
|
|
|
* hrtimer_start_range_ns() can have popped in and enqueued the timer
|
|
|
|
* for us already.
|
2008-01-26 03:08:31 +07:00
|
|
|
*/
|
2014-05-20 20:49:48 +07:00
|
|
|
if (restart != HRTIMER_NORESTART &&
|
|
|
|
!(timer->state & HRTIMER_STATE_ENQUEUED))
|
2009-01-05 17:28:22 +07:00
|
|
|
enqueue_hrtimer(timer, base);
|
2010-10-12 21:25:19 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
/*
|
|
|
|
* Separate the ->running assignment from the ->state assignment.
|
|
|
|
*
|
|
|
|
* As with a regular write barrier, this ensures the read side in
|
|
|
|
* hrtimer_active() cannot observe cpu_base->running == NULL &&
|
|
|
|
* timer->state == INACTIVE.
|
|
|
|
*/
|
|
|
|
raw_write_seqcount_barrier(&cpu_base->seq);
|
2010-10-12 21:25:19 +07:00
|
|
|
|
2015-06-11 19:46:48 +07:00
|
|
|
WARN_ON_ONCE(cpu_base->running != timer);
|
|
|
|
cpu_base->running = NULL;
|
2008-01-26 03:08:31 +07:00
|
|
|
}
|
|
|
|
|
2015-04-15 04:08:35 +07:00
|
|
|
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
|
2007-02-16 16:28:11 +07:00
|
|
|
{
|
2015-04-15 04:08:41 +07:00
|
|
|
struct hrtimer_clock_base *base = cpu_base->clock_base;
|
|
|
|
unsigned int active = cpu_base->active_bases;
|
2009-07-10 19:57:05 +07:00
|
|
|
|
2015-04-15 04:08:41 +07:00
|
|
|
for (; active; base++, active >>= 1) {
|
2010-09-21 09:19:17 +07:00
|
|
|
struct timerqueue_node *node;
|
2011-05-20 18:05:15 +07:00
|
|
|
ktime_t basenow;
|
|
|
|
|
2015-04-15 04:08:41 +07:00
|
|
|
if (!(active & 0x01))
|
2011-05-20 18:05:15 +07:00
|
|
|
continue;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
basenow = ktime_add(now, base->offset);
|
|
|
|
|
2010-09-21 09:19:17 +07:00
|
|
|
while ((node = timerqueue_getnext(&base->active))) {
|
2007-02-16 16:28:11 +07:00
|
|
|
struct hrtimer *timer;
|
|
|
|
|
2010-09-21 09:19:17 +07:00
|
|
|
timer = container_of(node, struct hrtimer, node);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2008-09-02 05:47:08 +07:00
|
|
|
/*
|
|
|
|
* The immediate goal for using the softexpires is
|
|
|
|
* minimizing wakeups, not running timers at the
|
|
|
|
* earliest interrupt after their soft expiration.
|
|
|
|
* This allows us to avoid using a Priority Search
|
|
|
|
* Tree, which can answer a stabbing querry for
|
|
|
|
* overlapping intervals and instead use the simple
|
|
|
|
* BST we already have.
|
|
|
|
* We don't add extra wakeups by delaying timers that
|
|
|
|
* are right-of a not yet expired timer, because that
|
|
|
|
* timer will have to trigger a wakeup anyway.
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (basenow < hrtimer_get_softexpires_tv64(timer))
|
2007-02-16 16:28:11 +07:00
|
|
|
break;
|
|
|
|
|
2015-04-15 04:08:35 +07:00
|
|
|
__run_hrtimer(cpu_base, base, timer, &basenow);
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
}
|
2015-04-15 04:08:35 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
|
|
|
|
|
|
/*
|
|
|
|
* High resolution timer interrupt
|
|
|
|
* Called with interrupts disabled
|
|
|
|
*/
|
|
|
|
void hrtimer_interrupt(struct clock_event_device *dev)
|
|
|
|
{
|
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
|
|
|
ktime_t expires_next, now, entry_time, delta;
|
|
|
|
int retries = 0;
|
|
|
|
|
|
|
|
BUG_ON(!cpu_base->hres_active);
|
|
|
|
cpu_base->nr_events++;
|
2016-12-25 17:38:40 +07:00
|
|
|
dev->next_event = KTIME_MAX;
|
2015-04-15 04:08:35 +07:00
|
|
|
|
|
|
|
raw_spin_lock(&cpu_base->lock);
|
|
|
|
entry_time = now = hrtimer_update_base(cpu_base);
|
|
|
|
retry:
|
|
|
|
cpu_base->in_hrtirq = 1;
|
|
|
|
/*
|
|
|
|
* We set expires_next to KTIME_MAX here with cpu_base->lock
|
|
|
|
* held to prevent that a timer is enqueued in our queue via
|
|
|
|
* the migration code. This does not affect enqueueing of
|
|
|
|
* timers which run their callback and need to be requeued on
|
|
|
|
* this CPU.
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
cpu_base->expires_next = KTIME_MAX;
|
2015-04-15 04:08:35 +07:00
|
|
|
|
|
|
|
__hrtimer_run_queues(cpu_base, now);
|
|
|
|
|
2015-01-21 03:24:10 +07:00
|
|
|
/* Reevaluate the clock bases for the next expiry */
|
|
|
|
expires_next = __hrtimer_get_next_event(cpu_base);
|
2009-07-10 19:57:05 +07:00
|
|
|
/*
|
|
|
|
* Store the new expiry value so the migration code can verify
|
|
|
|
* against it.
|
|
|
|
*/
|
2007-02-16 16:28:11 +07:00
|
|
|
cpu_base->expires_next = expires_next;
|
2015-01-21 03:24:10 +07:00
|
|
|
cpu_base->in_hrtirq = 0;
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock(&cpu_base->lock);
|
2007-02-16 16:28:11 +07:00
|
|
|
|
|
|
|
/* Reprogramming necessary ? */
|
2015-04-03 10:34:05 +07:00
|
|
|
if (!tick_program_event(expires_next, 0)) {
|
2009-11-13 23:05:44 +07:00
|
|
|
cpu_base->hang_detected = 0;
|
|
|
|
return;
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
2009-11-13 23:05:44 +07:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The next timer was already expired due to:
|
|
|
|
* - tracing
|
|
|
|
* - long lasting callbacks
|
|
|
|
* - being scheduled away when running in a VM
|
|
|
|
*
|
|
|
|
* We need to prevent that we loop forever in the hrtimer
|
|
|
|
* interrupt routine. We give it 3 attempts to avoid
|
|
|
|
* overreacting on some spurious event.
|
2012-07-11 05:43:25 +07:00
|
|
|
*
|
|
|
|
* Acquire base lock for updating the offsets and retrieving
|
|
|
|
* the current time.
|
2009-11-13 23:05:44 +07:00
|
|
|
*/
|
2012-07-11 05:43:23 +07:00
|
|
|
raw_spin_lock(&cpu_base->lock);
|
2012-07-11 05:43:25 +07:00
|
|
|
now = hrtimer_update_base(cpu_base);
|
2009-11-13 23:05:44 +07:00
|
|
|
cpu_base->nr_retries++;
|
|
|
|
if (++retries < 3)
|
|
|
|
goto retry;
|
|
|
|
/*
|
|
|
|
* Give the system a chance to do something else than looping
|
|
|
|
* here. We stored the entry time, so we know exactly how long
|
|
|
|
* we spent here. We schedule the next event this amount of
|
|
|
|
* time away.
|
|
|
|
*/
|
|
|
|
cpu_base->nr_hangs++;
|
|
|
|
cpu_base->hang_detected = 1;
|
2012-07-11 05:43:23 +07:00
|
|
|
raw_spin_unlock(&cpu_base->lock);
|
2009-11-13 23:05:44 +07:00
|
|
|
delta = ktime_sub(now, entry_time);
|
2016-12-25 17:38:40 +07:00
|
|
|
if ((unsigned int)delta > cpu_base->max_hang_time)
|
|
|
|
cpu_base->max_hang_time = (unsigned int) delta;
|
2009-11-13 23:05:44 +07:00
|
|
|
/*
|
|
|
|
* Limit it to a sensible value as we enforce a longer
|
|
|
|
* delay. Give the CPU at least 100ms to catch up.
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (delta > 100 * NSEC_PER_MSEC)
|
2009-11-13 23:05:44 +07:00
|
|
|
expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
|
|
|
|
else
|
|
|
|
expires_next = ktime_add(now, delta);
|
|
|
|
tick_program_event(expires_next, 1);
|
|
|
|
printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n",
|
|
|
|
ktime_to_ns(delta));
|
2007-02-16 16:28:11 +07:00
|
|
|
}
|
|
|
|
|
2017-03-17 08:08:13 +07:00
|
|
|
/* called with interrupts disabled */
|
2015-04-15 04:08:51 +07:00
|
|
|
static inline void __hrtimer_peek_ahead_timers(void)
|
2009-01-05 17:28:19 +07:00
|
|
|
{
|
|
|
|
struct tick_device *td;
|
|
|
|
|
|
|
|
if (!hrtimer_hres_active())
|
|
|
|
return;
|
|
|
|
|
2014-08-18 00:30:25 +07:00
|
|
|
td = this_cpu_ptr(&tick_cpu_device);
|
2009-01-05 17:28:19 +07:00
|
|
|
if (td && td->evtdev)
|
|
|
|
hrtimer_interrupt(td->evtdev);
|
|
|
|
}
|
|
|
|
|
2009-01-05 20:11:10 +07:00
|
|
|
#else /* CONFIG_HIGH_RES_TIMERS */
|
|
|
|
|
|
|
|
static inline void __hrtimer_peek_ahead_timers(void) { }
|
|
|
|
|
|
|
|
#endif /* !CONFIG_HIGH_RES_TIMERS */
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 16:28:13 +07:00
|
|
|
|
2008-01-26 03:08:31 +07:00
|
|
|
/*
|
2015-04-15 04:08:51 +07:00
|
|
|
* Called from run_local_timers in hardirq context every jiffy
|
2008-01-26 03:08:31 +07:00
|
|
|
*/
|
2008-04-19 03:39:00 +07:00
|
|
|
void hrtimer_run_queues(void)
|
2008-01-26 03:08:31 +07:00
|
|
|
{
|
2014-08-18 00:30:26 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
|
2015-04-15 04:08:35 +07:00
|
|
|
ktime_t now;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2015-04-15 04:08:39 +07:00
|
|
|
if (__hrtimer_hres_active(cpu_base))
|
2008-01-26 03:08:31 +07:00
|
|
|
return;
|
2007-02-16 16:28:11 +07:00
|
|
|
|
2008-01-26 03:08:31 +07:00
|
|
|
/*
|
2015-04-15 04:08:51 +07:00
|
|
|
* This _is_ ugly: We have to check periodically, whether we
|
|
|
|
* can switch to highres and / or nohz mode. The clocksource
|
|
|
|
* switch happens with xtime_lock held. Notification from
|
|
|
|
* there only sets the check bit in the tick_oneshot code,
|
|
|
|
* otherwise we might deadlock vs. xtime_lock.
|
2008-01-26 03:08:31 +07:00
|
|
|
*/
|
2015-04-15 04:08:51 +07:00
|
|
|
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
|
2008-01-26 03:08:31 +07:00
|
|
|
hrtimer_switch_to_hres();
|
2006-03-31 17:31:20 +07:00
|
|
|
return;
|
2008-04-19 03:39:00 +07:00
|
|
|
}
|
2015-04-15 04:08:51 +07:00
|
|
|
|
2015-04-15 04:08:35 +07:00
|
|
|
raw_spin_lock(&cpu_base->lock);
|
|
|
|
now = hrtimer_update_base(cpu_base);
|
|
|
|
__hrtimer_run_queues(cpu_base, now);
|
|
|
|
raw_spin_unlock(&cpu_base->lock);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
2006-01-10 11:52:35 +07:00
|
|
|
/*
|
|
|
|
* Sleep related functions:
|
|
|
|
*/
|
2007-02-16 16:27:49 +07:00
|
|
|
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
|
2006-03-31 17:31:17 +07:00
|
|
|
{
|
|
|
|
struct hrtimer_sleeper *t =
|
|
|
|
container_of(timer, struct hrtimer_sleeper, timer);
|
|
|
|
struct task_struct *task = t->task;
|
|
|
|
|
|
|
|
t->task = NULL;
|
|
|
|
if (task)
|
|
|
|
wake_up_process(task);
|
|
|
|
|
|
|
|
return HRTIMER_NORESTART;
|
|
|
|
}
|
|
|
|
|
2006-07-03 14:25:41 +07:00
|
|
|
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
|
2006-03-31 17:31:17 +07:00
|
|
|
{
|
|
|
|
sl->timer.function = hrtimer_wakeup;
|
|
|
|
sl->task = task;
|
|
|
|
}
|
2009-08-29 13:41:29 +07:00
|
|
|
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
|
2006-03-31 17:31:17 +07:00
|
|
|
|
2017-06-25 01:45:06 +07:00
|
|
|
int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
|
2017-06-07 15:42:32 +07:00
|
|
|
{
|
|
|
|
switch(restart->nanosleep.type) {
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
case TT_COMPAT:
|
2017-06-25 01:45:06 +07:00
|
|
|
if (compat_put_timespec64(ts, restart->nanosleep.compat_rmtp))
|
2017-06-07 15:42:32 +07:00
|
|
|
return -EFAULT;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
case TT_NATIVE:
|
2017-06-25 01:45:06 +07:00
|
|
|
if (put_timespec64(ts, restart->nanosleep.rmtp))
|
2017-06-07 15:42:32 +07:00
|
|
|
return -EFAULT;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
return -ERESTART_RESTARTBLOCK;
|
|
|
|
}
|
|
|
|
|
2006-03-31 17:31:19 +07:00
|
|
|
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
|
2006-03-26 16:38:08 +07:00
|
|
|
{
|
2017-06-07 15:42:31 +07:00
|
|
|
struct restart_block *restart;
|
|
|
|
|
2006-03-31 17:31:19 +07:00
|
|
|
hrtimer_init_sleeper(t, current);
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2006-03-26 16:38:08 +07:00
|
|
|
do {
|
|
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
2008-09-02 05:02:30 +07:00
|
|
|
hrtimer_start_expires(&t->timer, mode);
|
2006-03-26 16:38:08 +07:00
|
|
|
|
2007-02-16 16:28:11 +07:00
|
|
|
if (likely(t->task))
|
2013-05-07 06:50:19 +07:00
|
|
|
freezable_schedule();
|
2006-03-26 16:38:08 +07:00
|
|
|
|
2006-03-31 17:31:19 +07:00
|
|
|
hrtimer_cancel(&t->timer);
|
2007-02-16 16:27:49 +07:00
|
|
|
mode = HRTIMER_MODE_ABS;
|
2006-03-31 17:31:19 +07:00
|
|
|
|
|
|
|
} while (t->task && !signal_pending(current));
|
2006-03-26 16:38:08 +07:00
|
|
|
|
2008-02-01 23:45:13 +07:00
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
2017-06-07 15:42:29 +07:00
|
|
|
if (!t->task)
|
2008-02-01 21:29:05 +07:00
|
|
|
return 0;
|
|
|
|
|
2017-06-07 15:42:31 +07:00
|
|
|
restart = ¤t->restart_block;
|
|
|
|
if (restart->nanosleep.type != TT_NONE) {
|
2017-06-07 15:42:29 +07:00
|
|
|
ktime_t rem = hrtimer_expires_remaining(&t->timer);
|
2017-06-25 01:45:06 +07:00
|
|
|
struct timespec64 rmt;
|
2017-06-07 15:42:31 +07:00
|
|
|
|
2017-06-07 15:42:29 +07:00
|
|
|
if (rem <= 0)
|
|
|
|
return 0;
|
2017-06-25 01:45:06 +07:00
|
|
|
rmt = ktime_to_timespec64(rem);
|
2017-06-07 15:42:29 +07:00
|
|
|
|
2017-06-07 15:42:32 +07:00
|
|
|
return nanosleep_copyout(restart, &rmt);
|
2017-06-07 15:42:29 +07:00
|
|
|
}
|
|
|
|
return -ERESTART_RESTARTBLOCK;
|
2008-02-01 21:29:05 +07:00
|
|
|
}
|
|
|
|
|
2017-06-07 15:42:33 +07:00
|
|
|
static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
|
2006-01-10 11:52:35 +07:00
|
|
|
{
|
2006-03-31 17:31:19 +07:00
|
|
|
struct hrtimer_sleeper t;
|
2017-06-07 15:42:29 +07:00
|
|
|
int ret;
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2011-05-20 18:05:15 +07:00
|
|
|
hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
|
2008-04-30 14:55:04 +07:00
|
|
|
HRTIMER_MODE_ABS);
|
2008-09-02 05:02:30 +07:00
|
|
|
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2017-06-07 15:42:29 +07:00
|
|
|
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
|
2008-04-30 14:55:04 +07:00
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-10 11:52:35 +07:00
|
|
|
}
|
|
|
|
|
2017-06-14 04:34:33 +07:00
|
|
|
long hrtimer_nanosleep(const struct timespec64 *rqtp,
|
2006-01-10 11:52:35 +07:00
|
|
|
const enum hrtimer_mode mode, const clockid_t clockid)
|
|
|
|
{
|
2017-06-07 15:42:29 +07:00
|
|
|
struct restart_block *restart;
|
2006-03-31 17:31:19 +07:00
|
|
|
struct hrtimer_sleeper t;
|
2008-04-30 14:55:04 +07:00
|
|
|
int ret = 0;
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 04:20:51 +07:00
|
|
|
u64 slack;
|
2008-09-08 22:58:59 +07:00
|
|
|
|
|
|
|
slack = current->timer_slack_ns;
|
sched/deadline: Add SCHED_DEADLINE structures & implementation
Introduces the data structures, constants and symbols needed for
SCHED_DEADLINE implementation.
Core data structure of SCHED_DEADLINE are defined, along with their
initializers. Hooks for checking if a task belong to the new policy
are also added where they are needed.
Adds a scheduling class, in sched/dl.c and a new policy called
SCHED_DEADLINE. It is an implementation of the Earliest Deadline
First (EDF) scheduling algorithm, augmented with a mechanism (called
Constant Bandwidth Server, CBS) that makes it possible to isolate
the behaviour of tasks between each other.
The typical -deadline task will be made up of a computation phase
(instance) which is activated on a periodic or sporadic fashion. The
expected (maximum) duration of such computation is called the task's
runtime; the time interval by which each instance need to be completed
is called the task's relative deadline. The task's absolute deadline
is dynamically calculated as the time instant a task (better, an
instance) activates plus the relative deadline.
The EDF algorithms selects the task with the smallest absolute
deadline as the one to be executed first, while the CBS ensures each
task to run for at most its runtime every (relative) deadline
length time interval, avoiding any interference between different
tasks (bandwidth isolation).
Thanks to this feature, also tasks that do not strictly comply with
the computational model sketched above can effectively use the new
policy.
To summarize, this patch:
- introduces the data structures, constants and symbols needed;
- implements the core logic of the scheduling algorithm in the new
scheduling class file;
- provides all the glue code between the new scheduling class and
the core scheduler and refines the interactions between sched/dl
and the other existing scheduling classes.
Signed-off-by: Dario Faggioli <raistlin@linux.it>
Signed-off-by: Michael Trimarchi <michael@amarulasolutions.com>
Signed-off-by: Fabio Checconi <fchecconi@gmail.com>
Signed-off-by: Juri Lelli <juri.lelli@gmail.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1383831828-15501-4-git-send-email-juri.lelli@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-11-28 17:14:43 +07:00
|
|
|
if (dl_task(current) || rt_task(current))
|
2008-09-08 22:58:59 +07:00
|
|
|
slack = 0;
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2008-04-30 14:55:04 +07:00
|
|
|
hrtimer_init_on_stack(&t.timer, clockid, mode);
|
2017-03-27 02:04:18 +07:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
|
2017-06-07 15:42:29 +07:00
|
|
|
ret = do_nanosleep(&t, mode);
|
|
|
|
if (ret != -ERESTART_RESTARTBLOCK)
|
2008-04-30 14:55:04 +07:00
|
|
|
goto out;
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2006-02-01 18:05:11 +07:00
|
|
|
/* Absolute timers do not update the rmtp value and restart: */
|
2008-04-30 14:55:04 +07:00
|
|
|
if (mode == HRTIMER_MODE_ABS) {
|
|
|
|
ret = -ERESTARTNOHAND;
|
|
|
|
goto out;
|
|
|
|
}
|
2006-01-10 11:52:35 +07:00
|
|
|
|
2017-06-07 15:42:29 +07:00
|
|
|
restart = ¤t->restart_block;
|
2006-09-29 16:00:28 +07:00
|
|
|
restart->fn = hrtimer_nanosleep_restart;
|
2011-05-20 18:05:15 +07:00
|
|
|
restart->nanosleep.clockid = t.timer.base->clockid;
|
2008-09-02 05:02:30 +07:00
|
|
|
restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
|
2008-04-30 14:55:04 +07:00
|
|
|
out:
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
return ret;
|
2006-01-10 11:52:35 +07:00
|
|
|
}
|
|
|
|
|
2009-01-14 20:14:03 +07:00
|
|
|
SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp,
|
|
|
|
struct timespec __user *, rmtp)
|
2006-01-10 11:52:36 +07:00
|
|
|
{
|
2017-06-25 01:45:06 +07:00
|
|
|
struct timespec64 tu;
|
2006-01-10 11:52:36 +07:00
|
|
|
|
2017-06-25 01:45:06 +07:00
|
|
|
if (get_timespec64(&tu, rqtp))
|
2006-01-10 11:52:36 +07:00
|
|
|
return -EFAULT;
|
|
|
|
|
2017-06-25 01:45:06 +07:00
|
|
|
if (!timespec64_valid(&tu))
|
2006-01-10 11:52:36 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
2017-06-07 15:42:31 +07:00
|
|
|
current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
|
2017-06-07 15:42:28 +07:00
|
|
|
current->restart_block.nanosleep.rmtp = rmtp;
|
2017-06-25 01:45:06 +07:00
|
|
|
return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
|
2006-01-10 11:52:36 +07:00
|
|
|
}
|
|
|
|
|
2017-06-07 15:42:31 +07:00
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
|
|
|
COMPAT_SYSCALL_DEFINE2(nanosleep, struct compat_timespec __user *, rqtp,
|
|
|
|
struct compat_timespec __user *, rmtp)
|
|
|
|
{
|
2017-06-25 01:45:06 +07:00
|
|
|
struct timespec64 tu;
|
2017-06-07 15:42:31 +07:00
|
|
|
|
2017-06-25 01:45:06 +07:00
|
|
|
if (compat_get_timespec64(&tu, rqtp))
|
2017-06-07 15:42:31 +07:00
|
|
|
return -EFAULT;
|
|
|
|
|
2017-06-25 01:45:06 +07:00
|
|
|
if (!timespec64_valid(&tu))
|
2017-06-07 15:42:31 +07:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
|
|
|
|
current->restart_block.nanosleep.compat_rmtp = rmtp;
|
2017-06-25 01:45:06 +07:00
|
|
|
return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
|
2017-06-07 15:42:31 +07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
/*
|
|
|
|
* Functions related to boot-time initialization:
|
|
|
|
*/
|
2016-07-15 15:41:04 +07:00
|
|
|
int hrtimers_prepare_cpu(unsigned int cpu)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
|
2006-01-10 11:52:32 +07:00
|
|
|
int i;
|
|
|
|
|
2010-09-21 09:19:17 +07:00
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
2007-02-16 16:27:50 +07:00
|
|
|
cpu_base->clock_base[i].cpu_base = cpu_base;
|
2010-09-21 09:19:17 +07:00
|
|
|
timerqueue_init_head(&cpu_base->clock_base[i].active);
|
|
|
|
}
|
2007-02-16 16:27:50 +07:00
|
|
|
|
2014-06-22 06:29:15 +07:00
|
|
|
cpu_base->cpu = cpu;
|
2007-02-16 16:28:11 +07:00
|
|
|
hrtimer_init_hres(cpu_base);
|
2016-07-15 15:41:04 +07:00
|
|
|
return 0;
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
2008-11-25 18:43:51 +07:00
|
|
|
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
|
2008-12-04 17:17:10 +07:00
|
|
|
struct hrtimer_clock_base *new_base)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
|
|
|
struct hrtimer *timer;
|
2010-09-21 09:19:17 +07:00
|
|
|
struct timerqueue_node *node;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2010-09-21 09:19:17 +07:00
|
|
|
while ((node = timerqueue_getnext(&old_base->active))) {
|
|
|
|
timer = container_of(node, struct hrtimer, node);
|
2007-02-16 16:28:11 +07:00
|
|
|
BUG_ON(hrtimer_callback_running(timer));
|
2009-08-10 09:51:23 +07:00
|
|
|
debug_deactivate(timer);
|
2008-09-29 20:44:46 +07:00
|
|
|
|
|
|
|
/*
|
2015-06-11 19:46:44 +07:00
|
|
|
* Mark it as ENQUEUED not INACTIVE otherwise the
|
2008-09-29 20:44:46 +07:00
|
|
|
* timer could be seen as !active and just vanish away
|
|
|
|
* under us on another CPU
|
|
|
|
*/
|
2015-06-11 19:46:44 +07:00
|
|
|
__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
|
2006-01-10 11:52:32 +07:00
|
|
|
timer->base = new_base;
|
2007-02-16 16:28:11 +07:00
|
|
|
/*
|
2009-01-05 17:28:23 +07:00
|
|
|
* Enqueue the timers on the new cpu. This does not
|
|
|
|
* reprogram the event device in case the timer
|
|
|
|
* expires before the earliest on this CPU, but we run
|
|
|
|
* hrtimer_interrupt after we migrated everything to
|
|
|
|
* sort out already expired timers and reprogram the
|
|
|
|
* event device.
|
2007-02-16 16:28:11 +07:00
|
|
|
*/
|
2009-01-05 17:28:22 +07:00
|
|
|
enqueue_hrtimer(timer, new_base);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-15 15:41:04 +07:00
|
|
|
int hrtimers_dead_cpu(unsigned int scpu)
|
2006-01-10 11:52:32 +07:00
|
|
|
{
|
2007-02-16 16:27:50 +07:00
|
|
|
struct hrtimer_cpu_base *old_base, *new_base;
|
2009-01-05 17:28:21 +07:00
|
|
|
int i;
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2008-12-04 17:17:10 +07:00
|
|
|
BUG_ON(cpu_online(scpu));
|
|
|
|
tick_cancel_sched_timer(scpu);
|
2009-01-05 17:28:21 +07:00
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
old_base = &per_cpu(hrtimer_bases, scpu);
|
2014-08-18 00:30:26 +07:00
|
|
|
new_base = this_cpu_ptr(&hrtimer_bases);
|
2008-08-21 06:46:04 +07:00
|
|
|
/*
|
|
|
|
* The caller is globally serialized and nobody else
|
|
|
|
* takes two locks at once, deadlock is not possible.
|
|
|
|
*/
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_lock(&new_base->lock);
|
|
|
|
raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
|
2006-01-10 11:52:32 +07:00
|
|
|
|
2007-02-16 16:27:50 +07:00
|
|
|
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
|
2008-11-25 18:43:51 +07:00
|
|
|
migrate_hrtimer_list(&old_base->clock_base[i],
|
2008-12-04 17:17:10 +07:00
|
|
|
&new_base->clock_base[i]);
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
2009-11-17 22:36:54 +07:00
|
|
|
raw_spin_unlock(&old_base->lock);
|
|
|
|
raw_spin_unlock(&new_base->lock);
|
2008-12-04 17:17:10 +07:00
|
|
|
|
2009-01-05 17:28:21 +07:00
|
|
|
/* Check, if we got expired work to do */
|
|
|
|
__hrtimer_peek_ahead_timers();
|
|
|
|
local_irq_enable();
|
2016-07-15 15:41:04 +07:00
|
|
|
return 0;
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
2008-12-04 17:17:10 +07:00
|
|
|
|
2006-01-10 11:52:32 +07:00
|
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
|
|
|
|
void __init hrtimers_init(void)
|
|
|
|
{
|
2016-07-15 15:41:04 +07:00
|
|
|
hrtimers_prepare_cpu(smp_processor_id());
|
2006-01-10 11:52:32 +07:00
|
|
|
}
|
|
|
|
|
2008-08-31 22:05:58 +07:00
|
|
|
/**
|
2010-04-03 03:40:19 +07:00
|
|
|
* schedule_hrtimeout_range_clock - sleep until timeout
|
2008-08-31 22:05:58 +07:00
|
|
|
* @expires: timeout value (ktime_t)
|
2008-09-02 05:47:08 +07:00
|
|
|
* @delta: slack in expires timeout (ktime_t)
|
2008-08-31 22:05:58 +07:00
|
|
|
* @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
|
2010-04-03 03:40:19 +07:00
|
|
|
* @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME
|
2008-08-31 22:05:58 +07:00
|
|
|
*/
|
2010-04-03 03:40:19 +07:00
|
|
|
int __sched
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 04:20:51 +07:00
|
|
|
schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
|
2010-04-03 03:40:19 +07:00
|
|
|
const enum hrtimer_mode mode, int clock)
|
2008-08-31 22:05:58 +07:00
|
|
|
{
|
|
|
|
struct hrtimer_sleeper t;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimize when a zero timeout value is given. It does not
|
|
|
|
* matter whether this is an absolute or a relative time.
|
|
|
|
*/
|
2016-12-25 17:38:40 +07:00
|
|
|
if (expires && *expires == 0) {
|
2008-08-31 22:05:58 +07:00
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2010-12-23 01:01:47 +07:00
|
|
|
* A NULL parameter means "infinite"
|
2008-08-31 22:05:58 +07:00
|
|
|
*/
|
|
|
|
if (!expires) {
|
|
|
|
schedule();
|
|
|
|
return -EINTR;
|
|
|
|
}
|
|
|
|
|
2010-04-03 03:40:19 +07:00
|
|
|
hrtimer_init_on_stack(&t.timer, clock, mode);
|
2008-09-02 05:47:08 +07:00
|
|
|
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
|
2008-08-31 22:05:58 +07:00
|
|
|
|
|
|
|
hrtimer_init_sleeper(&t, current);
|
|
|
|
|
2008-09-02 05:02:30 +07:00
|
|
|
hrtimer_start_expires(&t.timer, mode);
|
2008-08-31 22:05:58 +07:00
|
|
|
|
|
|
|
if (likely(t.task))
|
|
|
|
schedule();
|
|
|
|
|
|
|
|
hrtimer_cancel(&t.timer);
|
|
|
|
destroy_hrtimer_on_stack(&t.timer);
|
|
|
|
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
|
|
|
return !t.task ? 0 : -EINTR;
|
|
|
|
}
|
2010-04-03 03:40:19 +07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* schedule_hrtimeout_range - sleep until timeout
|
|
|
|
* @expires: timeout value (ktime_t)
|
|
|
|
* @delta: slack in expires timeout (ktime_t)
|
|
|
|
* @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
|
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
|
|
|
* The @delta argument gives the kernel the freedom to schedule the
|
|
|
|
* actual wakeup to a time that is both power and performance friendly.
|
|
|
|
* The kernel give the normal best effort behavior for "@expires+@delta",
|
|
|
|
* but may decide to fire the timer earlier, but no earlier than @expires.
|
|
|
|
*
|
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
2016-10-21 22:58:51 +07:00
|
|
|
* pass before the routine returns unless the current task is explicitly
|
|
|
|
* woken up, (e.g. by wake_up_process()).
|
2010-04-03 03:40:19 +07:00
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
2016-10-21 22:58:51 +07:00
|
|
|
* delivered to the current task or the current task is explicitly woken
|
|
|
|
* up.
|
2010-04-03 03:40:19 +07:00
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
2016-10-21 22:58:51 +07:00
|
|
|
* Returns 0 when the timer has expired. If the task was woken before the
|
|
|
|
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
|
|
|
|
* by an explicit wakeup, it returns -EINTR.
|
2010-04-03 03:40:19 +07:00
|
|
|
*/
|
timer: convert timer_slack_ns from unsigned long to u64
This patchset introduces a /proc/<pid>/timerslack_ns interface which
would allow controlling processes to be able to set the timerslack value
on other processes in order to save power by avoiding wakeups (Something
Android currently does via out-of-tree patches).
The first patch tries to fix the internal timer_slack_ns usage which was
defined as a long, which limits the slack range to ~4 seconds on 32bit
systems. It converts it to a u64, which provides the same basically
unlimited slack (500 years) on both 32bit and 64bit machines.
The second patch introduces the /proc/<pid>/timerslack_ns interface
which allows the full 64bit slack range for a task to be read or set on
both 32bit and 64bit machines.
With these two patches, on a 32bit machine, after setting the slack on
bash to 10 seconds:
$ time sleep 1
real 0m10.747s
user 0m0.001s
sys 0m0.005s
The first patch is a little ugly, since I had to chase the slack delta
arguments through a number of functions converting them to u64s. Let me
know if it makes sense to break that up more or not.
Other than that things are fairly straightforward.
This patch (of 2):
The timer_slack_ns value in the task struct is currently a unsigned
long. This means that on 32bit applications, the maximum slack is just
over 4 seconds. However, on 64bit machines, its much much larger (~500
years).
This disparity could make application development a little (as well as
the default_slack) to a u64. This means both 32bit and 64bit systems
have the same effective internal slack range.
Now the existing ABI via PR_GET_TIMERSLACK and PR_SET_TIMERSLACK specify
the interface as a unsigned long, so we preserve that limitation on
32bit systems, where SET_TIMERSLACK can only set the slack to a unsigned
long value, and GET_TIMERSLACK will return ULONG_MAX if the slack is
actually larger then what can be stored by an unsigned long.
This patch also modifies hrtimer functions which specified the slack
delta as a unsigned long.
Signed-off-by: John Stultz <john.stultz@linaro.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Oren Laadan <orenl@cellrox.com>
Cc: Ruchi Kandoi <kandoiruchi@google.com>
Cc: Rom Lemarchand <romlem@android.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Android Kernel Team <kernel-team@android.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-03-18 04:20:51 +07:00
|
|
|
int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
|
2010-04-03 03:40:19 +07:00
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return schedule_hrtimeout_range_clock(expires, delta, mode,
|
|
|
|
CLOCK_MONOTONIC);
|
|
|
|
}
|
2008-09-02 05:47:08 +07:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* schedule_hrtimeout - sleep until timeout
|
|
|
|
* @expires: timeout value (ktime_t)
|
|
|
|
* @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
|
|
|
|
*
|
|
|
|
* Make the current task sleep until the given expiry time has
|
|
|
|
* elapsed. The routine will return immediately unless
|
|
|
|
* the current task state has been set (see set_current_state()).
|
|
|
|
*
|
|
|
|
* You can set the task state as follows -
|
|
|
|
*
|
|
|
|
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
|
2016-10-21 22:58:51 +07:00
|
|
|
* pass before the routine returns unless the current task is explicitly
|
|
|
|
* woken up, (e.g. by wake_up_process()).
|
2008-09-02 05:47:08 +07:00
|
|
|
*
|
|
|
|
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
|
2016-10-21 22:58:51 +07:00
|
|
|
* delivered to the current task or the current task is explicitly woken
|
|
|
|
* up.
|
2008-09-02 05:47:08 +07:00
|
|
|
*
|
|
|
|
* The current task state is guaranteed to be TASK_RUNNING when this
|
|
|
|
* routine returns.
|
|
|
|
*
|
2016-10-21 22:58:51 +07:00
|
|
|
* Returns 0 when the timer has expired. If the task was woken before the
|
|
|
|
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
|
|
|
|
* by an explicit wakeup, it returns -EINTR.
|
2008-09-02 05:47:08 +07:00
|
|
|
*/
|
|
|
|
int __sched schedule_hrtimeout(ktime_t *expires,
|
|
|
|
const enum hrtimer_mode mode)
|
|
|
|
{
|
|
|
|
return schedule_hrtimeout_range(expires, 0, mode);
|
|
|
|
}
|
2008-08-31 22:05:58 +07:00
|
|
|
EXPORT_SYMBOL_GPL(schedule_hrtimeout);
|