linux_dsm_epyc7002/arch/alpha/kernel/time.c
Christoph Lameter 2999a4b354 alpha: Replace __get_cpu_var
__get_cpu_var() is used for multiple purposes in the kernel source. One of
them is address calculation via the form &__get_cpu_var(x).  This calculates
the address for the instance of the percpu variable of the current processor
based on an offset.

Other use cases are for storing and retrieving data from the current
processors percpu area.  __get_cpu_var() can be used as an lvalue when
writing data or on the right side of an assignment.

__get_cpu_var() is defined as :

#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))

__get_cpu_var() always only does an address determination. However, store
and retrieve operations could use a segment prefix (or global register on
other platforms) to avoid the address calculation.

this_cpu_write() and this_cpu_read() can directly take an offset into a
percpu area and use optimized assembly code to read and write per cpu
variables.

This patch converts __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations that
use the offset.  Thereby address calculations are avoided and less registers
are used when code is generated.

At the end of the patch set all uses of __get_cpu_var have been removed so
the macro is removed too.

The patch set includes passes over all arches as well. Once these operations
are used throughout then specialized macros can be defined in non -x86
arches as well in order to optimize per cpu access by f.e.  using a global
register that may be set to the per cpu base.

Transformations done to __get_cpu_var()

1. Determine the address of the percpu instance of the current processor.

	DEFINE_PER_CPU(int, y);
	int *x = &__get_cpu_var(y);

    Converts to

	int *x = this_cpu_ptr(&y);

2. Same as #1 but this time an array structure is involved.

	DEFINE_PER_CPU(int, y[20]);
	int *x = __get_cpu_var(y);

    Converts to

	int *x = this_cpu_ptr(y);

3. Retrieve the content of the current processors instance of a per cpu
variable.

	DEFINE_PER_CPU(int, y);
	int x = __get_cpu_var(y)

   Converts to

	int x = __this_cpu_read(y);

4. Retrieve the content of a percpu struct

	DEFINE_PER_CPU(struct mystruct, y);
	struct mystruct x = __get_cpu_var(y);

   Converts to

	memcpy(&x, this_cpu_ptr(&y), sizeof(x));

5. Assignment to a per cpu variable

	DEFINE_PER_CPU(int, y)
	__get_cpu_var(y) = x;

   Converts to

	__this_cpu_write(y, x);

6. Increment/Decrement etc of a per cpu variable

	DEFINE_PER_CPU(int, y);
	__get_cpu_var(y)++

   Converts to

	__this_cpu_inc(y)

CC: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Matt Turner <mattst88@gmail.com>
Acked-by: Richard Henderson <rth@twiddle.net>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
2014-08-26 13:45:53 -04:00

466 lines
12 KiB
C
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* linux/arch/alpha/kernel/time.c
*
* Copyright (C) 1991, 1992, 1995, 1999, 2000 Linus Torvalds
*
* This file contains the clocksource time handling.
* 1997-09-10 Updated NTP code according to technical memorandum Jan '96
* "A Kernel Model for Precision Timekeeping" by Dave Mills
* 1997-01-09 Adrian Sun
* use interval timer if CONFIG_RTC=y
* 1997-10-29 John Bowman (bowman@math.ualberta.ca)
* fixed tick loss calculation in timer_interrupt
* (round system clock to nearest tick instead of truncating)
* fixed algorithm in time_init for getting time from CMOS clock
* 1999-04-16 Thorsten Kranzkowski (dl8bcu@gmx.net)
* fixed algorithm in do_gettimeofday() for calculating the precise time
* from processor cycle counter (now taking lost_ticks into account)
* 2003-06-03 R. Scott Bailey <scott.bailey@eds.com>
* Tighten sanity in time_init from 1% (10,000 PPM) to 250 PPM
*/
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/param.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/ioport.h>
#include <linux/irq.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/bcd.h>
#include <linux/profile.h>
#include <linux/irq_work.h>
#include <asm/uaccess.h>
#include <asm/io.h>
#include <asm/hwrpb.h>
#include <linux/mc146818rtc.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/clocksource.h>
#include <linux/clockchips.h>
#include "proto.h"
#include "irq_impl.h"
DEFINE_SPINLOCK(rtc_lock);
EXPORT_SYMBOL(rtc_lock);
unsigned long est_cycle_freq;
#ifdef CONFIG_IRQ_WORK
DEFINE_PER_CPU(u8, irq_work_pending);
#define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1)
#define test_irq_work_pending() __this_cpu_read(irq_work_pending)
#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0)
void arch_irq_work_raise(void)
{
set_irq_work_pending_flag();
}
#else /* CONFIG_IRQ_WORK */
#define test_irq_work_pending() 0
#define clear_irq_work_pending()
#endif /* CONFIG_IRQ_WORK */
static inline __u32 rpcc(void)
{
return __builtin_alpha_rpcc();
}
/*
* The RTC as a clock_event_device primitive.
*/
static DEFINE_PER_CPU(struct clock_event_device, cpu_ce);
irqreturn_t
rtc_timer_interrupt(int irq, void *dev)
{
int cpu = smp_processor_id();
struct clock_event_device *ce = &per_cpu(cpu_ce, cpu);
/* Don't run the hook for UNUSED or SHUTDOWN. */
if (likely(ce->mode == CLOCK_EVT_MODE_PERIODIC))
ce->event_handler(ce);
if (test_irq_work_pending()) {
clear_irq_work_pending();
irq_work_run();
}
return IRQ_HANDLED;
}
static void
rtc_ce_set_mode(enum clock_event_mode mode, struct clock_event_device *ce)
{
/* The mode member of CE is updated in generic code.
Since we only support periodic events, nothing to do. */
}
static int
rtc_ce_set_next_event(unsigned long evt, struct clock_event_device *ce)
{
/* This hook is for oneshot mode, which we don't support. */
return -EINVAL;
}
static void __init
init_rtc_clockevent(void)
{
int cpu = smp_processor_id();
struct clock_event_device *ce = &per_cpu(cpu_ce, cpu);
*ce = (struct clock_event_device){
.name = "rtc",
.features = CLOCK_EVT_FEAT_PERIODIC,
.rating = 100,
.cpumask = cpumask_of(cpu),
.set_mode = rtc_ce_set_mode,
.set_next_event = rtc_ce_set_next_event,
};
clockevents_config_and_register(ce, CONFIG_HZ, 0, 0);
}
/*
* The QEMU clock as a clocksource primitive.
*/
static cycle_t
qemu_cs_read(struct clocksource *cs)
{
return qemu_get_vmtime();
}
static struct clocksource qemu_cs = {
.name = "qemu",
.rating = 400,
.read = qemu_cs_read,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
.max_idle_ns = LONG_MAX
};
/*
* The QEMU alarm as a clock_event_device primitive.
*/
static void
qemu_ce_set_mode(enum clock_event_mode mode, struct clock_event_device *ce)
{
/* The mode member of CE is updated for us in generic code.
Just make sure that the event is disabled. */
qemu_set_alarm_abs(0);
}
static int
qemu_ce_set_next_event(unsigned long evt, struct clock_event_device *ce)
{
qemu_set_alarm_rel(evt);
return 0;
}
static irqreturn_t
qemu_timer_interrupt(int irq, void *dev)
{
int cpu = smp_processor_id();
struct clock_event_device *ce = &per_cpu(cpu_ce, cpu);
ce->event_handler(ce);
return IRQ_HANDLED;
}
static void __init
init_qemu_clockevent(void)
{
int cpu = smp_processor_id();
struct clock_event_device *ce = &per_cpu(cpu_ce, cpu);
*ce = (struct clock_event_device){
.name = "qemu",
.features = CLOCK_EVT_FEAT_ONESHOT,
.rating = 400,
.cpumask = cpumask_of(cpu),
.set_mode = qemu_ce_set_mode,
.set_next_event = qemu_ce_set_next_event,
};
clockevents_config_and_register(ce, NSEC_PER_SEC, 1000, LONG_MAX);
}
void __init
common_init_rtc(void)
{
unsigned char x, sel = 0;
/* Reset periodic interrupt frequency. */
#if CONFIG_HZ == 1024 || CONFIG_HZ == 1200
x = CMOS_READ(RTC_FREQ_SELECT) & 0x3f;
/* Test includes known working values on various platforms
where 0x26 is wrong; we refuse to change those. */
if (x != 0x26 && x != 0x25 && x != 0x19 && x != 0x06) {
sel = RTC_REF_CLCK_32KHZ + 6;
}
#elif CONFIG_HZ == 256 || CONFIG_HZ == 128 || CONFIG_HZ == 64 || CONFIG_HZ == 32
sel = RTC_REF_CLCK_32KHZ + __builtin_ffs(32768 / CONFIG_HZ);
#else
# error "Unknown HZ from arch/alpha/Kconfig"
#endif
if (sel) {
printk(KERN_INFO "Setting RTC_FREQ to %d Hz (%x)\n",
CONFIG_HZ, sel);
CMOS_WRITE(sel, RTC_FREQ_SELECT);
}
/* Turn on periodic interrupts. */
x = CMOS_READ(RTC_CONTROL);
if (!(x & RTC_PIE)) {
printk("Turning on RTC interrupts.\n");
x |= RTC_PIE;
x &= ~(RTC_AIE | RTC_UIE);
CMOS_WRITE(x, RTC_CONTROL);
}
(void) CMOS_READ(RTC_INTR_FLAGS);
outb(0x36, 0x43); /* pit counter 0: system timer */
outb(0x00, 0x40);
outb(0x00, 0x40);
outb(0xb6, 0x43); /* pit counter 2: speaker */
outb(0x31, 0x42);
outb(0x13, 0x42);
init_rtc_irq();
}
#ifndef CONFIG_ALPHA_WTINT
/*
* The RPCC as a clocksource primitive.
*
* While we have free-running timecounters running on all CPUs, and we make
* a half-hearted attempt in init_rtc_rpcc_info to sync the timecounter
* with the wall clock, that initialization isn't kept up-to-date across
* different time counters in SMP mode. Therefore we can only use this
* method when there's only one CPU enabled.
*
* When using the WTINT PALcall, the RPCC may shift to a lower frequency,
* or stop altogether, while waiting for the interrupt. Therefore we cannot
* use this method when WTINT is in use.
*/
static cycle_t read_rpcc(struct clocksource *cs)
{
return rpcc();
}
static struct clocksource clocksource_rpcc = {
.name = "rpcc",
.rating = 300,
.read = read_rpcc,
.mask = CLOCKSOURCE_MASK(32),
.flags = CLOCK_SOURCE_IS_CONTINUOUS
};
#endif /* ALPHA_WTINT */
/* Validate a computed cycle counter result against the known bounds for
the given processor core. There's too much brokenness in the way of
timing hardware for any one method to work everywhere. :-(
Return 0 if the result cannot be trusted, otherwise return the argument. */
static unsigned long __init
validate_cc_value(unsigned long cc)
{
static struct bounds {
unsigned int min, max;
} cpu_hz[] __initdata = {
[EV3_CPU] = { 50000000, 200000000 }, /* guess */
[EV4_CPU] = { 100000000, 300000000 },
[LCA4_CPU] = { 100000000, 300000000 }, /* guess */
[EV45_CPU] = { 200000000, 300000000 },
[EV5_CPU] = { 250000000, 433000000 },
[EV56_CPU] = { 333000000, 667000000 },
[PCA56_CPU] = { 400000000, 600000000 }, /* guess */
[PCA57_CPU] = { 500000000, 600000000 }, /* guess */
[EV6_CPU] = { 466000000, 600000000 },
[EV67_CPU] = { 600000000, 750000000 },
[EV68AL_CPU] = { 750000000, 940000000 },
[EV68CB_CPU] = { 1000000000, 1333333333 },
/* None of the following are shipping as of 2001-11-01. */
[EV68CX_CPU] = { 1000000000, 1700000000 }, /* guess */
[EV69_CPU] = { 1000000000, 1700000000 }, /* guess */
[EV7_CPU] = { 800000000, 1400000000 }, /* guess */
[EV79_CPU] = { 1000000000, 2000000000 }, /* guess */
};
/* Allow for some drift in the crystal. 10MHz is more than enough. */
const unsigned int deviation = 10000000;
struct percpu_struct *cpu;
unsigned int index;
cpu = (struct percpu_struct *)((char*)hwrpb + hwrpb->processor_offset);
index = cpu->type & 0xffffffff;
/* If index out of bounds, no way to validate. */
if (index >= ARRAY_SIZE(cpu_hz))
return cc;
/* If index contains no data, no way to validate. */
if (cpu_hz[index].max == 0)
return cc;
if (cc < cpu_hz[index].min - deviation
|| cc > cpu_hz[index].max + deviation)
return 0;
return cc;
}
/*
* Calibrate CPU clock using legacy 8254 timer/counter. Stolen from
* arch/i386/time.c.
*/
#define CALIBRATE_LATCH 0xffff
#define TIMEOUT_COUNT 0x100000
static unsigned long __init
calibrate_cc_with_pit(void)
{
int cc, count = 0;
/* Set the Gate high, disable speaker */
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
/*
* Now let's take care of CTC channel 2
*
* Set the Gate high, program CTC channel 2 for mode 0,
* (interrupt on terminal count mode), binary count,
* load 5 * LATCH count, (LSB and MSB) to begin countdown.
*/
outb(0xb0, 0x43); /* binary, mode 0, LSB/MSB, Ch 2 */
outb(CALIBRATE_LATCH & 0xff, 0x42); /* LSB of count */
outb(CALIBRATE_LATCH >> 8, 0x42); /* MSB of count */
cc = rpcc();
do {
count++;
} while ((inb(0x61) & 0x20) == 0 && count < TIMEOUT_COUNT);
cc = rpcc() - cc;
/* Error: ECTCNEVERSET or ECPUTOOFAST. */
if (count <= 1 || count == TIMEOUT_COUNT)
return 0;
return ((long)cc * PIT_TICK_RATE) / (CALIBRATE_LATCH + 1);
}
/* The Linux interpretation of the CMOS clock register contents:
When the Update-In-Progress (UIP) flag goes from 1 to 0, the
RTC registers show the second which has precisely just started.
Let's hope other operating systems interpret the RTC the same way. */
static unsigned long __init
rpcc_after_update_in_progress(void)
{
do { } while (!(CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP));
do { } while (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP);
return rpcc();
}
void __init
time_init(void)
{
unsigned int cc1, cc2;
unsigned long cycle_freq, tolerance;
long diff;
if (alpha_using_qemu) {
clocksource_register_hz(&qemu_cs, NSEC_PER_SEC);
init_qemu_clockevent();
timer_irqaction.handler = qemu_timer_interrupt;
init_rtc_irq();
return;
}
/* Calibrate CPU clock -- attempt #1. */
if (!est_cycle_freq)
est_cycle_freq = validate_cc_value(calibrate_cc_with_pit());
cc1 = rpcc();
/* Calibrate CPU clock -- attempt #2. */
if (!est_cycle_freq) {
cc1 = rpcc_after_update_in_progress();
cc2 = rpcc_after_update_in_progress();
est_cycle_freq = validate_cc_value(cc2 - cc1);
cc1 = cc2;
}
cycle_freq = hwrpb->cycle_freq;
if (est_cycle_freq) {
/* If the given value is within 250 PPM of what we calculated,
accept it. Otherwise, use what we found. */
tolerance = cycle_freq / 4000;
diff = cycle_freq - est_cycle_freq;
if (diff < 0)
diff = -diff;
if ((unsigned long)diff > tolerance) {
cycle_freq = est_cycle_freq;
printk("HWRPB cycle frequency bogus. "
"Estimated %lu Hz\n", cycle_freq);
} else {
est_cycle_freq = 0;
}
} else if (! validate_cc_value (cycle_freq)) {
printk("HWRPB cycle frequency bogus, "
"and unable to estimate a proper value!\n");
}
/* See above for restrictions on using clocksource_rpcc. */
#ifndef CONFIG_ALPHA_WTINT
if (hwrpb->nr_processors == 1)
clocksource_register_hz(&clocksource_rpcc, cycle_freq);
#endif
/* Startup the timer source. */
alpha_mv.init_rtc();
init_rtc_clockevent();
}
/* Initialize the clock_event_device for secondary cpus. */
#ifdef CONFIG_SMP
void __init
init_clockevent(void)
{
if (alpha_using_qemu)
init_qemu_clockevent();
else
init_rtc_clockevent();
}
#endif