arch/arm/kernel/arch_timer.c

   1 /*
   2  *  linux/arch/arm/kernel/arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/delay.h>
  14 #include <linux/device.h>
  15 #include <linux/smp.h>
  16 #include <linux/cpu.h>
  17 #include <linux/jiffies.h>
  18 #include <linux/clockchips.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/io.h>
  22
  23 #include <asm/delay.h>
  24 #include <asm/localtimer.h>
  25 #include <asm/arch_timer.h>
  26 #include <asm/sched_clock.h>
  27
  28 static u32 arch_timer_rate;
  29
  30 enum ppi_nr {
  31         PHYS_SECURE_PPI,
  32         PHYS_NONSECURE_PPI,
  33         VIRT_PPI,
  34         HYP_PPI,
  35         MAX_TIMER_PPI
  36 };
  37
  38 static int arch_timer_ppi[MAX_TIMER_PPI];
  39
  40 static struct clock_event_device __percpu **arch_timer_evt;
  41 static struct delay_timer arch_delay_timer;
  42
  43 static bool arch_timer_use_virtual = true;
  44
  45 /*
  46  * Architected system timer support.
  47  */
  48
  49 #define ARCH_TIMER_CTRL_ENABLE          (1 << 0)
  50 #define ARCH_TIMER_CTRL_IT_MASK         (1 << 1)
  51 #define ARCH_TIMER_CTRL_IT_STAT         (1 << 2)
  52
  53 #define ARCH_TIMER_REG_CTRL             0
  54 #define ARCH_TIMER_REG_FREQ             1
  55 #define ARCH_TIMER_REG_TVAL             2
  56
  57 #define ARCH_TIMER_PHYS_ACCESS          0
  58 #define ARCH_TIMER_VIRT_ACCESS          1
  59
  60 /*
  61  * These register accessors are marked inline so the compiler can
  62  * nicely work out which register we want, and chuck away the rest of
  63  * the code. At least it does so with a recent GCC (4.6.3).
  64  */
  65 static inline void arch_timer_reg_write(const int access, const int reg, u32 val)
  66 {
  67         if (access == ARCH_TIMER_PHYS_ACCESS) {
  68                 switch (reg) {
  69                 case ARCH_TIMER_REG_CTRL:
  70                         asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" (val));
  71                         break;
  72                 case ARCH_TIMER_REG_TVAL:
  73                         asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r" (val));
  74                         break;
  75                 }
  76         }
  77
  78         if (access == ARCH_TIMER_VIRT_ACCESS) {
  79                 switch (reg) {
  80                 case ARCH_TIMER_REG_CTRL:
  81                         asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" (val));
  82                         break;
  83                 case ARCH_TIMER_REG_TVAL:
  84                         asm volatile("mcr p15, 0, %0, c14, c3, 0" : : "r" (val));
  85                         break;
  86                 }
  87         }
  88
  89         isb();
  90 }
  91
  92 static inline u32 arch_timer_reg_read(const int access, const int reg)
  93 {
  94         u32 val = 0;
  95
  96         if (access == ARCH_TIMER_PHYS_ACCESS) {
  97                 switch (reg) {
  98                 case ARCH_TIMER_REG_CTRL:
  99                         asm volatile("mrc p15, 0, %0, c14, c2, 1" : "=r" (val));
 100                         break;
 101                 case ARCH_TIMER_REG_TVAL:
 102                         asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (val));
 103                         break;
 104                 case ARCH_TIMER_REG_FREQ:
 105                         asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (val));
 106                         break;
 107                 }
 108         }
 109
 110         if (access == ARCH_TIMER_VIRT_ACCESS) {
 111                 switch (reg) {
 112                 case ARCH_TIMER_REG_CTRL:
 113                         asm volatile("mrc p15, 0, %0, c14, c3, 1" : "=r" (val));
 114                         break;
 115                 case ARCH_TIMER_REG_TVAL:
 116                         asm volatile("mrc p15, 0, %0, c14, c3, 0" : "=r" (val));
 117                         break;
 118                 }
 119         }
 120
 121         return val;
 122 }
 123
 124 static inline u64 arch_counter_get_cntpct(void)
 125 {
 126         u64 cval;
 127         asm volatile("mrrc p15, 0, %Q0, %R0, c14" : "=r" (cval));
 128         return cval;
 129 }
 130
 131 static inline u64 arch_counter_get_cntvct(void)
 132 {
 133         u64 cval;
 134         asm volatile("mrrc p15, 1, %Q0, %R0, c14" : "=r" (cval));
 135         return cval;
 136 }
 137
 138 static irqreturn_t inline timer_handler(const int access,
 139                                         struct clock_event_device *evt)
 140 {
 141         unsigned long ctrl;
 142         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
 143         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 144                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 145                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
 146                 evt->event_handler(evt);
 147                 return IRQ_HANDLED;
 148         }
 149
 150         return IRQ_NONE;
 151 }
 152
 153 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 154 {
 155         struct clock_event_device *evt = *(struct clock_event_device **)dev_id;
 156
 157         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 158 }
 159
 160 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 161 {
 162         struct clock_event_device *evt = *(struct clock_event_device **)dev_id;
 163
 164         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 165 }
 166
 167 static inline void timer_set_mode(const int access, int mode)
 168 {
 169         unsigned long ctrl;
 170         switch (mode) {
 171         case CLOCK_EVT_MODE_UNUSED:
 172         case CLOCK_EVT_MODE_SHUTDOWN:
 173                 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
 174                 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 175                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
 176                 break;
 177         default:
 178                 break;
 179         }
 180 }
 181
 182 static void arch_timer_set_mode_virt(enum clock_event_mode mode,
 183                                      struct clock_event_device *clk)
 184 {
 185         timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode);
 186 }
 187
 188 static void arch_timer_set_mode_phys(enum clock_event_mode mode,
 189                                      struct clock_event_device *clk)
 190 {
 191         timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode);
 192 }
 193
 194 static inline void set_next_event(const int access, unsigned long evt)
 195 {
 196         unsigned long ctrl;
 197         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
 198         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 199         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 200         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt);
 201         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
 202 }
 203
 204 static int arch_timer_set_next_event_virt(unsigned long evt,
 205                                           struct clock_event_device *unused)
 206 {
 207         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt);
 208         return 0;
 209 }
 210
 211 static int arch_timer_set_next_event_phys(unsigned long evt,
 212                                           struct clock_event_device *unused)
 213 {
 214         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt);
 215         return 0;
 216 }
 217
 218 static int __cpuinit arch_timer_setup(struct clock_event_device *clk)
 219 {
 220         clk->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP;
 221         clk->name = "arch_sys_timer";
 222         clk->rating = 450;
 223         if (arch_timer_use_virtual) {
 224                 clk->irq = arch_timer_ppi[VIRT_PPI];
 225                 clk->set_mode = arch_timer_set_mode_virt;
 226                 clk->set_next_event = arch_timer_set_next_event_virt;
 227         } else {
 228                 clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
 229                 clk->set_mode = arch_timer_set_mode_phys;
 230                 clk->set_next_event = arch_timer_set_next_event_phys;
 231         }
 232
 233         clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, NULL);
 234
 235         clockevents_config_and_register(clk, arch_timer_rate,
 236                                         0xf, 0x7fffffff);
 237
 238         *__this_cpu_ptr(arch_timer_evt) = clk;
 239
 240         if (arch_timer_use_virtual)
 241                 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
 242         else {
 243                 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
 244                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 245                         enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 246         }
 247
 248         return 0;
 249 }
 250
 251 static int arch_timer_available(void)
 252 {
 253         u32 freq;
 254
 255         if (arch_timer_rate == 0) {
 256                 freq = arch_timer_reg_read(ARCH_TIMER_PHYS_ACCESS,
 257                                            ARCH_TIMER_REG_FREQ);
 258
 259                 /* Check the timer frequency. */
 260                 if (freq == 0) {
 261                         pr_warn("Architected timer frequency not available\n");
 262                         return -EINVAL;
 263                 }
 264
 265                 arch_timer_rate = freq;
 266         }
 267
 268         pr_info_once("Architected local timer running at %lu.%02luMHz (%s).\n",
 269                      (unsigned long)arch_timer_rate / 1000000,
 270                      (unsigned long)(arch_timer_rate / 10000) % 100,
 271                      arch_timer_use_virtual ? "virt" : "phys");
 272         return 0;
 273 }
 274
 275 /*
 276  * Some external users of arch_timer_read_counter (e.g. sched_clock) may try to
 277  * call it before it has been initialised. Rather than incur a performance
 278  * penalty checking for initialisation, provide a default implementation that
 279  * won't lead to time appearing to jump backwards.
 280  */
 281 static u64 arch_timer_read_zero(void)
 282 {
 283         return 0;
 284 }
 285
 286 u64 (*arch_timer_read_counter)(void) = arch_timer_read_zero;
 287
 288 static u32 arch_timer_read_counter32(void)
 289 {
 290         return arch_timer_read_counter();
 291 }
 292
 293 static cycle_t arch_counter_read(struct clocksource *cs)
 294 {
 295         return arch_timer_read_counter();
 296 }
 297
 298 static unsigned long arch_timer_read_current_timer(void)
 299 {
 300         return arch_timer_read_counter();
 301 }
 302
 303 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 304 {
 305         return arch_timer_read_counter();
 306 }
 307
 308 static struct clocksource clocksource_counter = {
 309         .name   = "arch_sys_counter",
 310         .rating = 400,
 311         .read   = arch_counter_read,
 312         .mask   = CLOCKSOURCE_MASK(56),
 313         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 314 };
 315
 316 static struct cyclecounter cyclecounter = {
 317         .read   = arch_counter_read_cc,
 318         .mask   = CLOCKSOURCE_MASK(56),
 319 };
 320
 321 static struct timecounter timecounter;
 322
 323 struct timecounter *arch_timer_get_timecounter(void)
 324 {
 325         return &timecounter;
 326 }
 327
 328 static void __cpuinit arch_timer_stop(struct clock_event_device *clk)
 329 {
 330         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 331                  clk->irq, smp_processor_id());
 332
 333         if (arch_timer_use_virtual)
 334                 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
 335         else {
 336                 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
 337                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 338                         disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 339         }
 340
 341         clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
 342 }
 343
 344 static struct local_timer_ops arch_timer_ops __cpuinitdata = {
 345         .setup  = arch_timer_setup,
 346         .stop   = arch_timer_stop,
 347 };
 348
 349 static struct clock_event_device arch_timer_global_evt;
 350
 351 static int __init arch_timer_register(void)
 352 {
 353         int err;
 354         int ppi;
 355
 356         err = arch_timer_available();
 357         if (err)
 358                 goto out;
 359
 360         arch_timer_evt = alloc_percpu(struct clock_event_device *);
 361         if (!arch_timer_evt) {
 362                 err = -ENOMEM;
 363                 goto out;
 364         }
 365
 366         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 367         cyclecounter.mult = clocksource_counter.mult;
 368         cyclecounter.shift = clocksource_counter.shift;
 369         timecounter_init(&timecounter, &cyclecounter,
 370                          arch_counter_get_cntpct());
 371
 372         if (arch_timer_use_virtual) {
 373                 ppi = arch_timer_ppi[VIRT_PPI];
 374                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 375                                          "arch_timer", arch_timer_evt);
 376         } else {
 377                 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
 378                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 379                                          "arch_timer", arch_timer_evt);
 380                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 381                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 382                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 383                                                  "arch_timer", arch_timer_evt);
 384                         if (err)
 385                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 386                                                 arch_timer_evt);
 387                 }
 388         }
 389
 390         if (err) {
 391                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 392                        ppi, err);
 393                 goto out_free;
 394         }
 395
 396         err = local_timer_register(&arch_timer_ops);
 397         if (err) {
 398                 /*
 399                  * We couldn't register as a local timer (could be
 400                  * because we're on a UP platform, or because some
 401                  * other local timer is already present...). Try as a
 402                  * global timer instead.
 403                  */
 404                 arch_timer_global_evt.cpumask = cpumask_of(0);
 405                 err = arch_timer_setup(&arch_timer_global_evt);
 406         }
 407         if (err)
 408                 goto out_free_irq;
 409
 410         /* Use the architected timer for the delay loop. */
 411         arch_delay_timer.read_current_timer = &arch_timer_read_current_timer;
 412         arch_delay_timer.freq = arch_timer_rate;
 413         register_current_timer_delay(&arch_delay_timer);
 414         return 0;
 415
 416 out_free_irq:
 417         if (arch_timer_use_virtual)
 418                 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
 419         else {
 420                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 421                                 arch_timer_evt);
 422                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 423                         free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 424                                         arch_timer_evt);
 425         }
 426
 427 out_free:
 428         free_percpu(arch_timer_evt);
 429 out:
 430         return err;
 431 }
 432
 433 static const struct of_device_id arch_timer_of_match[] __initconst = {
 434         { .compatible   = "arm,armv7-timer",    },
 435         {},
 436 };
 437
 438 int __init arch_timer_of_register(void)
 439 {
 440         struct device_node *np;
 441         u32 freq;
 442         int i;
 443
 444         np = of_find_matching_node(NULL, arch_timer_of_match);
 445         if (!np) {
 446                 pr_err("arch_timer: can't find DT node\n");
 447                 return -ENODEV;
 448         }
 449
 450         /* Try to determine the frequency from the device tree or CNTFRQ */
 451         if (!of_property_read_u32(np, "clock-frequency", &freq))
 452                 arch_timer_rate = freq;
 453
 454         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 455                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 456
 457         of_node_put(np);
 458
 459         /*
 460          * If no interrupt provided for virtual timer, we'll have to
 461          * stick to the physical timer. It'd better be accessible...
 462          */
 463         if (!arch_timer_ppi[VIRT_PPI]) {
 464                 arch_timer_use_virtual = false;
 465
 466                 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
 467                     !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 468                         pr_warn("arch_timer: No interrupt available, giving up\n");
 469                         return -EINVAL;
 470                 }
 471         }
 472
 473         if (arch_timer_use_virtual)
 474                 arch_timer_read_counter = arch_counter_get_cntvct;
 475         else
 476                 arch_timer_read_counter = arch_counter_get_cntpct;
 477
 478         return arch_timer_register();
 479 }
 480
 481 int __init arch_timer_sched_clock_init(void)
 482 {
 483         int err;
 484
 485         err = arch_timer_available();
 486         if (err)
 487                 return err;
 488
 489         setup_sched_clock(arch_timer_read_counter32,
 490                           32, arch_timer_rate);
 491         return 0;
 492 }