arch/x86/kernel/apic/ipi.c

   1 #include <linux/cpumask.h>
   2 #include <linux/interrupt.h>
   3
   4 #include <linux/mm.h>
   5 #include <linux/delay.h>
   6 #include <linux/spinlock.h>
   7 #include <linux/kernel_stat.h>
   8 #include <linux/mc146818rtc.h>
   9 #include <linux/cache.h>
  10 #include <linux/cpu.h>
  11
  12 #include <asm/smp.h>
  13 #include <asm/mtrr.h>
  14 #include <asm/tlbflush.h>
  15 #include <asm/mmu_context.h>
  16 #include <asm/apic.h>
  17 #include <asm/proto.h>
  18 #include <asm/ipi.h>
  19
  20 void __default_send_IPI_shortcut(unsigned int shortcut, int vector, unsigned int dest)
  21 {
  22         /*
  23          * Subtle. In the case of the 'never do double writes' workaround
  24          * we have to lock out interrupts to be safe.  As we don't care
  25          * of the value read we use an atomic rmw access to avoid costly
  26          * cli/sti.  Otherwise we use an even cheaper single atomic write
  27          * to the APIC.
  28          */
  29         unsigned int cfg;
  30
  31         /*
  32          * Wait for idle.
  33          */
  34         __xapic_wait_icr_idle();
  35
  36         /*
  37          * No need to touch the target chip field
  38          */
  39         cfg = __prepare_ICR(shortcut, vector, dest);
  40
  41         /*
  42          * Send the IPI. The write to APIC_ICR fires this off.
  43          */
  44         native_apic_mem_write(APIC_ICR, cfg);
  45 }
  46
  47 /*
  48  * This is used to send an IPI with no shorthand notation (the destination is
  49  * specified in bits 56 to 63 of the ICR).
  50  */
  51 void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest)
  52 {
  53         unsigned long cfg;
  54
  55         /*
  56          * Wait for idle.
  57          */
  58         if (unlikely(vector == NMI_VECTOR))
  59                 safe_apic_wait_icr_idle();
  60         else
  61                 __xapic_wait_icr_idle();
  62
  63         /*
  64          * prepare target chip field
  65          */
  66         cfg = __prepare_ICR2(mask);
  67         native_apic_mem_write(APIC_ICR2, cfg);
  68
  69         /*
  70          * program the ICR
  71          */
  72         cfg = __prepare_ICR(0, vector, dest);
  73
  74         /*
  75          * Send the IPI. The write to APIC_ICR fires this off.
  76          */
  77         native_apic_mem_write(APIC_ICR, cfg);
  78 }
  79
  80 void default_send_IPI_single_phys(int cpu, int vector)
  81 {
  82         unsigned long flags;
  83
  84         local_irq_save(flags);
  85         __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
  86                                       vector, APIC_DEST_PHYSICAL);
  87         local_irq_restore(flags);
  88 }
  89
  90 void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
  91 {
  92         unsigned long query_cpu;
  93         unsigned long flags;
  94
  95         /*
  96          * Hack. The clustered APIC addressing mode doesn't allow us to send
  97          * to an arbitrary mask, so I do a unicast to each CPU instead.
  98          * - mbligh
  99          */
 100         local_irq_save(flags);
 101         for_each_cpu(query_cpu, mask) {
 102                 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 103                                 query_cpu), vector, APIC_DEST_PHYSICAL);
 104         }
 105         local_irq_restore(flags);
 106 }
 107
 108 void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
 109                                                  int vector)
 110 {
 111         unsigned int this_cpu = smp_processor_id();
 112         unsigned int query_cpu;
 113         unsigned long flags;
 114
 115         /* See Hack comment above */
 116
 117         local_irq_save(flags);
 118         for_each_cpu(query_cpu, mask) {
 119                 if (query_cpu == this_cpu)
 120                         continue;
 121                 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 122                                  query_cpu), vector, APIC_DEST_PHYSICAL);
 123         }
 124         local_irq_restore(flags);
 125 }
 126
 127 /*
 128  * Helper function for APICs which insist on cpumasks
 129  */
 130 void default_send_IPI_single(int cpu, int vector)
 131 {
 132         apic->send_IPI_mask(cpumask_of(cpu), vector);
 133 }
 134
 135 #ifdef CONFIG_X86_32
 136
 137 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
 138                                                  int vector)
 139 {
 140         unsigned long flags;
 141         unsigned int query_cpu;
 142
 143         /*
 144          * Hack. The clustered APIC addressing mode doesn't allow us to send
 145          * to an arbitrary mask, so I do a unicasts to each CPU instead. This
 146          * should be modified to do 1 message per cluster ID - mbligh
 147          */
 148
 149         local_irq_save(flags);
 150         for_each_cpu(query_cpu, mask)
 151                 __default_send_IPI_dest_field(
 152                         early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 153                         vector, apic->dest_logical);
 154         local_irq_restore(flags);
 155 }
 156
 157 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
 158                                                  int vector)
 159 {
 160         unsigned long flags;
 161         unsigned int query_cpu;
 162         unsigned int this_cpu = smp_processor_id();
 163
 164         /* See Hack comment above */
 165
 166         local_irq_save(flags);
 167         for_each_cpu(query_cpu, mask) {
 168                 if (query_cpu == this_cpu)
 169                         continue;
 170                 __default_send_IPI_dest_field(
 171                         early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 172                         vector, apic->dest_logical);
 173                 }
 174         local_irq_restore(flags);
 175 }
 176
 177 /*
 178  * This is only used on smaller machines.
 179  */
 180 void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
 181 {
 182         unsigned long mask = cpumask_bits(cpumask)[0];
 183         unsigned long flags;
 184
 185         if (!mask)
 186                 return;
 187
 188         local_irq_save(flags);
 189         WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
 190         __default_send_IPI_dest_field(mask, vector, apic->dest_logical);
 191         local_irq_restore(flags);
 192 }
 193
 194 void default_send_IPI_allbutself(int vector)
 195 {
 196         /*
 197          * if there are no other CPUs in the system then we get an APIC send
 198          * error if we try to broadcast, thus avoid sending IPIs in this case.
 199          */
 200         if (!(num_online_cpus() > 1))
 201                 return;
 202
 203         __default_local_send_IPI_allbutself(vector);
 204 }
 205
 206 void default_send_IPI_all(int vector)
 207 {
 208         __default_local_send_IPI_all(vector);
 209 }
 210
 211 void default_send_IPI_self(int vector)
 212 {
 213         __default_send_IPI_shortcut(APIC_DEST_SELF, vector, apic->dest_logical);
 214 }
 215
 216 /* must come after the send_IPI functions above for inlining */
 217 static int convert_apicid_to_cpu(int apic_id)
 218 {
 219         int i;
 220
 221         for_each_possible_cpu(i) {
 222                 if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
 223                         return i;
 224         }
 225         return -1;
 226 }
 227
 228 int safe_smp_processor_id(void)
 229 {
 230         int apicid, cpuid;
 231
 232         if (!boot_cpu_has(X86_FEATURE_APIC))
 233                 return 0;
 234
 235         apicid = hard_smp_processor_id();
 236         if (apicid == BAD_APICID)
 237                 return 0;
 238
 239         cpuid = convert_apicid_to_cpu(apicid);
 240
 241         return cpuid >= 0 ? cpuid : 0;
 242 }
 243 #endif