projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
EDAC, i7core: Remove double buffering of error records
[deliverable/linux.git] / drivers / edac / sb_edac.c
CommitLineData
eebf11a0
MCC		 1/* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module
2 *
3 * This driver supports the memory controllers found on the Intel
4 * processor family Sandy Bridge.
5 *
6 * This file may be distributed under the terms of the
7 * GNU General Public License version 2 only.
8 *
9 * Copyright (c) 2011 by:
37e59f87 10 * Mauro Carvalho Chehab
eebf11a0
MCC		 11 */
12
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/pci.h>
16#include <linux/pci_ids.h>
17#include <linux/slab.h>
18#include <linux/delay.h>
19#include <linux/edac.h>
20#include <linux/mmzone.h>
eebf11a0
MCC		 21#include <linux/smp.h>
22#include <linux/bitmap.h>
5b889e37 23#include <linux/math64.h>
eebf11a0 24#include <asm/processor.h>
3d78c9af 25#include <asm/mce.h>
eebf11a0
MCC		 26
27#include "edac_core.h"
28
29/* Static vars */
30static LIST_HEAD(sbridge_edac_list);
31static DEFINE_MUTEX(sbridge_edac_lock);
32static int probed;
33
34/*
35 * Alter this version for the module when modifications are made
36 */
7d375bff 37#define SBRIDGE_REVISION " Ver: 1.1.1 "
eebf11a0
MCC		 38#define EDAC_MOD_STR "sbridge_edac"
39
40/*
41 * Debug macros
42 */
43#define sbridge_printk(level, fmt, arg...) \
44 edac_printk(level, "sbridge", fmt, ##arg)
45
46#define sbridge_mc_printk(mci, level, fmt, arg...) \
47 edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg)
48
49/*
50 * Get a bit field at register value <v>, from bit <lo> to bit <hi>
51 */
52#define GET_BITFIELD(v, lo, hi) \
10ef6b0d 53 (((v) & GENMASK_ULL(hi, lo)) >> (lo))
eebf11a0 54
eebf11a0 55/* Devices 12 Function 6, Offsets 0x80 to 0xcc */
464f1d82 56static const u32 sbridge_dram_rule[] = {
eebf11a0
MCC		 57 0x80, 0x88, 0x90, 0x98, 0xa0,
58 0xa8, 0xb0, 0xb8, 0xc0, 0xc8,
59};
eebf11a0 60
4d715a80
AR		 61static const u32 ibridge_dram_rule[] = {
62 0x60, 0x68, 0x70, 0x78, 0x80,
63 0x88, 0x90, 0x98, 0xa0, 0xa8,
64 0xb0, 0xb8, 0xc0, 0xc8, 0xd0,
65 0xd8, 0xe0, 0xe8, 0xf0, 0xf8,
66};
eebf11a0 67
d0cdf900
JS		 68static const u32 knl_dram_rule[] = {
69 0x60, 0x68, 0x70, 0x78, 0x80, /* 0-4 */
70 0x88, 0x90, 0x98, 0xa0, 0xa8, /* 5-9 */
71 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, /* 10-14 */
72 0xd8, 0xe0, 0xe8, 0xf0, 0xf8, /* 15-19 */
73 0x100, 0x108, 0x110, 0x118, /* 20-23 */
74};
75
eebf11a0 76#define DRAM_RULE_ENABLE(reg) GET_BITFIELD(reg, 0, 0)
50d1bb93 77#define A7MODE(reg) GET_BITFIELD(reg, 26, 26)
eebf11a0 78
c59f9c06 79static char *show_dram_attr(u32 attr)
eebf11a0 80{
c59f9c06 81 switch (attr) {
eebf11a0
MCC		 82 case 0:
83 return "DRAM";
84 case 1:
85 return "MMCFG";
86 case 2:
87 return "NXM";
88 default:
89 return "unknown";
90 }
91}
92
ef1ce51e 93static const u32 sbridge_interleave_list[] = {
eebf11a0
MCC		 94 0x84, 0x8c, 0x94, 0x9c, 0xa4,
95 0xac, 0xb4, 0xbc, 0xc4, 0xcc,
96};
eebf11a0 97
4d715a80
AR		 98static const u32 ibridge_interleave_list[] = {
99 0x64, 0x6c, 0x74, 0x7c, 0x84,
100 0x8c, 0x94, 0x9c, 0xa4, 0xac,
101 0xb4, 0xbc, 0xc4, 0xcc, 0xd4,
102 0xdc, 0xe4, 0xec, 0xf4, 0xfc,
103};
104
d0cdf900
JS		 105static const u32 knl_interleave_list[] = {
106 0x64, 0x6c, 0x74, 0x7c, 0x84, /* 0-4 */
107 0x8c, 0x94, 0x9c, 0xa4, 0xac, /* 5-9 */
108 0xb4, 0xbc, 0xc4, 0xcc, 0xd4, /* 10-14 */
109 0xdc, 0xe4, 0xec, 0xf4, 0xfc, /* 15-19 */
110 0x104, 0x10c, 0x114, 0x11c, /* 20-23 */
111};
112
cc311991
AR		 113struct interleave_pkg {
114 unsigned char start;
115 unsigned char end;
116};
117
118static const struct interleave_pkg sbridge_interleave_pkg[] = {
119 { 0, 2 },
120 { 3, 5 },
121 { 8, 10 },
122 { 11, 13 },
123 { 16, 18 },
124 { 19, 21 },
125 { 24, 26 },
126 { 27, 29 },
127};
128
4d715a80
AR		 129static const struct interleave_pkg ibridge_interleave_pkg[] = {
130 { 0, 3 },
131 { 4, 7 },
132 { 8, 11 },
133 { 12, 15 },
134 { 16, 19 },
135 { 20, 23 },
136 { 24, 27 },
137 { 28, 31 },
138};
139
cc311991
AR		 140static inline int sad_pkg(const struct interleave_pkg *table, u32 reg,
141 int interleave)
eebf11a0 142{
cc311991
AR		 143 return GET_BITFIELD(reg, table[interleave].start,
144 table[interleave].end);
eebf11a0
MCC		 145}
146
147/* Devices 12 Function 7 */
148
149#define TOLM 0x80
d0cdf900 150#define TOHM 0x84
f7cf2a22 151#define HASWELL_TOLM 0xd0
50d1bb93
AR		 152#define HASWELL_TOHM_0 0xd4
153#define HASWELL_TOHM_1 0xd8
d0cdf900
JS		 154#define KNL_TOLM 0xd0
155#define KNL_TOHM_0 0xd4
156#define KNL_TOHM_1 0xd8
eebf11a0
MCC		 157
158#define GET_TOLM(reg) ((GET_BITFIELD(reg, 0, 3) << 28) | 0x3ffffff)
159#define GET_TOHM(reg) ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff)
160
161/* Device 13 Function 6 */
162
163#define SAD_TARGET 0xf0
164
165#define SOURCE_ID(reg) GET_BITFIELD(reg, 9, 11)
166
d0cdf900
JS		 167#define SOURCE_ID_KNL(reg) GET_BITFIELD(reg, 12, 14)
168
eebf11a0
MCC		 169#define SAD_CONTROL 0xf4
170
eebf11a0
MCC		 171/* Device 14 function 0 */
172
173static const u32 tad_dram_rule[] = {
174 0x40, 0x44, 0x48, 0x4c,
175 0x50, 0x54, 0x58, 0x5c,
176 0x60, 0x64, 0x68, 0x6c,
177};
178#define MAX_TAD ARRAY_SIZE(tad_dram_rule)
179
180#define TAD_LIMIT(reg) ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff)
181#define TAD_SOCK(reg) GET_BITFIELD(reg, 10, 11)
182#define TAD_CH(reg) GET_BITFIELD(reg, 8, 9)
183#define TAD_TGT3(reg) GET_BITFIELD(reg, 6, 7)
184#define TAD_TGT2(reg) GET_BITFIELD(reg, 4, 5)
185#define TAD_TGT1(reg) GET_BITFIELD(reg, 2, 3)
186#define TAD_TGT0(reg) GET_BITFIELD(reg, 0, 1)
187
188/* Device 15, function 0 */
189
190#define MCMTR 0x7c
d0cdf900 191#define KNL_MCMTR 0x624
eebf11a0
MCC		 192
193#define IS_ECC_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 2, 2)
194#define IS_LOCKSTEP_ENABLED(mcmtr) GET_BITFIELD(mcmtr, 1, 1)
195#define IS_CLOSE_PG(mcmtr) GET_BITFIELD(mcmtr, 0, 0)
196
197/* Device 15, function 1 */
198
199#define RASENABLES 0xac
200#define IS_MIRROR_ENABLED(reg) GET_BITFIELD(reg, 0, 0)
201
202/* Device 15, functions 2-5 */
203
204static const int mtr_regs[] = {
205 0x80, 0x84, 0x88,
206};
207
d0cdf900
JS		 208static const int knl_mtr_reg = 0xb60;
209
eebf11a0
MCC		 210#define RANK_DISABLE(mtr) GET_BITFIELD(mtr, 16, 19)
211#define IS_DIMM_PRESENT(mtr) GET_BITFIELD(mtr, 14, 14)
212#define RANK_CNT_BITS(mtr) GET_BITFIELD(mtr, 12, 13)
213#define RANK_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 2, 4)
214#define COL_WIDTH_BITS(mtr) GET_BITFIELD(mtr, 0, 1)
215
216static const u32 tad_ch_nilv_offset[] = {
217 0x90, 0x94, 0x98, 0x9c,
218 0xa0, 0xa4, 0xa8, 0xac,
219 0xb0, 0xb4, 0xb8, 0xbc,
220};
221#define CHN_IDX_OFFSET(reg) GET_BITFIELD(reg, 28, 29)
222#define TAD_OFFSET(reg) (GET_BITFIELD(reg, 6, 25) << 26)
223
224static const u32 rir_way_limit[] = {
225 0x108, 0x10c, 0x110, 0x114, 0x118,
226};
227#define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit)
228
229#define IS_RIR_VALID(reg) GET_BITFIELD(reg, 31, 31)
230#define RIR_WAY(reg) GET_BITFIELD(reg, 28, 29)
eebf11a0
MCC		 231
232#define MAX_RIR_WAY 8
233
234static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = {
235 { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c },
236 { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c },
237 { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c },
238 { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c },
239 { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc },
240};
241
242#define RIR_RNK_TGT(reg) GET_BITFIELD(reg, 16, 19)
243#define RIR_OFFSET(reg) GET_BITFIELD(reg, 2, 14)
244
245/* Device 16, functions 2-7 */
246
247/*
248 * FIXME: Implement the error count reads directly
249 */
250
251static const u32 correrrcnt[] = {
252 0x104, 0x108, 0x10c, 0x110,
253};
254
255#define RANK_ODD_OV(reg) GET_BITFIELD(reg, 31, 31)
256#define RANK_ODD_ERR_CNT(reg) GET_BITFIELD(reg, 16, 30)
257#define RANK_EVEN_OV(reg) GET_BITFIELD(reg, 15, 15)
258#define RANK_EVEN_ERR_CNT(reg) GET_BITFIELD(reg, 0, 14)
259
260static const u32 correrrthrsld[] = {
261 0x11c, 0x120, 0x124, 0x128,
262};
263
264#define RANK_ODD_ERR_THRSLD(reg) GET_BITFIELD(reg, 16, 30)
265#define RANK_EVEN_ERR_THRSLD(reg) GET_BITFIELD(reg, 0, 14)
266
267
268/* Device 17, function 0 */
269
ef1e8d03 270#define SB_RANK_CFG_A 0x0328
eebf11a0 271
4d715a80 272#define IB_RANK_CFG_A 0x0320
eebf11a0 273
eebf11a0
MCC		 274/*
275 * sbridge structs
276 */
277
7d375bff 278#define NUM_CHANNELS 8 /* 2MC per socket, four chan per MC */
351fc4a9 279#define MAX_DIMMS 3 /* Max DIMMS per channel */
d0cdf900
JS		 280#define KNL_MAX_CHAS 38 /* KNL max num. of Cache Home Agents */
281#define KNL_MAX_CHANNELS 6 /* KNL max num. of PCI channels */
282#define KNL_MAX_EDCS 8 /* Embedded DRAM controllers */
351fc4a9 283#define CHANNEL_UNSPECIFIED 0xf /* Intel IA32 SDM 15-14 */
eebf11a0 284
4d715a80
AR		 285enum type {
286 SANDY_BRIDGE,
287 IVY_BRIDGE,
50d1bb93 288 HASWELL,
1f39581a 289 BROADWELL,
d0cdf900 290 KNIGHTS_LANDING,
4d715a80
AR		 291};
292
fb79a509 293struct sbridge_pvt;
eebf11a0 294struct sbridge_info {
4d715a80 295 enum type type;
464f1d82
AR		 296 u32 mcmtr;
297 u32 rankcfgr;
298 u64 (*get_tolm)(struct sbridge_pvt *pvt);
299 u64 (*get_tohm)(struct sbridge_pvt *pvt);
b976bcf2 300 u64 (*rir_limit)(u32 reg);
c59f9c06
JS		 301 u64 (*sad_limit)(u32 reg);
302 u32 (*interleave_mode)(u32 reg);
303 char* (*show_interleave_mode)(u32 reg);
304 u32 (*dram_attr)(u32 reg);
464f1d82 305 const u32 *dram_rule;
ef1ce51e 306 const u32 *interleave_list;
cc311991 307 const struct interleave_pkg *interleave_pkg;
464f1d82 308 u8 max_sad;
ef1ce51e 309 u8 max_interleave;
f14d6892 310 u8 (*get_node_id)(struct sbridge_pvt *pvt);
9e375446 311 enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt);
12f0721c 312 enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr);
50d1bb93 313 struct pci_dev *pci_vtd;
eebf11a0
MCC		 314};
315
316struct sbridge_channel {
317 u32 ranks;
318 u32 dimms;
319};
320
321struct pci_id_descr {
c41afdca 322 int dev_id;
eebf11a0
MCC		 323 int optional;
324};
325
326struct pci_id_table {
327 const struct pci_id_descr *descr;
328 int n_devs;
329};
330
331struct sbridge_dev {
332 struct list_head list;
333 u8 bus, mc;
334 u8 node_id, source_id;
335 struct pci_dev **pdev;
336 int n_devs;
337 struct mem_ctl_info *mci;
338};
339
d0cdf900
JS		 340struct knl_pvt {
341 struct pci_dev *pci_cha[KNL_MAX_CHAS];
342 struct pci_dev *pci_channel[KNL_MAX_CHANNELS];
343 struct pci_dev *pci_mc0;
344 struct pci_dev *pci_mc1;
345 struct pci_dev *pci_mc0_misc;
346 struct pci_dev *pci_mc1_misc;
347 struct pci_dev *pci_mc_info; /* tolm, tohm */
348};
349
eebf11a0
MCC		 350struct sbridge_pvt {
351 struct pci_dev *pci_ta, *pci_ddrio, *pci_ras;
4d715a80
AR		 352 struct pci_dev *pci_sad0, *pci_sad1;
353 struct pci_dev *pci_ha0, *pci_ha1;
354 struct pci_dev *pci_br0, *pci_br1;
50d1bb93 355 struct pci_dev *pci_ha1_ta;
eebf11a0
MCC		 356 struct pci_dev *pci_tad[NUM_CHANNELS];
357
358 struct sbridge_dev *sbridge_dev;
359
360 struct sbridge_info info;
361 struct sbridge_channel channel[NUM_CHANNELS];
362
eebf11a0
MCC		 363 /* Memory type detection */
364 bool is_mirrored, is_lockstep, is_close_pg;
365
eebf11a0
MCC		 366 /* Memory description */
367 u64 tolm, tohm;
d0cdf900 368 struct knl_pvt knl;
eebf11a0
MCC		 369};
370
dbc954dd
AR		 371#define PCI_DESCR(device_id, opt) \
372 .dev_id = (device_id), \
de4772c6 373 .optional = opt
eebf11a0
MCC		 374
375static const struct pci_id_descr pci_dev_descr_sbridge[] = {
376 /* Processor Home Agent */
dbc954dd 377 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0) },
eebf11a0
MCC		 378
379 /* Memory controller */
dbc954dd
AR		 380 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0) },
381 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0) },
382 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0) },
383 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0) },
384 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0) },
385 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0) },
386 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1) },
eebf11a0
MCC		 387
388 /* System Address Decoder */
dbc954dd
AR		 389 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0) },
390 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0) },
eebf11a0
MCC		 391
392 /* Broadcast Registers */
dbc954dd 393 { PCI_DESCR(PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0) },
eebf11a0
MCC		 394};
395
396#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
397static const struct pci_id_table pci_dev_descr_sbridge_table[] = {
398 PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge),
399 {0,} /* 0 terminated list. */
400};
401
4d715a80
AR		 402/* This changes depending if 1HA or 2HA:
403 * 1HA:
404 * 0x0eb8 (17.0) is DDRIO0
405 * 2HA:
406 * 0x0ebc (17.4) is DDRIO0
407 */
408#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0 0x0eb8
409#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0 0x0ebc
410
411/* pci ids */
412#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0 0x0ea0
413#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA 0x0ea8
414#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS 0x0e71
415#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0 0x0eaa
416#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1 0x0eab
417#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2 0x0eac
418#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3 0x0ead
419#define PCI_DEVICE_ID_INTEL_IBRIDGE_SAD 0x0ec8
420#define PCI_DEVICE_ID_INTEL_IBRIDGE_BR0 0x0ec9
421#define PCI_DEVICE_ID_INTEL_IBRIDGE_BR1 0x0eca
422#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1 0x0e60
423#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA 0x0e68
424#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS 0x0e79
425#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 0x0e6a
426#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1 0x0e6b
7d375bff
TL		 427#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2 0x0e6c
428#define PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3 0x0e6d
4d715a80
AR		 429
430static const struct pci_id_descr pci_dev_descr_ibridge[] = {
431 /* Processor Home Agent */
dbc954dd 432 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0, 0) },
4d715a80
AR		 433
434 /* Memory controller */
dbc954dd
AR		 435 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA, 0) },
436 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS, 0) },
437 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0, 0) },
438 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1, 0) },
439 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2, 0) },
440 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3, 0) },
4d715a80
AR		 441
442 /* System Address Decoder */
dbc954dd 443 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_SAD, 0) },
4d715a80
AR		 444
445 /* Broadcast Registers */
dbc954dd
AR		 446 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR0, 1) },
447 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_BR1, 0) },
4d715a80
AR		 448
449 /* Optional, mode 2HA */
dbc954dd 450 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1, 1) },
4d715a80 451#if 0
dbc954dd
AR		 452 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TA, 1) },
453 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_RAS, 1) },
4d715a80 454#endif
dbc954dd
AR		 455 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0, 1) },
456 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1, 1) },
7d375bff
TL		 457 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2, 1) },
458 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3, 1) },
4d715a80 459
dbc954dd
AR		 460 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0, 1) },
461 { PCI_DESCR(PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0, 1) },
4d715a80
AR		 462};
463
464static const struct pci_id_table pci_dev_descr_ibridge_table[] = {
465 PCI_ID_TABLE_ENTRY(pci_dev_descr_ibridge),
466 {0,} /* 0 terminated list. */
467};
468
50d1bb93
AR		 469/* Haswell support */
470/* EN processor:
471 * - 1 IMC
472 * - 3 DDR3 channels, 2 DPC per channel
473 * EP processor:
474 * - 1 or 2 IMC
475 * - 4 DDR4 channels, 3 DPC per channel
476 * EP 4S processor:
477 * - 2 IMC
478 * - 4 DDR4 channels, 3 DPC per channel
479 * EX processor:
480 * - 2 IMC
481 * - each IMC interfaces with a SMI 2 channel
482 * - each SMI channel interfaces with a scalable memory buffer
483 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC
484 */
1f39581a 485#define HASWELL_DDRCRCLKCONTROLS 0xa10 /* Ditto on Broadwell */
50d1bb93
AR		 486#define HASWELL_HASYSDEFEATURE2 0x84
487#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC 0x2f28
488#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0 0x2fa0
489#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1 0x2f60
490#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA 0x2fa8
491#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL 0x2f71
492#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA 0x2f68
493#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_THERMAL 0x2f79
494#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0 0x2ffc
495#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1 0x2ffd
496#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0 0x2faa
497#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1 0x2fab
498#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2 0x2fac
499#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3 0x2fad
500#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 0x2f6a
501#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1 0x2f6b
502#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c
503#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d
504#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd
7179385a
AR		 505#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf
506#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9
507#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb
50d1bb93
AR		 508static const struct pci_id_descr pci_dev_descr_haswell[] = {
509 /* first item must be the HA */
510 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0) },
511
512 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0, 0) },
513 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1, 0) },
514
515 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1, 1) },
516
517 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA, 0) },
518 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL, 0) },
519 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0, 0) },
520 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1, 0) },
521 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2, 1) },
522 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1) },
523
524 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1) },
7179385a
AR		 525 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1) },
526 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1) },
527 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1) },
50d1bb93
AR		 528
529 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1) },
530 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_THERMAL, 1) },
531 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0, 1) },
532 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1, 1) },
533 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2, 1) },
534 { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3, 1) },
535};
536
537static const struct pci_id_table pci_dev_descr_haswell_table[] = {
538 PCI_ID_TABLE_ENTRY(pci_dev_descr_haswell),
539 {0,} /* 0 terminated list. */
540};
541
d0cdf900
JS		 542/* Knight's Landing Support */
543/*
544 * KNL's memory channels are swizzled between memory controllers.
545 * MC0 is mapped to CH3,5,6 and MC1 is mapped to CH0,1,2
546 */
547#define knl_channel_remap(channel) ((channel + 3) % 6)
548
549/* Memory controller, TAD tables, error injection - 2-8-0, 2-9-0 (2 of these) */
550#define PCI_DEVICE_ID_INTEL_KNL_IMC_MC 0x7840
551/* DRAM channel stuff; bank addrs, dimmmtr, etc.. 2-8-2 - 2-9-4 (6 of these) */
552#define PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL 0x7843
553/* kdrwdbu TAD limits/offsets, MCMTR - 2-10-1, 2-11-1 (2 of these) */
554#define PCI_DEVICE_ID_INTEL_KNL_IMC_TA 0x7844
555/* CHA broadcast registers, dram rules - 1-29-0 (1 of these) */
556#define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0 0x782a
557/* SAD target - 1-29-1 (1 of these) */
558#define PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1 0x782b
559/* Caching / Home Agent */
560#define PCI_DEVICE_ID_INTEL_KNL_IMC_CHA 0x782c
561/* Device with TOLM and TOHM, 0-5-0 (1 of these) */
562#define PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM 0x7810
563
564/*
565 * KNL differs from SB, IB, and Haswell in that it has multiple
566 * instances of the same device with the same device ID, so we handle that
567 * by creating as many copies in the table as we expect to find.
568 * (Like device ID must be grouped together.)
569 */
570
571static const struct pci_id_descr pci_dev_descr_knl[] = {
572 [0] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0, 0) },
573 [1] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1, 0) },
574 [2 ... 3] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_MC, 0)},
575 [4 ... 41] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHA, 0) },
576 [42 ... 47] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL, 0) },
577 [48] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TA, 0) },
578 [49] = { PCI_DESCR(PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM, 0) },
579};
580
581static const struct pci_id_table pci_dev_descr_knl_table[] = {
582 PCI_ID_TABLE_ENTRY(pci_dev_descr_knl),
583 {0,}
584};
585
1f39581a
TL		 586/*
587 * Broadwell support
588 *
589 * DE processor:
590 * - 1 IMC
591 * - 2 DDR3 channels, 2 DPC per channel
fa2ce64f
TL		 592 * EP processor:
593 * - 1 or 2 IMC
594 * - 4 DDR4 channels, 3 DPC per channel
595 * EP 4S processor:
596 * - 2 IMC
597 * - 4 DDR4 channels, 3 DPC per channel
598 * EX processor:
599 * - 2 IMC
600 * - each IMC interfaces with a SMI 2 channel
601 * - each SMI channel interfaces with a scalable memory buffer
602 * - each scalable memory buffer supports 4 DDR3/DDR4 channels, 3 DPC
1f39581a
TL		 603 */
604#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC 0x6f28
605#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0 0x6fa0
fa2ce64f 606#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1 0x6f60
1f39581a
TL		 607#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA 0x6fa8
608#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL 0x6f71
fa2ce64f
TL		 609#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA 0x6f68
610#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_THERMAL 0x6f79
1f39581a
TL		 611#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0 0x6ffc
612#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1 0x6ffd
613#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0 0x6faa
614#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1 0x6fab
615#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2 0x6fac
616#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3 0x6fad
fa2ce64f
TL		 617#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 0x6f6a
618#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1 0x6f6b
619#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2 0x6f6c
620#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3 0x6f6d
1f39581a
TL		 621#define PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0 0x6faf
622
623static const struct pci_id_descr pci_dev_descr_broadwell[] = {
624 /* first item must be the HA */
625 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0, 0) },
626
627 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0, 0) },
628 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1, 0) },
629
fa2ce64f
TL		 630 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1, 1) },
631
1f39581a
TL		 632 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA, 0) },
633 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL, 0) },
634 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0, 0) },
635 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1, 0) },
fa2ce64f
TL		 636 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2, 1) },
637 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3, 1) },
638
1f39581a 639 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0, 1) },
fa2ce64f
TL		 640
641 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA, 1) },
642 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_THERMAL, 1) },
643 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0, 1) },
644 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1, 1) },
645 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2, 1) },
646 { PCI_DESCR(PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3, 1) },
1f39581a
TL		 647};
648
649static const struct pci_id_table pci_dev_descr_broadwell_table[] = {
650 PCI_ID_TABLE_ENTRY(pci_dev_descr_broadwell),
651 {0,} /* 0 terminated list. */
652};
653
eebf11a0
MCC		 654/*
655 * pci_device_id table for which devices we are looking for
656 */
ba935f40 657static const struct pci_device_id sbridge_pci_tbl[] = {
d0585cd8 658 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0)},
4d715a80 659 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA)},
50d1bb93 660 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0)},
1f39581a 661 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0)},
d0cdf900 662 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0)},
eebf11a0
MCC		 663 {0,} /* 0 terminated list. */
664};
665
666
667/****************************************************************************
15ed103a 668 Ancillary status routines
eebf11a0
MCC		 669 ****************************************************************************/
670
50d1bb93 671static inline int numrank(enum type type, u32 mtr)
eebf11a0
MCC		 672{
673 int ranks = (1 << RANK_CNT_BITS(mtr));
50d1bb93
AR		 674 int max = 4;
675
d0cdf900 676 if (type == HASWELL || type == BROADWELL || type == KNIGHTS_LANDING)
50d1bb93 677 max = 8;
eebf11a0 678
50d1bb93
AR		 679 if (ranks > max) {
680 edac_dbg(0, "Invalid number of ranks: %d (max = %i) raw value = %x (%04x)\n",
681 ranks, max, (unsigned int)RANK_CNT_BITS(mtr), mtr);
eebf11a0
MCC		 682 return -EINVAL;
683 }
684
685 return ranks;
686}
687
688static inline int numrow(u32 mtr)
689{
690 int rows = (RANK_WIDTH_BITS(mtr) + 12);
691
692 if (rows < 13 || rows > 18) {
956b9ba1
JP		 693 edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n",
694 rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr);
eebf11a0
MCC		 695 return -EINVAL;
696 }
697
698 return 1 << rows;
699}
700
701static inline int numcol(u32 mtr)
702{
703 int cols = (COL_WIDTH_BITS(mtr) + 10);
704
705 if (cols > 12) {
956b9ba1
JP		 706 edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n",
707 cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr);
eebf11a0
MCC		 708 return -EINVAL;
709 }
710
711 return 1 << cols;
712}
713
c1979ba2 714static struct sbridge_dev *get_sbridge_dev(u8 bus, int multi_bus)
eebf11a0
MCC		 715{
716 struct sbridge_dev *sbridge_dev;
717
c1979ba2
JS		 718 /*
719 * If we have devices scattered across several busses that pertain
720 * to the same memory controller, we'll lump them all together.
721 */
722 if (multi_bus) {
723 return list_first_entry_or_null(&sbridge_edac_list,
724 struct sbridge_dev, list);
725 }
726
eebf11a0
MCC		 727 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
728 if (sbridge_dev->bus == bus)
729 return sbridge_dev;
730 }
731
732 return NULL;
733}
734
735static struct sbridge_dev *alloc_sbridge_dev(u8 bus,
736 const struct pci_id_table *table)
737{
738 struct sbridge_dev *sbridge_dev;
739
740 sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL);
741 if (!sbridge_dev)
742 return NULL;
743
744 sbridge_dev->pdev = kzalloc(sizeof(*sbridge_dev->pdev) * table->n_devs,
745 GFP_KERNEL);
746 if (!sbridge_dev->pdev) {
747 kfree(sbridge_dev);
748 return NULL;
749 }
750
751 sbridge_dev->bus = bus;
752 sbridge_dev->n_devs = table->n_devs;
753 list_add_tail(&sbridge_dev->list, &sbridge_edac_list);
754
755 return sbridge_dev;
756}
757
758static void free_sbridge_dev(struct sbridge_dev *sbridge_dev)
759{
760 list_del(&sbridge_dev->list);
761 kfree(sbridge_dev->pdev);
762 kfree(sbridge_dev);
763}
764
fb79a509
AR		 765static u64 sbridge_get_tolm(struct sbridge_pvt *pvt)
766{
767 u32 reg;
768
769 /* Address range is 32:28 */
770 pci_read_config_dword(pvt->pci_sad1, TOLM, &reg);
771 return GET_TOLM(reg);
772}
773
8fd6a43a
AR		 774static u64 sbridge_get_tohm(struct sbridge_pvt *pvt)
775{
776 u32 reg;
777
778 pci_read_config_dword(pvt->pci_sad1, TOHM, &reg);
779 return GET_TOHM(reg);
780}
781
4d715a80
AR		 782static u64 ibridge_get_tolm(struct sbridge_pvt *pvt)
783{
784 u32 reg;
785
786 pci_read_config_dword(pvt->pci_br1, TOLM, &reg);
787
788 return GET_TOLM(reg);
789}
790
791static u64 ibridge_get_tohm(struct sbridge_pvt *pvt)
792{
793 u32 reg;
794
795 pci_read_config_dword(pvt->pci_br1, TOHM, &reg);
796
797 return GET_TOHM(reg);
798}
799
b976bcf2
AR		 800static u64 rir_limit(u32 reg)
801{
802 return ((u64)GET_BITFIELD(reg, 1, 10) << 29) | 0x1fffffff;
803}
804
c59f9c06
JS		 805static u64 sad_limit(u32 reg)
806{
807 return (GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff;
808}
809
810static u32 interleave_mode(u32 reg)
811{
812 return GET_BITFIELD(reg, 1, 1);
813}
814
815char *show_interleave_mode(u32 reg)
816{
817 return interleave_mode(reg) ? "8:6" : "[8:6]XOR[18:16]";
818}
819
820static u32 dram_attr(u32 reg)
821{
822 return GET_BITFIELD(reg, 2, 3);
823}
824
d0cdf900
JS		 825static u64 knl_sad_limit(u32 reg)
826{
827 return (GET_BITFIELD(reg, 7, 26) << 26) | 0x3ffffff;
828}
829
830static u32 knl_interleave_mode(u32 reg)
831{
832 return GET_BITFIELD(reg, 1, 2);
833}
834
835static char *knl_show_interleave_mode(u32 reg)
836{
837 char *s;
838
839 switch (knl_interleave_mode(reg)) {
840 case 0:
841 s = "use address bits [8:6]";
842 break;
843 case 1:
844 s = "use address bits [10:8]";
845 break;
846 case 2:
847 s = "use address bits [14:12]";
848 break;
849 case 3:
850 s = "use address bits [32:30]";
851 break;
852 default:
853 WARN_ON(1);
854 break;
855 }
856
857 return s;
858}
859
860static u32 dram_attr_knl(u32 reg)
861{
862 return GET_BITFIELD(reg, 3, 4);
863}
864
865
9e375446
AR		 866static enum mem_type get_memory_type(struct sbridge_pvt *pvt)
867{
868 u32 reg;
869 enum mem_type mtype;
870
871 if (pvt->pci_ddrio) {
872 pci_read_config_dword(pvt->pci_ddrio, pvt->info.rankcfgr,
873 &reg);
874 if (GET_BITFIELD(reg, 11, 11))
875 /* FIXME: Can also be LRDIMM */
876 mtype = MEM_RDDR3;
877 else
878 mtype = MEM_DDR3;
879 } else
880 mtype = MEM_UNKNOWN;
881
882 return mtype;
883}
884
50d1bb93
AR		 885static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt)
886{
887 u32 reg;
888 bool registered = false;
889 enum mem_type mtype = MEM_UNKNOWN;
890
891 if (!pvt->pci_ddrio)
892 goto out;
893
894 pci_read_config_dword(pvt->pci_ddrio,
895 HASWELL_DDRCRCLKCONTROLS, &reg);
896 /* Is_Rdimm */
897 if (GET_BITFIELD(reg, 16, 16))
898 registered = true;
899
900 pci_read_config_dword(pvt->pci_ta, MCMTR, &reg);
901 if (GET_BITFIELD(reg, 14, 14)) {
902 if (registered)
903 mtype = MEM_RDDR4;
904 else
905 mtype = MEM_DDR4;
906 } else {
907 if (registered)
908 mtype = MEM_RDDR3;
909 else
910 mtype = MEM_DDR3;
911 }
912
913out:
914 return mtype;
915}
916
45f4d3ab
HC		 917static enum dev_type knl_get_width(struct sbridge_pvt *pvt, u32 mtr)
918{
919 /* for KNL value is fixed */
920 return DEV_X16;
921}
922
12f0721c
AR		 923static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr)
924{
925 /* there's no way to figure out */
926 return DEV_UNKNOWN;
927}
928
929static enum dev_type __ibridge_get_width(u32 mtr)
930{
931 enum dev_type type;
932
933 switch (mtr) {
934 case 3:
935 type = DEV_UNKNOWN;
936 break;
937 case 2:
938 type = DEV_X16;
939 break;
940 case 1:
941 type = DEV_X8;
942 break;
943 case 0:
944 type = DEV_X4;
945 break;
946 }
947
948 return type;
949}
950
951static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr)
952{
953 /*
954 * ddr3_width on the documentation but also valid for DDR4 on
955 * Haswell
956 */
957 return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8));
958}
959
960static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr)
961{
962 /* ddr3_width on the documentation but also valid for DDR4 */
963 return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9));
964}
965
d0cdf900
JS		 966static enum mem_type knl_get_memory_type(struct sbridge_pvt *pvt)
967{
968 /* DDR4 RDIMMS and LRDIMMS are supported */
969 return MEM_RDDR4;
970}
971
f14d6892
AR		 972static u8 get_node_id(struct sbridge_pvt *pvt)
973{
974 u32 reg;
975 pci_read_config_dword(pvt->pci_br0, SAD_CONTROL, &reg);
976 return GET_BITFIELD(reg, 0, 2);
977}
978
50d1bb93
AR		 979static u8 haswell_get_node_id(struct sbridge_pvt *pvt)
980{
981 u32 reg;
982
983 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, &reg);
984 return GET_BITFIELD(reg, 0, 3);
985}
986
d0cdf900
JS		 987static u8 knl_get_node_id(struct sbridge_pvt *pvt)
988{
989 u32 reg;
990
991 pci_read_config_dword(pvt->pci_sad1, SAD_CONTROL, &reg);
992 return GET_BITFIELD(reg, 0, 2);
993}
994
995
50d1bb93
AR		 996static u64 haswell_get_tolm(struct sbridge_pvt *pvt)
997{
998 u32 reg;
999
f7cf2a22
TL		 1000 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOLM, &reg);
1001 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff;
50d1bb93
AR		 1002}
1003
1004static u64 haswell_get_tohm(struct sbridge_pvt *pvt)
1005{
1006 u64 rc;
1007 u32 reg;
1008
1009 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_0, &reg);
1010 rc = GET_BITFIELD(reg, 26, 31);
1011 pci_read_config_dword(pvt->info.pci_vtd, HASWELL_TOHM_1, &reg);
1012 rc = ((reg << 6) | rc) << 26;
1013
1014 return rc | 0x1ffffff;
1015}
1016
d0cdf900
JS		 1017static u64 knl_get_tolm(struct sbridge_pvt *pvt)
1018{
1019 u32 reg;
1020
1021 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOLM, &reg);
1022 return (GET_BITFIELD(reg, 26, 31) << 26) | 0x3ffffff;
1023}
1024
1025static u64 knl_get_tohm(struct sbridge_pvt *pvt)
1026{
1027 u64 rc;
1028 u32 reg_lo, reg_hi;
1029
1030 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_0, &reg_lo);
1031 pci_read_config_dword(pvt->knl.pci_mc_info, KNL_TOHM_1, &reg_hi);
1032 rc = ((u64)reg_hi << 32) | reg_lo;
1033 return rc | 0x3ffffff;
1034}
1035
1036
50d1bb93
AR		 1037static u64 haswell_rir_limit(u32 reg)
1038{
1039 return (((u64)GET_BITFIELD(reg, 1, 11) + 1) << 29) - 1;
1040}
1041
4d715a80
AR		 1042static inline u8 sad_pkg_socket(u8 pkg)
1043{
1044 /* on Ivy Bridge, nodeID is SASS, where A is HA and S is node id */
2ff3a308 1045 return ((pkg >> 3) << 2) | (pkg & 0x3);
4d715a80
AR		 1046}
1047
1048static inline u8 sad_pkg_ha(u8 pkg)
1049{
1050 return (pkg >> 2) & 0x1;
1051}
1052
eebf11a0
MCC		 1053/****************************************************************************
1054 Memory check routines
1055 ****************************************************************************/
dbc954dd 1056static struct pci_dev *get_pdev_same_bus(u8 bus, u32 id)
eebf11a0 1057{
dbc954dd 1058 struct pci_dev *pdev = NULL;
eebf11a0 1059
dbc954dd
AR		 1060 do {
1061 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, id, pdev);
1062 if (pdev && pdev->bus->number == bus)
1063 break;
1064 } while (pdev);
eebf11a0 1065
dbc954dd 1066 return pdev;
eebf11a0
MCC		 1067}
1068
1069/**
c36e3e77 1070 * check_if_ecc_is_active() - Checks if ECC is active
50d1bb93
AR		 1071 * @bus: Device bus
1072 * @type: Memory controller type
1073 * returns: 0 in case ECC is active, -ENODEV if it can't be determined or
1074 * disabled
eebf11a0 1075 */
dbc954dd 1076static int check_if_ecc_is_active(const u8 bus, enum type type)
eebf11a0
MCC		 1077{
1078 struct pci_dev *pdev = NULL;
dbc954dd 1079 u32 mcmtr, id;
eebf11a0 1080
1f39581a
TL		 1081 switch (type) {
1082 case IVY_BRIDGE:
dbc954dd 1083 id = PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA;
1f39581a
TL		 1084 break;
1085 case HASWELL:
50d1bb93 1086 id = PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA;
1f39581a
TL		 1087 break;
1088 case SANDY_BRIDGE:
dbc954dd 1089 id = PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA;
1f39581a
TL		 1090 break;
1091 case BROADWELL:
1092 id = PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA;
1093 break;
d0cdf900
JS		 1094 case KNIGHTS_LANDING:
1095 /*
1096 * KNL doesn't group things by bus the same way
1097 * SB/IB/Haswell does.
1098 */
1099 id = PCI_DEVICE_ID_INTEL_KNL_IMC_TA;
1100 break;
1f39581a
TL		 1101 default:
1102 return -ENODEV;
1103 }
dbc954dd 1104
d0cdf900
JS		 1105 if (type != KNIGHTS_LANDING)
1106 pdev = get_pdev_same_bus(bus, id);
1107 else
1108 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, id, 0);
1109
eebf11a0
MCC		 1110 if (!pdev) {
1111 sbridge_printk(KERN_ERR, "Couldn't find PCI device "
dbc954dd
AR		 1112 "%04x:%04x! on bus %02d\n",
1113 PCI_VENDOR_ID_INTEL, id, bus);
eebf11a0
MCC		 1114 return -ENODEV;
1115 }
1116
d0cdf900
JS		 1117 pci_read_config_dword(pdev,
1118 type == KNIGHTS_LANDING ? KNL_MCMTR : MCMTR, &mcmtr);
eebf11a0
MCC		 1119 if (!IS_ECC_ENABLED(mcmtr)) {
1120 sbridge_printk(KERN_ERR, "ECC is disabled. Aborting\n");
1121 return -ENODEV;
1122 }
eebf11a0
MCC		 1123 return 0;
1124}
1125
d0cdf900
JS		 1126/* Low bits of TAD limit, and some metadata. */
1127static const u32 knl_tad_dram_limit_lo[] = {
1128 0x400, 0x500, 0x600, 0x700,
1129 0x800, 0x900, 0xa00, 0xb00,
1130};
1131
1132/* Low bits of TAD offset. */
1133static const u32 knl_tad_dram_offset_lo[] = {
1134 0x404, 0x504, 0x604, 0x704,
1135 0x804, 0x904, 0xa04, 0xb04,
1136};
1137
1138/* High 16 bits of TAD limit and offset. */
1139static const u32 knl_tad_dram_hi[] = {
1140 0x408, 0x508, 0x608, 0x708,
1141 0x808, 0x908, 0xa08, 0xb08,
1142};
1143
1144/* Number of ways a tad entry is interleaved. */
1145static const u32 knl_tad_ways[] = {
1146 8, 6, 4, 3, 2, 1,
1147};
1148
1149/*
1150 * Retrieve the n'th Target Address Decode table entry
1151 * from the memory controller's TAD table.
1152 *
1153 * @pvt: driver private data
1154 * @entry: which entry you want to retrieve
1155 * @mc: which memory controller (0 or 1)
1156 * @offset: output tad range offset
1157 * @limit: output address of first byte above tad range
1158 * @ways: output number of interleave ways
1159 *
1160 * The offset value has curious semantics. It's a sort of running total
1161 * of the sizes of all the memory regions that aren't mapped in this
1162 * tad table.
1163 */
1164static int knl_get_tad(const struct sbridge_pvt *pvt,
1165 const int entry,
1166 const int mc,
1167 u64 *offset,
1168 u64 *limit,
1169 int *ways)
1170{
1171 u32 reg_limit_lo, reg_offset_lo, reg_hi;
1172 struct pci_dev *pci_mc;
1173 int way_id;
1174
1175 switch (mc) {
1176 case 0:
1177 pci_mc = pvt->knl.pci_mc0;
1178 break;
1179 case 1:
1180 pci_mc = pvt->knl.pci_mc1;
1181 break;
1182 default:
1183 WARN_ON(1);
1184 return -EINVAL;
1185 }
1186
1187 pci_read_config_dword(pci_mc,
1188 knl_tad_dram_limit_lo[entry], &reg_limit_lo);
1189 pci_read_config_dword(pci_mc,
1190 knl_tad_dram_offset_lo[entry], &reg_offset_lo);
1191 pci_read_config_dword(pci_mc,
1192 knl_tad_dram_hi[entry], &reg_hi);
1193
1194 /* Is this TAD entry enabled? */
1195 if (!GET_BITFIELD(reg_limit_lo, 0, 0))
1196 return -ENODEV;
1197
1198 way_id = GET_BITFIELD(reg_limit_lo, 3, 5);
1199
1200 if (way_id < ARRAY_SIZE(knl_tad_ways)) {
1201 *ways = knl_tad_ways[way_id];
1202 } else {
1203 *ways = 0;
1204 sbridge_printk(KERN_ERR,
1205 "Unexpected value %d in mc_tad_limit_lo wayness field\n",
1206 way_id);
1207 return -ENODEV;
1208 }
1209
1210 /*
1211 * The least significant 6 bits of base and limit are truncated.
1212 * For limit, we fill the missing bits with 1s.
1213 */
1214 *offset = ((u64) GET_BITFIELD(reg_offset_lo, 6, 31) << 6) |
1215 ((u64) GET_BITFIELD(reg_hi, 0, 15) << 32);
1216 *limit = ((u64) GET_BITFIELD(reg_limit_lo, 6, 31) << 6) | 63 |
1217 ((u64) GET_BITFIELD(reg_hi, 16, 31) << 32);
1218
1219 return 0;
1220}
1221
1222/* Determine which memory controller is responsible for a given channel. */
1223static int knl_channel_mc(int channel)
1224{
1225 WARN_ON(channel < 0 || channel >= 6);
1226
1227 return channel < 3 ? 1 : 0;
1228}
1229
1230/*
1231 * Get the Nth entry from EDC_ROUTE_TABLE register.
1232 * (This is the per-tile mapping of logical interleave targets to
1233 * physical EDC modules.)
1234 *
1235 * entry 0: 0:2
1236 * 1: 3:5
1237 * 2: 6:8
1238 * 3: 9:11
1239 * 4: 12:14
1240 * 5: 15:17
1241 * 6: 18:20
1242 * 7: 21:23
1243 * reserved: 24:31
1244 */
1245static u32 knl_get_edc_route(int entry, u32 reg)
1246{
1247 WARN_ON(entry >= KNL_MAX_EDCS);
1248 return GET_BITFIELD(reg, entry*3, (entry*3)+2);
1249}
1250
1251/*
1252 * Get the Nth entry from MC_ROUTE_TABLE register.
1253 * (This is the per-tile mapping of logical interleave targets to
1254 * physical DRAM channels modules.)
1255 *
1256 * entry 0: mc 0:2 channel 18:19
1257 * 1: mc 3:5 channel 20:21
1258 * 2: mc 6:8 channel 22:23
1259 * 3: mc 9:11 channel 24:25
1260 * 4: mc 12:14 channel 26:27
1261 * 5: mc 15:17 channel 28:29
1262 * reserved: 30:31
1263 *
1264 * Though we have 3 bits to identify the MC, we should only see
1265 * the values 0 or 1.
1266 */
1267
1268static u32 knl_get_mc_route(int entry, u32 reg)
1269{
1270 int mc, chan;
1271
1272 WARN_ON(entry >= KNL_MAX_CHANNELS);
1273
1274 mc = GET_BITFIELD(reg, entry*3, (entry*3)+2);
1275 chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1);
1276
1277 return knl_channel_remap(mc*3 + chan);
1278}
1279
1280/*
1281 * Render the EDC_ROUTE register in human-readable form.
1282 * Output string s should be at least KNL_MAX_EDCS*2 bytes.
1283 */
1284static void knl_show_edc_route(u32 reg, char *s)
1285{
1286 int i;
1287
1288 for (i = 0; i < KNL_MAX_EDCS; i++) {
1289 s[i*2] = knl_get_edc_route(i, reg) + '0';
1290 s[i*2+1] = '-';
1291 }
1292
1293 s[KNL_MAX_EDCS*2 - 1] = '\0';
1294}
1295
1296/*
1297 * Render the MC_ROUTE register in human-readable form.
1298 * Output string s should be at least KNL_MAX_CHANNELS*2 bytes.
1299 */
1300static void knl_show_mc_route(u32 reg, char *s)
1301{
1302 int i;
1303
1304 for (i = 0; i < KNL_MAX_CHANNELS; i++) {
1305 s[i*2] = knl_get_mc_route(i, reg) + '0';
1306 s[i*2+1] = '-';
1307 }
1308
1309 s[KNL_MAX_CHANNELS*2 - 1] = '\0';
1310}
1311
1312#define KNL_EDC_ROUTE 0xb8
1313#define KNL_MC_ROUTE 0xb4
1314
1315/* Is this dram rule backed by regular DRAM in flat mode? */
1316#define KNL_EDRAM(reg) GET_BITFIELD(reg, 29, 29)
1317
1318/* Is this dram rule cached? */
1319#define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28)
1320
1321/* Is this rule backed by edc ? */
1322#define KNL_EDRAM_ONLY(reg) GET_BITFIELD(reg, 29, 29)
1323
1324/* Is this rule backed by DRAM, cacheable in EDRAM? */
1325#define KNL_CACHEABLE(reg) GET_BITFIELD(reg, 28, 28)
1326
1327/* Is this rule mod3? */
1328#define KNL_MOD3(reg) GET_BITFIELD(reg, 27, 27)
1329
1330/*
1331 * Figure out how big our RAM modules are.
1332 *
1333 * The DIMMMTR register in KNL doesn't tell us the size of the DIMMs, so we
1334 * have to figure this out from the SAD rules, interleave lists, route tables,
1335 * and TAD rules.
1336 *
1337 * SAD rules can have holes in them (e.g. the 3G-4G hole), so we have to
1338 * inspect the TAD rules to figure out how large the SAD regions really are.
1339 *
1340 * When we know the real size of a SAD region and how many ways it's
1341 * interleaved, we know the individual contribution of each channel to
1342 * TAD is size/ways.
1343 *
1344 * Finally, we have to check whether each channel participates in each SAD
1345 * region.
1346 *
1347 * Fortunately, KNL only supports one DIMM per channel, so once we know how
1348 * much memory the channel uses, we know the DIMM is at least that large.
1349 * (The BIOS might possibly choose not to map all available memory, in which
1350 * case we will underreport the size of the DIMM.)
1351 *
1352 * In theory, we could try to determine the EDC sizes as well, but that would
1353 * only work in flat mode, not in cache mode.
1354 *
1355 * @mc_sizes: Output sizes of channels (must have space for KNL_MAX_CHANNELS
1356 * elements)
1357 */
1358static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes)
1359{
1360 u64 sad_base, sad_size, sad_limit = 0;
1361 u64 tad_base, tad_size, tad_limit, tad_deadspace, tad_livespace;
1362 int sad_rule = 0;
1363 int tad_rule = 0;
1364 int intrlv_ways, tad_ways;
1365 u32 first_pkg, pkg;
1366 int i;
1367 u64 sad_actual_size[2]; /* sad size accounting for holes, per mc */
1368 u32 dram_rule, interleave_reg;
1369 u32 mc_route_reg[KNL_MAX_CHAS];
1370 u32 edc_route_reg[KNL_MAX_CHAS];
1371 int edram_only;
1372 char edc_route_string[KNL_MAX_EDCS*2];
1373 char mc_route_string[KNL_MAX_CHANNELS*2];
1374 int cur_reg_start;
1375 int mc;
1376 int channel;
1377 int way;
1378 int participants[KNL_MAX_CHANNELS];
1379 int participant_count = 0;
1380
1381 for (i = 0; i < KNL_MAX_CHANNELS; i++)
1382 mc_sizes[i] = 0;
1383
1384 /* Read the EDC route table in each CHA. */
1385 cur_reg_start = 0;
1386 for (i = 0; i < KNL_MAX_CHAS; i++) {
1387 pci_read_config_dword(pvt->knl.pci_cha[i],
1388 KNL_EDC_ROUTE, &edc_route_reg[i]);
1389
1390 if (i > 0 && edc_route_reg[i] != edc_route_reg[i-1]) {
1391 knl_show_edc_route(edc_route_reg[i-1],
1392 edc_route_string);
1393 if (cur_reg_start == i-1)
1394 edac_dbg(0, "edc route table for CHA %d: %s\n",
1395 cur_reg_start, edc_route_string);
1396 else
1397 edac_dbg(0, "edc route table for CHA %d-%d: %s\n",
1398 cur_reg_start, i-1, edc_route_string);
1399 cur_reg_start = i;
1400 }
1401 }
1402 knl_show_edc_route(edc_route_reg[i-1], edc_route_string);
1403 if (cur_reg_start == i-1)
1404 edac_dbg(0, "edc route table for CHA %d: %s\n",
1405 cur_reg_start, edc_route_string);
1406 else
1407 edac_dbg(0, "edc route table for CHA %d-%d: %s\n",
1408 cur_reg_start, i-1, edc_route_string);
1409
1410 /* Read the MC route table in each CHA. */
1411 cur_reg_start = 0;
1412 for (i = 0; i < KNL_MAX_CHAS; i++) {
1413 pci_read_config_dword(pvt->knl.pci_cha[i],
1414 KNL_MC_ROUTE, &mc_route_reg[i]);
1415
1416 if (i > 0 && mc_route_reg[i] != mc_route_reg[i-1]) {
1417 knl_show_mc_route(mc_route_reg[i-1], mc_route_string);
1418 if (cur_reg_start == i-1)
1419 edac_dbg(0, "mc route table for CHA %d: %s\n",
1420 cur_reg_start, mc_route_string);
1421 else
1422 edac_dbg(0, "mc route table for CHA %d-%d: %s\n",
1423 cur_reg_start, i-1, mc_route_string);
1424 cur_reg_start = i;
1425 }
1426 }
1427 knl_show_mc_route(mc_route_reg[i-1], mc_route_string);
1428 if (cur_reg_start == i-1)
1429 edac_dbg(0, "mc route table for CHA %d: %s\n",
1430 cur_reg_start, mc_route_string);
1431 else
1432 edac_dbg(0, "mc route table for CHA %d-%d: %s\n",
1433 cur_reg_start, i-1, mc_route_string);
1434
1435 /* Process DRAM rules */
1436 for (sad_rule = 0; sad_rule < pvt->info.max_sad; sad_rule++) {
1437 /* previous limit becomes the new base */
1438 sad_base = sad_limit;
1439
1440 pci_read_config_dword(pvt->pci_sad0,
1441 pvt->info.dram_rule[sad_rule], &dram_rule);
1442
1443 if (!DRAM_RULE_ENABLE(dram_rule))
1444 break;
1445
1446 edram_only = KNL_EDRAM_ONLY(dram_rule);
1447
1448 sad_limit = pvt->info.sad_limit(dram_rule)+1;
1449 sad_size = sad_limit - sad_base;
1450
1451 pci_read_config_dword(pvt->pci_sad0,
1452 pvt->info.interleave_list[sad_rule], &interleave_reg);
1453
1454 /*
1455 * Find out how many ways this dram rule is interleaved.
1456 * We stop when we see the first channel again.
1457 */
1458 first_pkg = sad_pkg(pvt->info.interleave_pkg,
1459 interleave_reg, 0);
1460 for (intrlv_ways = 1; intrlv_ways < 8; intrlv_ways++) {
1461 pkg = sad_pkg(pvt->info.interleave_pkg,
1462 interleave_reg, intrlv_ways);
1463
1464 if ((pkg & 0x8) == 0) {
1465 /*
1466 * 0 bit means memory is non-local,
1467 * which KNL doesn't support
1468 */
1469 edac_dbg(0, "Unexpected interleave target %d\n",
1470 pkg);
1471 return -1;
1472 }
1473
1474 if (pkg == first_pkg)
1475 break;
1476 }
1477 if (KNL_MOD3(dram_rule))
1478 intrlv_ways *= 3;
1479
1480 edac_dbg(3, "dram rule %d (base 0x%llx, limit 0x%llx), %d way interleave%s\n",
1481 sad_rule,
1482 sad_base,
1483 sad_limit,
1484 intrlv_ways,
1485 edram_only ? ", EDRAM" : "");
1486
1487 /*
1488 * Find out how big the SAD region really is by iterating
1489 * over TAD tables (SAD regions may contain holes).
1490 * Each memory controller might have a different TAD table, so
1491 * we have to look at both.
1492 *
1493 * Livespace is the memory that's mapped in this TAD table,
1494 * deadspace is the holes (this could be the MMIO hole, or it
1495 * could be memory that's mapped by the other TAD table but
1496 * not this one).
1497 */
1498 for (mc = 0; mc < 2; mc++) {
1499 sad_actual_size[mc] = 0;
1500 tad_livespace = 0;
1501 for (tad_rule = 0;
1502 tad_rule < ARRAY_SIZE(
1503 knl_tad_dram_limit_lo);
1504 tad_rule++) {
1505 if (knl_get_tad(pvt,
1506 tad_rule,
1507 mc,
1508 &tad_deadspace,
1509 &tad_limit,
1510 &tad_ways))
1511 break;
1512
1513 tad_size = (tad_limit+1) -
1514 (tad_livespace + tad_deadspace);
1515 tad_livespace += tad_size;
1516 tad_base = (tad_limit+1) - tad_size;
1517
1518 if (tad_base < sad_base) {
1519 if (tad_limit > sad_base)
1520 edac_dbg(0, "TAD region overlaps lower SAD boundary -- TAD tables may be configured incorrectly.\n");
1521 } else if (tad_base < sad_limit) {
1522 if (tad_limit+1 > sad_limit) {
1523 edac_dbg(0, "TAD region overlaps upper SAD boundary -- TAD tables may be configured incorrectly.\n");
1524 } else {
1525 /* TAD region is completely inside SAD region */
1526 edac_dbg(3, "TAD region %d 0x%llx - 0x%llx (%lld bytes) table%d\n",
1527 tad_rule, tad_base,
1528 tad_limit, tad_size,
1529 mc);
1530 sad_actual_size[mc] += tad_size;
1531 }
1532 }
1533 tad_base = tad_limit+1;
1534 }
1535 }
1536
1537 for (mc = 0; mc < 2; mc++) {
1538 edac_dbg(3, " total TAD DRAM footprint in table%d : 0x%llx (%lld bytes)\n",
1539 mc, sad_actual_size[mc], sad_actual_size[mc]);
1540 }
1541
1542 /* Ignore EDRAM rule */
1543 if (edram_only)
1544 continue;
1545
1546 /* Figure out which channels participate in interleave. */
1547 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++)
1548 participants[channel] = 0;
1549
1550 /* For each channel, does at least one CHA have
1551 * this channel mapped to the given target?
1552 */
1553 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) {
1554 for (way = 0; way < intrlv_ways; way++) {
1555 int target;
1556 int cha;
1557
1558 if (KNL_MOD3(dram_rule))
1559 target = way;
1560 else
1561 target = 0x7 & sad_pkg(
1562 pvt->info.interleave_pkg, interleave_reg, way);
1563
1564 for (cha = 0; cha < KNL_MAX_CHAS; cha++) {
1565 if (knl_get_mc_route(target,
1566 mc_route_reg[cha]) == channel
83bdaad4 1567 && !participants[channel]) {
d0cdf900
JS		 1568 participant_count++;
1569 participants[channel] = 1;
1570 break;
1571 }
1572 }
1573 }
1574 }
1575
1576 if (participant_count != intrlv_ways)
1577 edac_dbg(0, "participant_count (%d) != interleave_ways (%d): DIMM size may be incorrect\n",
1578 participant_count, intrlv_ways);
1579
1580 for (channel = 0; channel < KNL_MAX_CHANNELS; channel++) {
1581 mc = knl_channel_mc(channel);
1582 if (participants[channel]) {
1583 edac_dbg(4, "mc channel %d contributes %lld bytes via sad entry %d\n",
1584 channel,
1585 sad_actual_size[mc]/intrlv_ways,
1586 sad_rule);
1587 mc_sizes[channel] +=
1588 sad_actual_size[mc]/intrlv_ways;
1589 }
1590 }
1591 }
1592
1593 return 0;
1594}
1595
084a4fcc 1596static int get_dimm_config(struct mem_ctl_info *mci)
eebf11a0
MCC		 1597{
1598 struct sbridge_pvt *pvt = mci->pvt_info;
c36e3e77 1599 struct dimm_info *dimm;
deb09dda
MCC		 1600 unsigned i, j, banks, ranks, rows, cols, npages;
1601 u64 size;
eebf11a0
MCC		 1602 u32 reg;
1603 enum edac_type mode;
c6e13b52 1604 enum mem_type mtype;
d0cdf900
JS		 1605 int channels = pvt->info.type == KNIGHTS_LANDING ?
1606 KNL_MAX_CHANNELS : NUM_CHANNELS;
1607 u64 knl_mc_sizes[KNL_MAX_CHANNELS];
eebf11a0 1608
d0cdf900
JS		 1609 if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL ||
1610 pvt->info.type == KNIGHTS_LANDING)
50d1bb93
AR		 1611 pci_read_config_dword(pvt->pci_sad1, SAD_TARGET, &reg);
1612 else
1613 pci_read_config_dword(pvt->pci_br0, SAD_TARGET, &reg);
1614
d0cdf900
JS		 1615 if (pvt->info.type == KNIGHTS_LANDING)
1616 pvt->sbridge_dev->source_id = SOURCE_ID_KNL(reg);
1617 else
1618 pvt->sbridge_dev->source_id = SOURCE_ID(reg);
eebf11a0 1619
f14d6892 1620 pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt);
956b9ba1
JP		 1621 edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n",
1622 pvt->sbridge_dev->mc,
1623 pvt->sbridge_dev->node_id,
1624 pvt->sbridge_dev->source_id);
eebf11a0 1625
d0cdf900
JS		 1626 /* KNL doesn't support mirroring or lockstep,
1627 * and is always closed page
1628 */
1629 if (pvt->info.type == KNIGHTS_LANDING) {
1630 mode = EDAC_S4ECD4ED;
eebf11a0 1631 pvt->is_mirrored = false;
eebf11a0 1632
d0cdf900
JS		 1633 if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0)
1634 return -1;
eebf11a0 1635 } else {
d0cdf900
JS		 1636 pci_read_config_dword(pvt->pci_ras, RASENABLES, &reg);
1637 if (IS_MIRROR_ENABLED(reg)) {
1638 edac_dbg(0, "Memory mirror is enabled\n");
1639 pvt->is_mirrored = true;
1640 } else {
1641 edac_dbg(0, "Memory mirror is disabled\n");
1642 pvt->is_mirrored = false;
1643 }
1644
1645 pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr);
1646 if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) {
1647 edac_dbg(0, "Lockstep is enabled\n");
1648 mode = EDAC_S8ECD8ED;
1649 pvt->is_lockstep = true;
1650 } else {
1651 edac_dbg(0, "Lockstep is disabled\n");
1652 mode = EDAC_S4ECD4ED;
1653 pvt->is_lockstep = false;
1654 }
1655 if (IS_CLOSE_PG(pvt->info.mcmtr)) {
1656 edac_dbg(0, "address map is on closed page mode\n");
1657 pvt->is_close_pg = true;
1658 } else {
1659 edac_dbg(0, "address map is on open page mode\n");
1660 pvt->is_close_pg = false;
1661 }
eebf11a0
MCC		 1662 }
1663
9e375446 1664 mtype = pvt->info.get_memory_type(pvt);
50d1bb93 1665 if (mtype == MEM_RDDR3 || mtype == MEM_RDDR4)
9e375446
AR		 1666 edac_dbg(0, "Memory is registered\n");
1667 else if (mtype == MEM_UNKNOWN)
de4772c6 1668 edac_dbg(0, "Cannot determine memory type\n");
9e375446
AR		 1669 else
1670 edac_dbg(0, "Memory is unregistered\n");
eebf11a0 1671
fec53af5 1672 if (mtype == MEM_DDR4 || mtype == MEM_RDDR4)
50d1bb93
AR		 1673 banks = 16;
1674 else
1675 banks = 8;
eebf11a0 1676
d0cdf900 1677 for (i = 0; i < channels; i++) {
eebf11a0
MCC		 1678 u32 mtr;
1679
d0cdf900
JS		 1680 int max_dimms_per_channel;
1681
1682 if (pvt->info.type == KNIGHTS_LANDING) {
1683 max_dimms_per_channel = 1;
1684 if (!pvt->knl.pci_channel[i])
1685 continue;
1686 } else {
1687 max_dimms_per_channel = ARRAY_SIZE(mtr_regs);
1688 if (!pvt->pci_tad[i])
1689 continue;
1690 }
1691
1692 for (j = 0; j < max_dimms_per_channel; j++) {
c36e3e77
MCC		 1693 dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
1694 i, j, 0);
d0cdf900
JS		 1695 if (pvt->info.type == KNIGHTS_LANDING) {
1696 pci_read_config_dword(pvt->knl.pci_channel[i],
1697 knl_mtr_reg, &mtr);
1698 } else {
1699 pci_read_config_dword(pvt->pci_tad[i],
1700 mtr_regs[j], &mtr);
1701 }
956b9ba1 1702 edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr);
eebf11a0
MCC		 1703 if (IS_DIMM_PRESENT(mtr)) {
1704 pvt->channel[i].dimms++;
1705
50d1bb93 1706 ranks = numrank(pvt->info.type, mtr);
d0cdf900
JS		 1707
1708 if (pvt->info.type == KNIGHTS_LANDING) {
1709 /* For DDR4, this is fixed. */
1710 cols = 1 << 10;
1711 rows = knl_mc_sizes[i] /
1712 ((u64) cols * ranks * banks * 8);
1713 } else {
1714 rows = numrow(mtr);
1715 cols = numcol(mtr);
1716 }
eebf11a0 1717
deb09dda 1718 size = ((u64)rows * cols * banks * ranks) >> (20 - 3);
eebf11a0
MCC		 1719 npages = MiB_TO_PAGES(size);
1720
7d375bff
TL		 1721 edac_dbg(0, "mc#%d: ha %d channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n",
1722 pvt->sbridge_dev->mc, i/4, i%4, j,
956b9ba1
JP		 1723 size, npages,
1724 banks, ranks, rows, cols);
eebf11a0 1725
a895bf8b 1726 dimm->nr_pages = npages;
084a4fcc 1727 dimm->grain = 32;
12f0721c 1728 dimm->dtype = pvt->info.get_width(pvt, mtr);
084a4fcc
MCC		 1729 dimm->mtype = mtype;
1730 dimm->edac_mode = mode;
1731 snprintf(dimm->label, sizeof(dimm->label),
7d375bff
TL		 1732 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u",
1733 pvt->sbridge_dev->source_id, i/4, i%4, j);
eebf11a0
MCC		 1734 }
1735 }
1736 }
1737
1738 return 0;
1739}
1740
1741static void get_memory_layout(const struct mem_ctl_info *mci)
1742{
1743 struct sbridge_pvt *pvt = mci->pvt_info;
1744 int i, j, k, n_sads, n_tads, sad_interl;
1745 u32 reg;
1746 u64 limit, prv = 0;
1747 u64 tmp_mb;
8c009100 1748 u32 gb, mb;
eebf11a0
MCC		 1749 u32 rir_way;
1750
1751 /*
1752 * Step 1) Get TOLM/TOHM ranges
1753 */
1754
fb79a509 1755 pvt->tolm = pvt->info.get_tolm(pvt);
eebf11a0
MCC		 1756 tmp_mb = (1 + pvt->tolm) >> 20;
1757
8c009100
JS		 1758 gb = div_u64_rem(tmp_mb, 1024, &mb);
1759 edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n",
1760 gb, (mb*1000)/1024, (u64)pvt->tolm);
eebf11a0
MCC		 1761
1762 /* Address range is already 45:25 */
8fd6a43a 1763 pvt->tohm = pvt->info.get_tohm(pvt);
eebf11a0
MCC		 1764 tmp_mb = (1 + pvt->tohm) >> 20;
1765
8c009100
JS		 1766 gb = div_u64_rem(tmp_mb, 1024, &mb);
1767 edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n",
1768 gb, (mb*1000)/1024, (u64)pvt->tohm);
eebf11a0
MCC		 1769
1770 /*
1771 * Step 2) Get SAD range and SAD Interleave list
1772 * TAD registers contain the interleave wayness. However, it
1773 * seems simpler to just discover it indirectly, with the
1774 * algorithm bellow.
1775 */
1776 prv = 0;
464f1d82 1777 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) {
eebf11a0 1778 /* SAD_LIMIT Address range is 45:26 */
464f1d82 1779 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads],
eebf11a0 1780 &reg);
c59f9c06 1781 limit = pvt->info.sad_limit(reg);
eebf11a0
MCC		 1782
1783 if (!DRAM_RULE_ENABLE(reg))
1784 continue;
1785
1786 if (limit <= prv)
1787 break;
1788
1789 tmp_mb = (limit + 1) >> 20;
8c009100 1790 gb = div_u64_rem(tmp_mb, 1024, &mb);
956b9ba1
JP		 1791 edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n",
1792 n_sads,
c59f9c06 1793 show_dram_attr(pvt->info.dram_attr(reg)),
8c009100 1794 gb, (mb*1000)/1024,
956b9ba1 1795 ((u64)tmp_mb) << 20L,
c59f9c06 1796 pvt->info.show_interleave_mode(reg),
956b9ba1 1797 reg);
eebf11a0
MCC		 1798 prv = limit;
1799
ef1ce51e 1800 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads],
eebf11a0 1801 &reg);
cc311991 1802 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0);
eebf11a0 1803 for (j = 0; j < 8; j++) {
cc311991
AR		 1804 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, j);
1805 if (j > 0 && sad_interl == pkg)
eebf11a0
MCC		 1806 break;
1807
956b9ba1 1808 edac_dbg(0, "SAD#%d, interleave #%d: %d\n",
cc311991 1809 n_sads, j, pkg);
eebf11a0
MCC		 1810 }
1811 }
1812
d0cdf900
JS		 1813 if (pvt->info.type == KNIGHTS_LANDING)
1814 return;
1815
eebf11a0
MCC		 1816 /*
1817 * Step 3) Get TAD range
1818 */
1819 prv = 0;
1820 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
1821 pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
1822 &reg);
1823 limit = TAD_LIMIT(reg);
1824 if (limit <= prv)
1825 break;
1826 tmp_mb = (limit + 1) >> 20;
1827
8c009100 1828 gb = div_u64_rem(tmp_mb, 1024, &mb);
956b9ba1 1829 edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n",
8c009100 1830 n_tads, gb, (mb*1000)/1024,
956b9ba1 1831 ((u64)tmp_mb) << 20L,
eb1af3b7
LT		 1832 (u32)(1 << TAD_SOCK(reg)),
1833 (u32)TAD_CH(reg) + 1,
956b9ba1
JP		 1834 (u32)TAD_TGT0(reg),
1835 (u32)TAD_TGT1(reg),
1836 (u32)TAD_TGT2(reg),
1837 (u32)TAD_TGT3(reg),
1838 reg);
7fae0db4 1839 prv = limit;
eebf11a0
MCC		 1840 }
1841
1842 /*
1843 * Step 4) Get TAD offsets, per each channel
1844 */
1845 for (i = 0; i < NUM_CHANNELS; i++) {
1846 if (!pvt->channel[i].dimms)
1847 continue;
1848 for (j = 0; j < n_tads; j++) {
1849 pci_read_config_dword(pvt->pci_tad[i],
1850 tad_ch_nilv_offset[j],
1851 &reg);
1852 tmp_mb = TAD_OFFSET(reg) >> 20;
8c009100 1853 gb = div_u64_rem(tmp_mb, 1024, &mb);
956b9ba1
JP		 1854 edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n",
1855 i, j,
8c009100 1856 gb, (mb*1000)/1024,
956b9ba1
JP		 1857 ((u64)tmp_mb) << 20L,
1858 reg);
eebf11a0
MCC		 1859 }
1860 }
1861
1862 /*
1863 * Step 6) Get RIR Wayness/Limit, per each channel
1864 */
1865 for (i = 0; i < NUM_CHANNELS; i++) {
1866 if (!pvt->channel[i].dimms)
1867 continue;
1868 for (j = 0; j < MAX_RIR_RANGES; j++) {
1869 pci_read_config_dword(pvt->pci_tad[i],
1870 rir_way_limit[j],
1871 &reg);
1872
1873 if (!IS_RIR_VALID(reg))
1874 continue;
1875
b976bcf2 1876 tmp_mb = pvt->info.rir_limit(reg) >> 20;
eebf11a0 1877 rir_way = 1 << RIR_WAY(reg);
8c009100 1878 gb = div_u64_rem(tmp_mb, 1024, &mb);
956b9ba1
JP		 1879 edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n",
1880 i, j,
8c009100 1881 gb, (mb*1000)/1024,
956b9ba1
JP		 1882 ((u64)tmp_mb) << 20L,
1883 rir_way,
1884 reg);
eebf11a0
MCC		 1885
1886 for (k = 0; k < rir_way; k++) {
1887 pci_read_config_dword(pvt->pci_tad[i],
1888 rir_offset[j][k],
1889 &reg);
1890 tmp_mb = RIR_OFFSET(reg) << 6;
1891
8c009100 1892 gb = div_u64_rem(tmp_mb, 1024, &mb);
956b9ba1
JP		 1893 edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n",
1894 i, j, k,
8c009100 1895 gb, (mb*1000)/1024,
956b9ba1
JP		 1896 ((u64)tmp_mb) << 20L,
1897 (u32)RIR_RNK_TGT(reg),
1898 reg);
eebf11a0
MCC		 1899 }
1900 }
1901 }
1902}
1903
8112c0cd 1904static struct mem_ctl_info *get_mci_for_node_id(u8 node_id)
eebf11a0
MCC		 1905{
1906 struct sbridge_dev *sbridge_dev;
1907
1908 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
1909 if (sbridge_dev->node_id == node_id)
1910 return sbridge_dev->mci;
1911 }
1912 return NULL;
1913}
1914
1915static int get_memory_error_data(struct mem_ctl_info *mci,
1916 u64 addr,
7d375bff 1917 u8 *socket, u8 *ha,
eebf11a0
MCC		 1918 long *channel_mask,
1919 u8 *rank,
e17a2f42 1920 char **area_type, char *msg)
eebf11a0
MCC		 1921{
1922 struct mem_ctl_info *new_mci;
1923 struct sbridge_pvt *pvt = mci->pvt_info;
4d715a80 1924 struct pci_dev *pci_ha;
c41afdca 1925 int n_rir, n_sads, n_tads, sad_way, sck_xch;
eebf11a0 1926 int sad_interl, idx, base_ch;
50d1bb93 1927 int interleave_mode, shiftup = 0;
ef1ce51e 1928 unsigned sad_interleave[pvt->info.max_interleave];
50d1bb93 1929 u32 reg, dram_rule;
7d375bff 1930 u8 ch_way, sck_way, pkg, sad_ha = 0, ch_add = 0;
eebf11a0
MCC		 1931 u32 tad_offset;
1932 u32 rir_way;
8c009100 1933 u32 mb, gb;
bd4b9683 1934 u64 ch_addr, offset, limit = 0, prv = 0;
eebf11a0
MCC		 1935
1936
1937 /*
1938 * Step 0) Check if the address is at special memory ranges
1939 * The check bellow is probably enough to fill all cases where
1940 * the error is not inside a memory, except for the legacy
1941 * range (e. g. VGA addresses). It is unlikely, however, that the
1942 * memory controller would generate an error on that range.
1943 */
5b889e37 1944 if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) {
eebf11a0 1945 sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr);
eebf11a0
MCC		 1946 return -EINVAL;
1947 }
1948 if (addr >= (u64)pvt->tohm) {
1949 sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr);
eebf11a0
MCC		 1950 return -EINVAL;
1951 }
1952
1953 /*
1954 * Step 1) Get socket
1955 */
464f1d82
AR		 1956 for (n_sads = 0; n_sads < pvt->info.max_sad; n_sads++) {
1957 pci_read_config_dword(pvt->pci_sad0, pvt->info.dram_rule[n_sads],
eebf11a0
MCC		 1958 &reg);
1959
1960 if (!DRAM_RULE_ENABLE(reg))
1961 continue;
1962
c59f9c06 1963 limit = pvt->info.sad_limit(reg);
eebf11a0
MCC		 1964 if (limit <= prv) {
1965 sprintf(msg, "Can't discover the memory socket");
eebf11a0
MCC		 1966 return -EINVAL;
1967 }
1968 if (addr <= limit)
1969 break;
1970 prv = limit;
1971 }
464f1d82 1972 if (n_sads == pvt->info.max_sad) {
eebf11a0 1973 sprintf(msg, "Can't discover the memory socket");
eebf11a0
MCC		 1974 return -EINVAL;
1975 }
50d1bb93 1976 dram_rule = reg;
c59f9c06
JS		 1977 *area_type = show_dram_attr(pvt->info.dram_attr(dram_rule));
1978 interleave_mode = pvt->info.interleave_mode(dram_rule);
eebf11a0 1979
ef1ce51e 1980 pci_read_config_dword(pvt->pci_sad0, pvt->info.interleave_list[n_sads],
eebf11a0 1981 &reg);
4d715a80
AR		 1982
1983 if (pvt->info.type == SANDY_BRIDGE) {
1984 sad_interl = sad_pkg(pvt->info.interleave_pkg, reg, 0);
1985 for (sad_way = 0; sad_way < 8; sad_way++) {
1986 u32 pkg = sad_pkg(pvt->info.interleave_pkg, reg, sad_way);
1987 if (sad_way > 0 && sad_interl == pkg)
1988 break;
1989 sad_interleave[sad_way] = pkg;
1990 edac_dbg(0, "SAD interleave #%d: %d\n",
1991 sad_way, sad_interleave[sad_way]);
1992 }
1993 edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n",
1994 pvt->sbridge_dev->mc,
1995 n_sads,
1996 addr,
1997 limit,
1998 sad_way + 7,
1999 !interleave_mode ? "" : "XOR[18:16]");
2000 if (interleave_mode)
2001 idx = ((addr >> 6) ^ (addr >> 16)) & 7;
2002 else
2003 idx = (addr >> 6) & 7;
2004 switch (sad_way) {
2005 case 1:
2006 idx = 0;
eebf11a0 2007 break;
4d715a80
AR		 2008 case 2:
2009 idx = idx & 1;
2010 break;
2011 case 4:
2012 idx = idx & 3;
2013 break;
2014 case 8:
2015 break;
2016 default:
2017 sprintf(msg, "Can't discover socket interleave");
2018 return -EINVAL;
2019 }
2020 *socket = sad_interleave[idx];
2021 edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n",
2022 idx, sad_way, *socket);
1f39581a 2023 } else if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) {
50d1bb93
AR		 2024 int bits, a7mode = A7MODE(dram_rule);
2025
2026 if (a7mode) {
2027 /* A7 mode swaps P9 with P6 */
2028 bits = GET_BITFIELD(addr, 7, 8) << 1;
2029 bits |= GET_BITFIELD(addr, 9, 9);
2030 } else
bb89e714 2031 bits = GET_BITFIELD(addr, 6, 8);
50d1bb93 2032
bb89e714 2033 if (interleave_mode == 0) {
50d1bb93
AR		 2034 /* interleave mode will XOR {8,7,6} with {18,17,16} */
2035 idx = GET_BITFIELD(addr, 16, 18);
2036 idx ^= bits;
2037 } else
2038 idx = bits;
2039
2040 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx);
2041 *socket = sad_pkg_socket(pkg);
2042 sad_ha = sad_pkg_ha(pkg);
7d375bff
TL		 2043 if (sad_ha)
2044 ch_add = 4;
50d1bb93
AR		 2045
2046 if (a7mode) {
2047 /* MCChanShiftUpEnable */
2048 pci_read_config_dword(pvt->pci_ha0,
2049 HASWELL_HASYSDEFEATURE2, &reg);
2050 shiftup = GET_BITFIELD(reg, 22, 22);
2051 }
2052
2053 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %i, shiftup: %i\n",
2054 idx, *socket, sad_ha, shiftup);
4d715a80
AR		 2055 } else {
2056 /* Ivy Bridge's SAD mode doesn't support XOR interleave mode */
eebf11a0 2057 idx = (addr >> 6) & 7;
4d715a80
AR		 2058 pkg = sad_pkg(pvt->info.interleave_pkg, reg, idx);
2059 *socket = sad_pkg_socket(pkg);
2060 sad_ha = sad_pkg_ha(pkg);
7d375bff
TL		 2061 if (sad_ha)
2062 ch_add = 4;
4d715a80
AR		 2063 edac_dbg(0, "SAD interleave package: %d = CPU socket %d, HA %d\n",
2064 idx, *socket, sad_ha);
eebf11a0 2065 }
eebf11a0 2066
7d375bff
TL		 2067 *ha = sad_ha;
2068
eebf11a0
MCC		 2069 /*
2070 * Move to the proper node structure, in order to access the
2071 * right PCI registers
2072 */
2073 new_mci = get_mci_for_node_id(*socket);
2074 if (!new_mci) {
2075 sprintf(msg, "Struct for socket #%u wasn't initialized",
2076 *socket);
eebf11a0
MCC		 2077 return -EINVAL;
2078 }
2079 mci = new_mci;
2080 pvt = mci->pvt_info;
2081
2082 /*
2083 * Step 2) Get memory channel
2084 */
2085 prv = 0;
4d715a80
AR		 2086 if (pvt->info.type == SANDY_BRIDGE)
2087 pci_ha = pvt->pci_ha0;
2088 else {
2089 if (sad_ha)
2090 pci_ha = pvt->pci_ha1;
2091 else
2092 pci_ha = pvt->pci_ha0;
2093 }
eebf11a0 2094 for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
4d715a80 2095 pci_read_config_dword(pci_ha, tad_dram_rule[n_tads], &reg);
eebf11a0
MCC		 2096 limit = TAD_LIMIT(reg);
2097 if (limit <= prv) {
2098 sprintf(msg, "Can't discover the memory channel");
eebf11a0
MCC		 2099 return -EINVAL;
2100 }
2101 if (addr <= limit)
2102 break;
2103 prv = limit;
2104 }
4d715a80
AR		 2105 if (n_tads == MAX_TAD) {
2106 sprintf(msg, "Can't discover the memory channel");
2107 return -EINVAL;
2108 }
2109
eebf11a0 2110 ch_way = TAD_CH(reg) + 1;
eb1af3b7 2111 sck_way = 1 << TAD_SOCK(reg);
eebf11a0
MCC		 2112
2113 if (ch_way == 3)
2114 idx = addr >> 6;
2115 else
50d1bb93 2116 idx = (addr >> (6 + sck_way + shiftup)) & 0x3;
eebf11a0
MCC		 2117 idx = idx % ch_way;
2118
2119 /*
2120 * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ???
2121 */
2122 switch (idx) {
2123 case 0:
2124 base_ch = TAD_TGT0(reg);
2125 break;
2126 case 1:
2127 base_ch = TAD_TGT1(reg);
2128 break;
2129 case 2:
2130 base_ch = TAD_TGT2(reg);
2131 break;
2132 case 3:
2133 base_ch = TAD_TGT3(reg);
2134 break;
2135 default:
2136 sprintf(msg, "Can't discover the TAD target");
eebf11a0
MCC		 2137 return -EINVAL;
2138 }
2139 *channel_mask = 1 << base_ch;
2140
7d375bff 2141 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
4d715a80
AR		 2142 tad_ch_nilv_offset[n_tads],
2143 &tad_offset);
2144
eebf11a0
MCC		 2145 if (pvt->is_mirrored) {
2146 *channel_mask |= 1 << ((base_ch + 2) % 4);
2147 switch(ch_way) {
2148 case 2:
2149 case 4:
2150 sck_xch = 1 << sck_way * (ch_way >> 1);
2151 break;
2152 default:
2153 sprintf(msg, "Invalid mirror set. Can't decode addr");
eebf11a0
MCC		 2154 return -EINVAL;
2155 }
2156 } else
2157 sck_xch = (1 << sck_way) * ch_way;
2158
2159 if (pvt->is_lockstep)
2160 *channel_mask |= 1 << ((base_ch + 1) % 4);
2161
2162 offset = TAD_OFFSET(tad_offset);
2163
956b9ba1
JP		 2164 edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n",
2165 n_tads,
2166 addr,
2167 limit,
eb1af3b7 2168 sck_way,
956b9ba1
JP		 2169 ch_way,
2170 offset,
2171 idx,
2172 base_ch,
2173 *channel_mask);
eebf11a0
MCC		 2174
2175 /* Calculate channel address */
2176 /* Remove the TAD offset */
2177
2178 if (offset > addr) {
2179 sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!",
2180 offset, addr);
eebf11a0
MCC		 2181 return -EINVAL;
2182 }
eb1af3b7
LT		 2183
2184 ch_addr = addr - offset;
2185 ch_addr >>= (6 + shiftup);
2186 ch_addr /= ch_way * sck_way;
2187 ch_addr <<= (6 + shiftup);
2188 ch_addr |= addr & ((1 << (6 + shiftup)) - 1);
eebf11a0
MCC		 2189
2190 /*
2191 * Step 3) Decode rank
2192 */
2193 for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) {
7d375bff 2194 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
eebf11a0
MCC		 2195 rir_way_limit[n_rir],
2196 &reg);
2197
2198 if (!IS_RIR_VALID(reg))
2199 continue;
2200
b976bcf2 2201 limit = pvt->info.rir_limit(reg);
8c009100 2202 gb = div_u64_rem(limit >> 20, 1024, &mb);
956b9ba1
JP		 2203 edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n",
2204 n_rir,
8c009100 2205 gb, (mb*1000)/1024,
956b9ba1
JP		 2206 limit,
2207 1 << RIR_WAY(reg));
eebf11a0
MCC		 2208 if (ch_addr <= limit)
2209 break;
2210 }
2211 if (n_rir == MAX_RIR_RANGES) {
2212 sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx",
2213 ch_addr);
eebf11a0
MCC		 2214 return -EINVAL;
2215 }
2216 rir_way = RIR_WAY(reg);
50d1bb93 2217
eebf11a0
MCC		 2218 if (pvt->is_close_pg)
2219 idx = (ch_addr >> 6);
2220 else
2221 idx = (ch_addr >> 13); /* FIXME: Datasheet says to shift by 15 */
2222 idx %= 1 << rir_way;
2223
7d375bff 2224 pci_read_config_dword(pvt->pci_tad[ch_add + base_ch],
eebf11a0
MCC		 2225 rir_offset[n_rir][idx],
2226 &reg);
2227 *rank = RIR_RNK_TGT(reg);
2228
956b9ba1
JP		 2229 edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n",
2230 n_rir,
2231 ch_addr,
2232 limit,
2233 rir_way,
2234 idx);
eebf11a0
MCC		 2235
2236 return 0;
2237}
2238
2239/****************************************************************************
2240 Device initialization routines: put/get, init/exit
2241 ****************************************************************************/
2242
2243/*
2244 * sbridge_put_all_devices 'put' all the devices that we have
2245 * reserved via 'get'
2246 */
2247static void sbridge_put_devices(struct sbridge_dev *sbridge_dev)
2248{
2249 int i;
2250
956b9ba1 2251 edac_dbg(0, "\n");
eebf11a0
MCC		 2252 for (i = 0; i < sbridge_dev->n_devs; i++) {
2253 struct pci_dev *pdev = sbridge_dev->pdev[i];
2254 if (!pdev)
2255 continue;
956b9ba1
JP		 2256 edac_dbg(0, "Removing dev %02x:%02x.%d\n",
2257 pdev->bus->number,
2258 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
eebf11a0
MCC		 2259 pci_dev_put(pdev);
2260 }
2261}
2262
2263static void sbridge_put_all_devices(void)
2264{
2265 struct sbridge_dev *sbridge_dev, *tmp;
2266
2267 list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) {
2268 sbridge_put_devices(sbridge_dev);
2269 free_sbridge_dev(sbridge_dev);
2270 }
2271}
2272
eebf11a0
MCC		 2273static int sbridge_get_onedevice(struct pci_dev **prev,
2274 u8 *num_mc,
2275 const struct pci_id_table *table,
c1979ba2
JS		 2276 const unsigned devno,
2277 const int multi_bus)
eebf11a0
MCC		 2278{
2279 struct sbridge_dev *sbridge_dev;
2280 const struct pci_id_descr *dev_descr = &table->descr[devno];
eebf11a0
MCC		 2281 struct pci_dev *pdev = NULL;
2282 u8 bus = 0;
2283
ec5a0b38 2284 sbridge_printk(KERN_DEBUG,
dbc954dd 2285 "Seeking for: PCI ID %04x:%04x\n",
eebf11a0
MCC		 2286 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2287
2288 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
2289 dev_descr->dev_id, *prev);
2290
2291 if (!pdev) {
2292 if (*prev) {
2293 *prev = pdev;
2294 return 0;
2295 }
2296
2297 if (dev_descr->optional)
2298 return 0;
2299
dbc954dd 2300 /* if the HA wasn't found */
eebf11a0
MCC		 2301 if (devno == 0)
2302 return -ENODEV;
2303
2304 sbridge_printk(KERN_INFO,
dbc954dd 2305 "Device not found: %04x:%04x\n",
eebf11a0
MCC		 2306 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2307
2308 /* End of list, leave */
2309 return -ENODEV;
2310 }
2311 bus = pdev->bus->number;
2312
c1979ba2 2313 sbridge_dev = get_sbridge_dev(bus, multi_bus);
eebf11a0
MCC		 2314 if (!sbridge_dev) {
2315 sbridge_dev = alloc_sbridge_dev(bus, table);
2316 if (!sbridge_dev) {
2317 pci_dev_put(pdev);
2318 return -ENOMEM;
2319 }
2320 (*num_mc)++;
2321 }
2322
2323 if (sbridge_dev->pdev[devno]) {
2324 sbridge_printk(KERN_ERR,
dbc954dd 2325 "Duplicated device for %04x:%04x\n",
eebf11a0
MCC		 2326 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2327 pci_dev_put(pdev);
2328 return -ENODEV;
2329 }
2330
2331 sbridge_dev->pdev[devno] = pdev;
2332
eebf11a0
MCC		 2333 /* Be sure that the device is enabled */
2334 if (unlikely(pci_enable_device(pdev) < 0)) {
2335 sbridge_printk(KERN_ERR,
dbc954dd 2336 "Couldn't enable %04x:%04x\n",
eebf11a0
MCC		 2337 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
2338 return -ENODEV;
2339 }
2340
dbc954dd 2341 edac_dbg(0, "Detected %04x:%04x\n",
956b9ba1 2342 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
eebf11a0
MCC		 2343
2344 /*
2345 * As stated on drivers/pci/search.c, the reference count for
2346 * @from is always decremented if it is not %NULL. So, as we need
2347 * to get all devices up to null, we need to do a get for the device
2348 */
2349 pci_dev_get(pdev);
2350
2351 *prev = pdev;
2352
2353 return 0;
2354}
2355
5153a0f9
AR		 2356/*
2357 * sbridge_get_all_devices - Find and perform 'get' operation on the MCH's
dbc954dd 2358 * devices we want to reference for this driver.
5153a0f9 2359 * @num_mc: pointer to the memory controllers count, to be incremented in case
c41afdca 2360 * of success.
5153a0f9 2361 * @table: model specific table
c1979ba2
JS		 2362 * @allow_dups: allow for multiple devices to exist with the same device id
2363 * (as implemented, this isn't expected to work correctly in the
2364 * multi-socket case).
2365 * @multi_bus: don't assume devices on different buses belong to different
2366 * memory controllers.
5153a0f9
AR		 2367 *
2368 * returns 0 in case of success or error code
2369 */
c1979ba2
JS		 2370static int sbridge_get_all_devices_full(u8 *num_mc,
2371 const struct pci_id_table *table,
2372 int allow_dups,
2373 int multi_bus)
eebf11a0
MCC		 2374{
2375 int i, rc;
2376 struct pci_dev *pdev = NULL;
eebf11a0
MCC		 2377
2378 while (table && table->descr) {
2379 for (i = 0; i < table->n_devs; i++) {
c1979ba2
JS		 2380 if (!allow_dups || i == 0 ||
2381 table->descr[i].dev_id !=
2382 table->descr[i-1].dev_id) {
2383 pdev = NULL;
2384 }
eebf11a0
MCC		 2385 do {
2386 rc = sbridge_get_onedevice(&pdev, num_mc,
c1979ba2 2387 table, i, multi_bus);
eebf11a0
MCC		 2388 if (rc < 0) {
2389 if (i == 0) {
2390 i = table->n_devs;
2391 break;
2392 }
2393 sbridge_put_all_devices();
2394 return -ENODEV;
2395 }
c1979ba2 2396 } while (pdev && !allow_dups);
eebf11a0
MCC		 2397 }
2398 table++;
2399 }
2400
2401 return 0;
2402}
2403
c1979ba2
JS		 2404#define sbridge_get_all_devices(num_mc, table) \
2405 sbridge_get_all_devices_full(num_mc, table, 0, 0)
d0cdf900
JS		 2406#define sbridge_get_all_devices_knl(num_mc, table) \
2407 sbridge_get_all_devices_full(num_mc, table, 1, 1)
c1979ba2 2408
ea779b5a
AR		 2409static int sbridge_mci_bind_devs(struct mem_ctl_info *mci,
2410 struct sbridge_dev *sbridge_dev)
eebf11a0
MCC		 2411{
2412 struct sbridge_pvt *pvt = mci->pvt_info;
2413 struct pci_dev *pdev;
2900ea60 2414 u8 saw_chan_mask = 0;
dbc954dd 2415 int i;
eebf11a0
MCC		 2416
2417 for (i = 0; i < sbridge_dev->n_devs; i++) {
2418 pdev = sbridge_dev->pdev[i];
2419 if (!pdev)
2420 continue;
dbc954dd
AR		 2421
2422 switch (pdev->device) {
2423 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0:
2424 pvt->pci_sad0 = pdev;
eebf11a0 2425 break;
dbc954dd
AR		 2426 case PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1:
2427 pvt->pci_sad1 = pdev;
eebf11a0 2428 break;
dbc954dd
AR		 2429 case PCI_DEVICE_ID_INTEL_SBRIDGE_BR:
2430 pvt->pci_br0 = pdev;
eebf11a0 2431 break;
dbc954dd
AR		 2432 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0:
2433 pvt->pci_ha0 = pdev;
eebf11a0 2434 break;
dbc954dd
AR		 2435 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA:
2436 pvt->pci_ta = pdev;
2437 break;
2438 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS:
2439 pvt->pci_ras = pdev;
2440 break;
2441 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0:
2442 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1:
2443 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2:
2444 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3:
2445 {
2446 int id = pdev->device - PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0;
2447 pvt->pci_tad[id] = pdev;
2900ea60 2448 saw_chan_mask |= 1 << id;
dbc954dd
AR		 2449 }
2450 break;
2451 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO:
2452 pvt->pci_ddrio = pdev;
eebf11a0
MCC		 2453 break;
2454 default:
2455 goto error;
2456 }
2457
dbc954dd
AR		 2458 edac_dbg(0, "Associated PCI %02x:%02x, bus %d with dev = %p\n",
2459 pdev->vendor, pdev->device,
956b9ba1 2460 sbridge_dev->bus,
956b9ba1 2461 pdev);
eebf11a0
MCC		 2462 }
2463
2464 /* Check if everything were registered */
2465 if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha0 ||
de4772c6 2466 !pvt-> pci_tad || !pvt->pci_ras || !pvt->pci_ta)
eebf11a0
MCC		 2467 goto enodev;
2468
2900ea60
SJ		 2469 if (saw_chan_mask != 0x0f)
2470 goto enodev;
eebf11a0
MCC		 2471 return 0;
2472
2473enodev:
2474 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2475 return -ENODEV;
2476
2477error:
dbc954dd
AR		 2478 sbridge_printk(KERN_ERR, "Unexpected device %02x:%02x\n",
2479 PCI_VENDOR_ID_INTEL, pdev->device);
eebf11a0
MCC		 2480 return -EINVAL;
2481}
2482
4d715a80
AR		 2483static int ibridge_mci_bind_devs(struct mem_ctl_info *mci,
2484 struct sbridge_dev *sbridge_dev)
2485{
2486 struct sbridge_pvt *pvt = mci->pvt_info;
7d375bff
TL		 2487 struct pci_dev *pdev;
2488 u8 saw_chan_mask = 0;
dbc954dd 2489 int i;
4d715a80
AR		 2490
2491 for (i = 0; i < sbridge_dev->n_devs; i++) {
2492 pdev = sbridge_dev->pdev[i];
2493 if (!pdev)
2494 continue;
4d715a80 2495
dbc954dd
AR		 2496 switch (pdev->device) {
2497 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0:
2498 pvt->pci_ha0 = pdev;
2499 break;
2500 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA:
2501 pvt->pci_ta = pdev;
2502 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_RAS:
2503 pvt->pci_ras = pdev;
2504 break;
dbc954dd
AR		 2505 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0:
2506 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD1:
7d375bff
TL		 2507 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD2:
2508 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD3:
dbc954dd
AR		 2509 {
2510 int id = pdev->device - PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TAD0;
2511 pvt->pci_tad[id] = pdev;
7d375bff 2512 saw_chan_mask |= 1 << id;
dbc954dd 2513 }
4d715a80 2514 break;
dbc954dd
AR		 2515 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_2HA_DDRIO0:
2516 pvt->pci_ddrio = pdev;
2517 break;
2518 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_1HA_DDRIO0:
7d375bff 2519 pvt->pci_ddrio = pdev;
4d715a80 2520 break;
dbc954dd
AR		 2521 case PCI_DEVICE_ID_INTEL_IBRIDGE_SAD:
2522 pvt->pci_sad0 = pdev;
2523 break;
2524 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR0:
2525 pvt->pci_br0 = pdev;
2526 break;
2527 case PCI_DEVICE_ID_INTEL_IBRIDGE_BR1:
2528 pvt->pci_br1 = pdev;
2529 break;
2530 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1:
2531 pvt->pci_ha1 = pdev;
2532 break;
2533 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0:
2534 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD1:
7d375bff
TL		 2535 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD2:
2536 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD3:
dbc954dd 2537 {
7d375bff 2538 int id = pdev->device - PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA1_TAD0 + 4;
dbc954dd 2539 pvt->pci_tad[id] = pdev;
7d375bff 2540 saw_chan_mask |= 1 << id;
dbc954dd
AR		 2541 }
2542 break;
4d715a80
AR		 2543 default:
2544 goto error;
2545 }
2546
2547 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2548 sbridge_dev->bus,
2549 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2550 pdev);
2551 }
2552
2553 /* Check if everything were registered */
2554 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_br0 ||
2555 !pvt->pci_br1 || !pvt->pci_tad || !pvt->pci_ras ||
2556 !pvt->pci_ta)
2557 goto enodev;
2558
7d375bff
TL		 2559 if (saw_chan_mask != 0x0f && /* -EN */
2560 saw_chan_mask != 0x33 && /* -EP */
2561 saw_chan_mask != 0xff) /* -EX */
2562 goto enodev;
4d715a80
AR		 2563 return 0;
2564
2565enodev:
2566 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2567 return -ENODEV;
2568
2569error:
2570 sbridge_printk(KERN_ERR,
dbc954dd
AR		 2571 "Unexpected device %02x:%02x\n", PCI_VENDOR_ID_INTEL,
2572 pdev->device);
4d715a80
AR		 2573 return -EINVAL;
2574}
2575
50d1bb93
AR		 2576static int haswell_mci_bind_devs(struct mem_ctl_info *mci,
2577 struct sbridge_dev *sbridge_dev)
2578{
2579 struct sbridge_pvt *pvt = mci->pvt_info;
7d375bff
TL		 2580 struct pci_dev *pdev;
2581 u8 saw_chan_mask = 0;
50d1bb93 2582 int i;
50d1bb93
AR		 2583
2584 /* there's only one device per system; not tied to any bus */
2585 if (pvt->info.pci_vtd == NULL)
2586 /* result will be checked later */
2587 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL,
2588 PCI_DEVICE_ID_INTEL_HASWELL_IMC_VTD_MISC,
2589 NULL);
2590
2591 for (i = 0; i < sbridge_dev->n_devs; i++) {
2592 pdev = sbridge_dev->pdev[i];
2593 if (!pdev)
2594 continue;
2595
2596 switch (pdev->device) {
2597 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD0:
2598 pvt->pci_sad0 = pdev;
2599 break;
2600 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_CBO_SAD1:
2601 pvt->pci_sad1 = pdev;
2602 break;
2603 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0:
2604 pvt->pci_ha0 = pdev;
2605 break;
2606 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TA:
2607 pvt->pci_ta = pdev;
2608 break;
2609 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_THERMAL:
2610 pvt->pci_ras = pdev;
2611 break;
2612 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0:
50d1bb93 2613 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD1:
50d1bb93 2614 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD2:
50d1bb93 2615 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3:
7d375bff
TL		 2616 {
2617 int id = pdev->device - PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD0;
2618
2619 pvt->pci_tad[id] = pdev;
2620 saw_chan_mask |= 1 << id;
2621 }
2622 break;
2623 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0:
2624 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD1:
2625 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2:
2626 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3:
2627 {
2628 int id = pdev->device - PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD0 + 4;
2629
2630 pvt->pci_tad[id] = pdev;
2631 saw_chan_mask |= 1 << id;
2632 }
50d1bb93
AR		 2633 break;
2634 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0:
7179385a
AR		 2635 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1:
2636 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2:
2637 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3:
2638 if (!pvt->pci_ddrio)
2639 pvt->pci_ddrio = pdev;
50d1bb93
AR		 2640 break;
2641 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1:
2642 pvt->pci_ha1 = pdev;
2643 break;
2644 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA:
2645 pvt->pci_ha1_ta = pdev;
2646 break;
50d1bb93
AR		 2647 default:
2648 break;
2649 }
2650
2651 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2652 sbridge_dev->bus,
2653 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2654 pdev);
2655 }
2656
2657 /* Check if everything were registered */
2658 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_sad1 ||
2659 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd)
2660 goto enodev;
2661
7d375bff
TL		 2662 if (saw_chan_mask != 0x0f && /* -EN */
2663 saw_chan_mask != 0x33 && /* -EP */
2664 saw_chan_mask != 0xff) /* -EX */
2665 goto enodev;
50d1bb93
AR		 2666 return 0;
2667
2668enodev:
2669 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2670 return -ENODEV;
2671}
2672
1f39581a
TL		 2673static int broadwell_mci_bind_devs(struct mem_ctl_info *mci,
2674 struct sbridge_dev *sbridge_dev)
2675{
2676 struct sbridge_pvt *pvt = mci->pvt_info;
2677 struct pci_dev *pdev;
fa2ce64f 2678 u8 saw_chan_mask = 0;
1f39581a
TL		 2679 int i;
2680
2681 /* there's only one device per system; not tied to any bus */
2682 if (pvt->info.pci_vtd == NULL)
2683 /* result will be checked later */
2684 pvt->info.pci_vtd = pci_get_device(PCI_VENDOR_ID_INTEL,
2685 PCI_DEVICE_ID_INTEL_BROADWELL_IMC_VTD_MISC,
2686 NULL);
2687
2688 for (i = 0; i < sbridge_dev->n_devs; i++) {
2689 pdev = sbridge_dev->pdev[i];
2690 if (!pdev)
2691 continue;
2692
2693 switch (pdev->device) {
2694 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD0:
2695 pvt->pci_sad0 = pdev;
2696 break;
2697 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_CBO_SAD1:
2698 pvt->pci_sad1 = pdev;
2699 break;
2700 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0:
2701 pvt->pci_ha0 = pdev;
2702 break;
2703 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TA:
2704 pvt->pci_ta = pdev;
2705 break;
2706 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_THERMAL:
2707 pvt->pci_ras = pdev;
2708 break;
2709 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0:
1f39581a 2710 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD1:
1f39581a 2711 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD2:
1f39581a 2712 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD3:
fa2ce64f
TL		 2713 {
2714 int id = pdev->device - PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0_TAD0;
2715 pvt->pci_tad[id] = pdev;
2716 saw_chan_mask |= 1 << id;
2717 }
2718 break;
2719 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0:
2720 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD1:
2721 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD2:
2722 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD3:
2723 {
2724 int id = pdev->device - PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TAD0 + 4;
2725 pvt->pci_tad[id] = pdev;
2726 saw_chan_mask |= 1 << id;
2727 }
1f39581a
TL		 2728 break;
2729 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_DDRIO0:
2730 pvt->pci_ddrio = pdev;
2731 break;
fa2ce64f
TL		 2732 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1:
2733 pvt->pci_ha1 = pdev;
2734 break;
2735 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA1_TA:
2736 pvt->pci_ha1_ta = pdev;
2737 break;
1f39581a
TL		 2738 default:
2739 break;
2740 }
2741
2742 edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
2743 sbridge_dev->bus,
2744 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
2745 pdev);
2746 }
2747
2748 /* Check if everything were registered */
2749 if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_sad1 ||
2750 !pvt->pci_ras || !pvt->pci_ta || !pvt->info.pci_vtd)
2751 goto enodev;
2752
fa2ce64f
TL		 2753 if (saw_chan_mask != 0x0f && /* -EN */
2754 saw_chan_mask != 0x33 && /* -EP */
2755 saw_chan_mask != 0xff) /* -EX */
2756 goto enodev;
1f39581a
TL		 2757 return 0;
2758
2759enodev:
2760 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2761 return -ENODEV;
2762}
2763
d0cdf900
JS		 2764static int knl_mci_bind_devs(struct mem_ctl_info *mci,
2765 struct sbridge_dev *sbridge_dev)
2766{
2767 struct sbridge_pvt *pvt = mci->pvt_info;
2768 struct pci_dev *pdev;
2769 int dev, func;
2770
2771 int i;
2772 int devidx;
2773
2774 for (i = 0; i < sbridge_dev->n_devs; i++) {
2775 pdev = sbridge_dev->pdev[i];
2776 if (!pdev)
2777 continue;
2778
2779 /* Extract PCI device and function. */
2780 dev = (pdev->devfn >> 3) & 0x1f;
2781 func = pdev->devfn & 0x7;
2782
2783 switch (pdev->device) {
2784 case PCI_DEVICE_ID_INTEL_KNL_IMC_MC:
2785 if (dev == 8)
2786 pvt->knl.pci_mc0 = pdev;
2787 else if (dev == 9)
2788 pvt->knl.pci_mc1 = pdev;
2789 else {
2790 sbridge_printk(KERN_ERR,
2791 "Memory controller in unexpected place! (dev %d, fn %d)\n",
2792 dev, func);
2793 continue;
2794 }
2795 break;
2796
2797 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0:
2798 pvt->pci_sad0 = pdev;
2799 break;
2800
2801 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD1:
2802 pvt->pci_sad1 = pdev;
2803 break;
2804
2805 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHA:
2806 /* There are one of these per tile, and range from
2807 * 1.14.0 to 1.18.5.
2808 */
2809 devidx = ((dev-14)*8)+func;
2810
2811 if (devidx < 0 || devidx >= KNL_MAX_CHAS) {
2812 sbridge_printk(KERN_ERR,
2813 "Caching and Home Agent in unexpected place! (dev %d, fn %d)\n",
2814 dev, func);
2815 continue;
2816 }
2817
2818 WARN_ON(pvt->knl.pci_cha[devidx] != NULL);
2819
2820 pvt->knl.pci_cha[devidx] = pdev;
2821 break;
2822
2823 case PCI_DEVICE_ID_INTEL_KNL_IMC_CHANNEL:
2824 devidx = -1;
2825
2826 /*
2827 * MC0 channels 0-2 are device 9 function 2-4,
2828 * MC1 channels 3-5 are device 8 function 2-4.
2829 */
2830
2831 if (dev == 9)
2832 devidx = func-2;
2833 else if (dev == 8)
2834 devidx = 3 + (func-2);
2835
2836 if (devidx < 0 || devidx >= KNL_MAX_CHANNELS) {
2837 sbridge_printk(KERN_ERR,
2838 "DRAM Channel Registers in unexpected place! (dev %d, fn %d)\n",
2839 dev, func);
2840 continue;
2841 }
2842
2843 WARN_ON(pvt->knl.pci_channel[devidx] != NULL);
2844 pvt->knl.pci_channel[devidx] = pdev;
2845 break;
2846
2847 case PCI_DEVICE_ID_INTEL_KNL_IMC_TOLHM:
2848 pvt->knl.pci_mc_info = pdev;
2849 break;
2850
2851 case PCI_DEVICE_ID_INTEL_KNL_IMC_TA:
2852 pvt->pci_ta = pdev;
2853 break;
2854
2855 default:
2856 sbridge_printk(KERN_ERR, "Unexpected device %d\n",
2857 pdev->device);
2858 break;
2859 }
2860 }
2861
2862 if (!pvt->knl.pci_mc0 || !pvt->knl.pci_mc1 ||
2863 !pvt->pci_sad0 || !pvt->pci_sad1 ||
2864 !pvt->pci_ta) {
2865 goto enodev;
2866 }
2867
2868 for (i = 0; i < KNL_MAX_CHANNELS; i++) {
2869 if (!pvt->knl.pci_channel[i]) {
2870 sbridge_printk(KERN_ERR, "Missing channel %d\n", i);
2871 goto enodev;
2872 }
2873 }
2874
2875 for (i = 0; i < KNL_MAX_CHAS; i++) {
2876 if (!pvt->knl.pci_cha[i]) {
2877 sbridge_printk(KERN_ERR, "Missing CHA %d\n", i);
2878 goto enodev;
2879 }
2880 }
2881
2882 return 0;
2883
2884enodev:
2885 sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
2886 return -ENODEV;
2887}
2888
eebf11a0
MCC		 2889/****************************************************************************
2890 Error check routines
2891 ****************************************************************************/
2892
2893/*
2894 * While Sandy Bridge has error count registers, SMI BIOS read values from
2895 * and resets the counters. So, they are not reliable for the OS to read
2896 * from them. So, we have no option but to just trust on whatever MCE is
2897 * telling us about the errors.
2898 */
2899static void sbridge_mce_output_error(struct mem_ctl_info *mci,
2900 const struct mce *m)
2901{
2902 struct mem_ctl_info *new_mci;
2903 struct sbridge_pvt *pvt = mci->pvt_info;
c36e3e77 2904 enum hw_event_mc_err_type tp_event;
e17a2f42 2905 char *type, *optype, msg[256];
eebf11a0
MCC		 2906 bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
2907 bool overflow = GET_BITFIELD(m->status, 62, 62);
2908 bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
4d715a80 2909 bool recoverable;
eebf11a0
MCC		 2910 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
2911 u32 mscod = GET_BITFIELD(m->status, 16, 31);
2912 u32 errcode = GET_BITFIELD(m->status, 0, 15);
2913 u32 channel = GET_BITFIELD(m->status, 0, 3);
2914 u32 optypenum = GET_BITFIELD(m->status, 4, 6);
2915 long channel_mask, first_channel;
7d375bff 2916 u8 rank, socket, ha;
c36e3e77 2917 int rc, dimm;
e17a2f42 2918 char *area_type = NULL;
eebf11a0 2919
fa2ce64f 2920 if (pvt->info.type != SANDY_BRIDGE)
4d715a80
AR		 2921 recoverable = true;
2922 else
2923 recoverable = GET_BITFIELD(m->status, 56, 56);
2924
c36e3e77
MCC		 2925 if (uncorrected_error) {
2926 if (ripv) {
2927 type = "FATAL";
2928 tp_event = HW_EVENT_ERR_FATAL;
2929 } else {
2930 type = "NON_FATAL";
2931 tp_event = HW_EVENT_ERR_UNCORRECTED;
2932 }
2933 } else {
2934 type = "CORRECTED";
2935 tp_event = HW_EVENT_ERR_CORRECTED;
2936 }
eebf11a0
MCC		 2937
2938 /*
15ed103a 2939 * According with Table 15-9 of the Intel Architecture spec vol 3A,
eebf11a0
MCC		 2940 * memory errors should fit in this mask:
2941 * 000f 0000 1mmm cccc (binary)
2942 * where:
2943 * f = Correction Report Filtering Bit. If 1, subsequent errors
2944 * won't be shown
2945 * mmm = error type
2946 * cccc = channel
2947 * If the mask doesn't match, report an error to the parsing logic
2948 */
2949 if (! ((errcode & 0xef80) == 0x80)) {
2950 optype = "Can't parse: it is not a mem";
2951 } else {
2952 switch (optypenum) {
2953 case 0:
c36e3e77 2954 optype = "generic undef request error";
eebf11a0
MCC		 2955 break;
2956 case 1:
c36e3e77 2957 optype = "memory read error";
eebf11a0
MCC		 2958 break;
2959 case 2:
c36e3e77 2960 optype = "memory write error";
eebf11a0
MCC		 2961 break;
2962 case 3:
c36e3e77 2963 optype = "addr/cmd error";
eebf11a0
MCC		 2964 break;
2965 case 4:
c36e3e77 2966 optype = "memory scrubbing error";
eebf11a0
MCC		 2967 break;
2968 default:
2969 optype = "reserved";
2970 break;
2971 }
2972 }
2973
be3036d2
AR		 2974 /* Only decode errors with an valid address (ADDRV) */
2975 if (!GET_BITFIELD(m->status, 58, 58))
2976 return;
2977
d0cdf900
JS		 2978 if (pvt->info.type == KNIGHTS_LANDING) {
2979 if (channel == 14) {
2980 edac_dbg(0, "%s%s err_code:%04x:%04x EDRAM bank %d\n",
2981 overflow ? " OVERFLOW" : "",
2982 (uncorrected_error && recoverable)
2983 ? " recoverable" : "",
2984 mscod, errcode,
2985 m->bank);
2986 } else {
2987 char A = *("A");
2988
2989 channel = knl_channel_remap(channel);
2990 channel_mask = 1 << channel;
2991 snprintf(msg, sizeof(msg),
2992 "%s%s err_code:%04x:%04x channel:%d (DIMM_%c)",
2993 overflow ? " OVERFLOW" : "",
2994 (uncorrected_error && recoverable)
2995 ? " recoverable" : " ",
2996 mscod, errcode, channel, A + channel);
2997 edac_mc_handle_error(tp_event, mci, core_err_cnt,
2998 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
2999 channel, 0, -1,
3000 optype, msg);
3001 }
3002 return;
3003 } else {
3004 rc = get_memory_error_data(mci, m->addr, &socket, &ha,
3005 &channel_mask, &rank, &area_type, msg);
3006 }
3007
eebf11a0 3008 if (rc < 0)
c36e3e77 3009 goto err_parsing;
eebf11a0
MCC		 3010 new_mci = get_mci_for_node_id(socket);
3011 if (!new_mci) {
c36e3e77
MCC		 3012 strcpy(msg, "Error: socket got corrupted!");
3013 goto err_parsing;
eebf11a0
MCC		 3014 }
3015 mci = new_mci;
3016 pvt = mci->pvt_info;
3017
3018 first_channel = find_first_bit(&channel_mask, NUM_CHANNELS);
3019
3020 if (rank < 4)
3021 dimm = 0;
3022 else if (rank < 8)
3023 dimm = 1;
3024 else
3025 dimm = 2;
3026
eebf11a0
MCC		 3027
3028 /*
e17a2f42
MCC		 3029 * FIXME: On some memory configurations (mirror, lockstep), the
3030 * Memory Controller can't point the error to a single DIMM. The
3031 * EDAC core should be handling the channel mask, in order to point
3032 * to the group of dimm's where the error may be happening.
eebf11a0 3033 */
d7c660b7
AR		 3034 if (!pvt->is_lockstep && !pvt->is_mirrored && !pvt->is_close_pg)
3035 channel = first_channel;
3036
c36e3e77 3037 snprintf(msg, sizeof(msg),
7d375bff 3038 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d",
e17a2f42
MCC		 3039 overflow ? " OVERFLOW" : "",
3040 (uncorrected_error && recoverable) ? " recoverable" : "",
3041 area_type,
3042 mscod, errcode,
7d375bff 3043 socket, ha,
e17a2f42
MCC		 3044 channel_mask,
3045 rank);
eebf11a0 3046
956b9ba1 3047 edac_dbg(0, "%s\n", msg);
eebf11a0 3048
c36e3e77
MCC		 3049 /* FIXME: need support for channel mask */
3050
351fc4a9
SJ		 3051 if (channel == CHANNEL_UNSPECIFIED)
3052 channel = -1;
3053
eebf11a0 3054 /* Call the helper to output message */
c1053839 3055 edac_mc_handle_error(tp_event, mci, core_err_cnt,
c36e3e77 3056 m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
7d375bff 3057 4*ha+channel, dimm, -1,
03f7eae8 3058 optype, msg);
c36e3e77
MCC		 3059 return;
3060err_parsing:
c1053839 3061 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0,
c36e3e77 3062 -1, -1, -1,
03f7eae8 3063 msg, "");
eebf11a0 3064
eebf11a0
MCC		 3065}
3066
3067/*
ad08c4e9
TL		 3068 * Check that logging is enabled and that this is the right type
3069 * of error for us to handle.
eebf11a0 3070 */
3d78c9af
MCC		 3071static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
3072 void *data)
eebf11a0 3073{
3d78c9af
MCC		 3074 struct mce *mce = (struct mce *)data;
3075 struct mem_ctl_info *mci;
3076 struct sbridge_pvt *pvt;
cf40f80c 3077 char *type;
3d78c9af 3078
fd521039
CG		 3079 if (get_edac_report_status() == EDAC_REPORTING_DISABLED)
3080 return NOTIFY_DONE;
3081
3d78c9af
MCC		 3082 mci = get_mci_for_node_id(mce->socketid);
3083 if (!mci)
3084 return NOTIFY_BAD;
3085 pvt = mci->pvt_info;
eebf11a0
MCC		 3086
3087 /*
3088 * Just let mcelog handle it if the error is
3089 * outside the memory controller. A memory error
3090 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0.
3091 * bit 12 has an special meaning.
3092 */
3093 if ((mce->status & 0xefff) >> 7 != 1)
3d78c9af 3094 return NOTIFY_DONE;
eebf11a0 3095
cf40f80c
AR		 3096 if (mce->mcgstatus & MCG_STATUS_MCIP)
3097 type = "Exception";
3098 else
3099 type = "Event";
3100
49856dc9 3101 sbridge_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n");
eebf11a0 3102
49856dc9
AR		 3103 sbridge_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx "
3104 "Bank %d: %016Lx\n", mce->extcpu, type,
3105 mce->mcgstatus, mce->bank, mce->status);
3106 sbridge_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc);
3107 sbridge_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr);
3108 sbridge_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc);
eebf11a0 3109
49856dc9
AR		 3110 sbridge_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET "
3111 "%u APIC %x\n", mce->cpuvendor, mce->cpuid,
3112 mce->time, mce->socketid, mce->apicid);
eebf11a0 3113
ad08c4e9 3114 sbridge_mce_output_error(mci, mce);
eebf11a0
MCC		 3115
3116 /* Advice mcelog that the error were handled */
3d78c9af 3117 return NOTIFY_STOP;
eebf11a0
MCC		 3118}
3119
3d78c9af
MCC		 3120static struct notifier_block sbridge_mce_dec = {
3121 .notifier_call = sbridge_mce_check_error,
3122};
3123
eebf11a0
MCC		 3124/****************************************************************************
3125 EDAC register/unregister logic
3126 ****************************************************************************/
3127
3128static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev)
3129{
3130 struct mem_ctl_info *mci = sbridge_dev->mci;
3131 struct sbridge_pvt *pvt;
3132
3133 if (unlikely(!mci || !mci->pvt_info)) {
956b9ba1 3134 edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev);
eebf11a0
MCC		 3135
3136 sbridge_printk(KERN_ERR, "Couldn't find mci handler\n");
3137 return;
3138 }
3139
3140 pvt = mci->pvt_info;
3141
956b9ba1
JP		 3142 edac_dbg(0, "MC: mci = %p, dev = %p\n",
3143 mci, &sbridge_dev->pdev[0]->dev);
eebf11a0 3144
eebf11a0 3145 /* Remove MC sysfs nodes */
fd687502 3146 edac_mc_del_mc(mci->pdev);
eebf11a0 3147
956b9ba1 3148 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
eebf11a0
MCC		 3149 kfree(mci->ctl_name);
3150 edac_mc_free(mci);
3151 sbridge_dev->mci = NULL;
3152}
3153
4d715a80 3154static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type)
eebf11a0
MCC		 3155{
3156 struct mem_ctl_info *mci;
c36e3e77 3157 struct edac_mc_layer layers[2];
eebf11a0 3158 struct sbridge_pvt *pvt;
4d715a80 3159 struct pci_dev *pdev = sbridge_dev->pdev[0];
c36e3e77 3160 int rc;
eebf11a0
MCC		 3161
3162 /* Check the number of active and not disabled channels */
dbc954dd 3163 rc = check_if_ecc_is_active(sbridge_dev->bus, type);
eebf11a0
MCC		 3164 if (unlikely(rc < 0))
3165 return rc;
3166
3167 /* allocate a new MC control structure */
c36e3e77 3168 layers[0].type = EDAC_MC_LAYER_CHANNEL;
d0cdf900
JS		 3169 layers[0].size = type == KNIGHTS_LANDING ?
3170 KNL_MAX_CHANNELS : NUM_CHANNELS;
c36e3e77
MCC		 3171 layers[0].is_virt_csrow = false;
3172 layers[1].type = EDAC_MC_LAYER_SLOT;
d0cdf900 3173 layers[1].size = type == KNIGHTS_LANDING ? 1 : MAX_DIMMS;
c36e3e77 3174 layers[1].is_virt_csrow = true;
ca0907b9 3175 mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers,
c36e3e77
MCC		 3176 sizeof(*pvt));
3177
eebf11a0
MCC		 3178 if (unlikely(!mci))
3179 return -ENOMEM;
3180
956b9ba1 3181 edac_dbg(0, "MC: mci = %p, dev = %p\n",
4d715a80 3182 mci, &pdev->dev);
eebf11a0
MCC		 3183
3184 pvt = mci->pvt_info;
3185 memset(pvt, 0, sizeof(*pvt));
3186
3187 /* Associate sbridge_dev and mci for future usage */
3188 pvt->sbridge_dev = sbridge_dev;
3189 sbridge_dev->mci = mci;
3190
d0cdf900
JS		 3191 mci->mtype_cap = type == KNIGHTS_LANDING ?
3192 MEM_FLAG_DDR4 : MEM_FLAG_DDR3;
eebf11a0
MCC		 3193 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3194 mci->edac_cap = EDAC_FLAG_NONE;
3195 mci->mod_name = "sbridge_edac.c";
3196 mci->mod_ver = SBRIDGE_REVISION;
4d715a80 3197 mci->dev_name = pci_name(pdev);
eebf11a0
MCC		 3198 mci->ctl_page_to_phys = NULL;
3199
4d715a80 3200 pvt->info.type = type;
50d1bb93
AR		 3201 switch (type) {
3202 case IVY_BRIDGE:
4d715a80
AR		 3203 pvt->info.rankcfgr = IB_RANK_CFG_A;
3204 pvt->info.get_tolm = ibridge_get_tolm;
3205 pvt->info.get_tohm = ibridge_get_tohm;
3206 pvt->info.dram_rule = ibridge_dram_rule;
9e375446 3207 pvt->info.get_memory_type = get_memory_type;
f14d6892 3208 pvt->info.get_node_id = get_node_id;
b976bcf2 3209 pvt->info.rir_limit = rir_limit;
c59f9c06
JS		 3210 pvt->info.sad_limit = sad_limit;
3211 pvt->info.interleave_mode = interleave_mode;
3212 pvt->info.show_interleave_mode = show_interleave_mode;
3213 pvt->info.dram_attr = dram_attr;
4d715a80
AR		 3214 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3215 pvt->info.interleave_list = ibridge_interleave_list;
3216 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3217 pvt->info.interleave_pkg = ibridge_interleave_pkg;
12f0721c 3218 pvt->info.get_width = ibridge_get_width;
4d715a80
AR		 3219 mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge Socket#%d", mci->mc_idx);
3220
3221 /* Store pci devices at mci for faster access */
3222 rc = ibridge_mci_bind_devs(mci, sbridge_dev);
3223 if (unlikely(rc < 0))
3224 goto fail0;
50d1bb93
AR		 3225 break;
3226 case SANDY_BRIDGE:
4d715a80
AR		 3227 pvt->info.rankcfgr = SB_RANK_CFG_A;
3228 pvt->info.get_tolm = sbridge_get_tolm;
3229 pvt->info.get_tohm = sbridge_get_tohm;
3230 pvt->info.dram_rule = sbridge_dram_rule;
9e375446 3231 pvt->info.get_memory_type = get_memory_type;
f14d6892 3232 pvt->info.get_node_id = get_node_id;
b976bcf2 3233 pvt->info.rir_limit = rir_limit;
c59f9c06
JS		 3234 pvt->info.sad_limit = sad_limit;
3235 pvt->info.interleave_mode = interleave_mode;
3236 pvt->info.show_interleave_mode = show_interleave_mode;
3237 pvt->info.dram_attr = dram_attr;
4d715a80
AR		 3238 pvt->info.max_sad = ARRAY_SIZE(sbridge_dram_rule);
3239 pvt->info.interleave_list = sbridge_interleave_list;
3240 pvt->info.max_interleave = ARRAY_SIZE(sbridge_interleave_list);
3241 pvt->info.interleave_pkg = sbridge_interleave_pkg;
12f0721c 3242 pvt->info.get_width = sbridge_get_width;
4d715a80
AR		 3243 mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge Socket#%d", mci->mc_idx);
3244
3245 /* Store pci devices at mci for faster access */
3246 rc = sbridge_mci_bind_devs(mci, sbridge_dev);
3247 if (unlikely(rc < 0))
3248 goto fail0;
50d1bb93
AR		 3249 break;
3250 case HASWELL:
3251 /* rankcfgr isn't used */
3252 pvt->info.get_tolm = haswell_get_tolm;
3253 pvt->info.get_tohm = haswell_get_tohm;
3254 pvt->info.dram_rule = ibridge_dram_rule;
3255 pvt->info.get_memory_type = haswell_get_memory_type;
3256 pvt->info.get_node_id = haswell_get_node_id;
3257 pvt->info.rir_limit = haswell_rir_limit;
c59f9c06
JS		 3258 pvt->info.sad_limit = sad_limit;
3259 pvt->info.interleave_mode = interleave_mode;
3260 pvt->info.show_interleave_mode = show_interleave_mode;
3261 pvt->info.dram_attr = dram_attr;
50d1bb93
AR		 3262 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3263 pvt->info.interleave_list = ibridge_interleave_list;
3264 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3265 pvt->info.interleave_pkg = ibridge_interleave_pkg;
12f0721c 3266 pvt->info.get_width = ibridge_get_width;
50d1bb93 3267 mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell Socket#%d", mci->mc_idx);
4d715a80 3268
50d1bb93
AR		 3269 /* Store pci devices at mci for faster access */
3270 rc = haswell_mci_bind_devs(mci, sbridge_dev);
3271 if (unlikely(rc < 0))
3272 goto fail0;
3273 break;
1f39581a
TL		 3274 case BROADWELL:
3275 /* rankcfgr isn't used */
3276 pvt->info.get_tolm = haswell_get_tolm;
3277 pvt->info.get_tohm = haswell_get_tohm;
3278 pvt->info.dram_rule = ibridge_dram_rule;
3279 pvt->info.get_memory_type = haswell_get_memory_type;
3280 pvt->info.get_node_id = haswell_get_node_id;
3281 pvt->info.rir_limit = haswell_rir_limit;
c59f9c06
JS		 3282 pvt->info.sad_limit = sad_limit;
3283 pvt->info.interleave_mode = interleave_mode;
3284 pvt->info.show_interleave_mode = show_interleave_mode;
3285 pvt->info.dram_attr = dram_attr;
1f39581a
TL		 3286 pvt->info.max_sad = ARRAY_SIZE(ibridge_dram_rule);
3287 pvt->info.interleave_list = ibridge_interleave_list;
3288 pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list);
3289 pvt->info.interleave_pkg = ibridge_interleave_pkg;
12f0721c 3290 pvt->info.get_width = broadwell_get_width;
1f39581a
TL		 3291 mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell Socket#%d", mci->mc_idx);
3292
3293 /* Store pci devices at mci for faster access */
3294 rc = broadwell_mci_bind_devs(mci, sbridge_dev);
3295 if (unlikely(rc < 0))
3296 goto fail0;
3297 break;
d0cdf900
JS		 3298 case KNIGHTS_LANDING:
3299 /* pvt->info.rankcfgr == ??? */
3300 pvt->info.get_tolm = knl_get_tolm;
3301 pvt->info.get_tohm = knl_get_tohm;
3302 pvt->info.dram_rule = knl_dram_rule;
3303 pvt->info.get_memory_type = knl_get_memory_type;
3304 pvt->info.get_node_id = knl_get_node_id;
3305 pvt->info.rir_limit = NULL;
3306 pvt->info.sad_limit = knl_sad_limit;
3307 pvt->info.interleave_mode = knl_interleave_mode;
3308 pvt->info.show_interleave_mode = knl_show_interleave_mode;
3309 pvt->info.dram_attr = dram_attr_knl;
3310 pvt->info.max_sad = ARRAY_SIZE(knl_dram_rule);
3311 pvt->info.interleave_list = knl_interleave_list;
3312 pvt->info.max_interleave = ARRAY_SIZE(knl_interleave_list);
3313 pvt->info.interleave_pkg = ibridge_interleave_pkg;
45f4d3ab 3314 pvt->info.get_width = knl_get_width;
d0cdf900
JS		 3315 mci->ctl_name = kasprintf(GFP_KERNEL,
3316 "Knights Landing Socket#%d", mci->mc_idx);
3317
3318 rc = knl_mci_bind_devs(mci, sbridge_dev);
3319 if (unlikely(rc < 0))
3320 goto fail0;
3321 break;
50d1bb93 3322 }
eebf11a0
MCC		 3323
3324 /* Get dimm basic config and the memory layout */
3325 get_dimm_config(mci);
3326 get_memory_layout(mci);
3327
3328 /* record ptr to the generic device */
4d715a80 3329 mci->pdev = &pdev->dev;
eebf11a0
MCC		 3330
3331 /* add this new MC control structure to EDAC's list of MCs */
3332 if (unlikely(edac_mc_add_mc(mci))) {
956b9ba1 3333 edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
eebf11a0
MCC		 3334 rc = -EINVAL;
3335 goto fail0;
3336 }
3337
eebf11a0 3338 return 0;
eebf11a0
MCC		 3339
3340fail0:
3341 kfree(mci->ctl_name);
3342 edac_mc_free(mci);
3343 sbridge_dev->mci = NULL;
3344 return rc;
3345}
3346
3347/*
3348 * sbridge_probe Probe for ONE instance of device to see if it is
3349 * present.
3350 * return:
3351 * 0 for FOUND a device
3352 * < 0 for error code
3353 */
3354
9b3c6e85 3355static int sbridge_probe(struct pci_dev *pdev, const struct pci_device_id *id)
eebf11a0 3356{
50d1bb93 3357 int rc = -ENODEV;
eebf11a0
MCC		 3358 u8 mc, num_mc = 0;
3359 struct sbridge_dev *sbridge_dev;
50d1bb93 3360 enum type type = SANDY_BRIDGE;
eebf11a0
MCC		 3361
3362 /* get the pci devices we want to reserve for our use */
3363 mutex_lock(&sbridge_edac_lock);
3364
3365 /*
3366 * All memory controllers are allocated at the first pass.
3367 */
3368 if (unlikely(probed >= 1)) {
3369 mutex_unlock(&sbridge_edac_lock);
3370 return -ENODEV;
3371 }
3372 probed++;
3373
50d1bb93
AR		 3374 switch (pdev->device) {
3375 case PCI_DEVICE_ID_INTEL_IBRIDGE_IMC_HA0_TA:
d0cdf900
JS		 3376 rc = sbridge_get_all_devices(&num_mc,
3377 pci_dev_descr_ibridge_table);
4d715a80 3378 type = IVY_BRIDGE;
50d1bb93 3379 break;
11249e73 3380 case PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0:
d0cdf900
JS		 3381 rc = sbridge_get_all_devices(&num_mc,
3382 pci_dev_descr_sbridge_table);
4d715a80 3383 type = SANDY_BRIDGE;
50d1bb93
AR		 3384 break;
3385 case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0:
d0cdf900
JS		 3386 rc = sbridge_get_all_devices(&num_mc,
3387 pci_dev_descr_haswell_table);
50d1bb93
AR		 3388 type = HASWELL;
3389 break;
1f39581a 3390 case PCI_DEVICE_ID_INTEL_BROADWELL_IMC_HA0:
d0cdf900
JS		 3391 rc = sbridge_get_all_devices(&num_mc,
3392 pci_dev_descr_broadwell_table);
1f39581a 3393 type = BROADWELL;
d0cdf900
JS		 3394 break;
3395 case PCI_DEVICE_ID_INTEL_KNL_IMC_SAD0:
3396 rc = sbridge_get_all_devices_knl(&num_mc,
3397 pci_dev_descr_knl_table);
3398 type = KNIGHTS_LANDING;
1f39581a 3399 break;
4d715a80 3400 }
11249e73
BP		 3401 if (unlikely(rc < 0)) {
3402 edac_dbg(0, "couldn't get all devices for 0x%x\n", pdev->device);
eebf11a0 3403 goto fail0;
11249e73
BP		 3404 }
3405
eebf11a0
MCC		 3406 mc = 0;
3407
3408 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
956b9ba1
JP		 3409 edac_dbg(0, "Registering MC#%d (%d of %d)\n",
3410 mc, mc + 1, num_mc);
50d1bb93 3411
eebf11a0 3412 sbridge_dev->mc = mc++;
4d715a80 3413 rc = sbridge_register_mci(sbridge_dev, type);
eebf11a0
MCC		 3414 if (unlikely(rc < 0))
3415 goto fail1;
3416 }
3417
11249e73 3418 sbridge_printk(KERN_INFO, "%s\n", SBRIDGE_REVISION);
eebf11a0
MCC		 3419
3420 mutex_unlock(&sbridge_edac_lock);
3421 return 0;
3422
3423fail1:
3424 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
3425 sbridge_unregister_mci(sbridge_dev);
3426
3427 sbridge_put_all_devices();
3428fail0:
3429 mutex_unlock(&sbridge_edac_lock);
3430 return rc;
3431}
3432
3433/*
3434 * sbridge_remove destructor for one instance of device
3435 *
3436 */
9b3c6e85 3437static void sbridge_remove(struct pci_dev *pdev)
eebf11a0
MCC		 3438{
3439 struct sbridge_dev *sbridge_dev;
3440
956b9ba1 3441 edac_dbg(0, "\n");
eebf11a0
MCC		 3442
3443 /*
3444 * we have a trouble here: pdev value for removal will be wrong, since
3445 * it will point to the X58 register used to detect that the machine
3446 * is a Nehalem or upper design. However, due to the way several PCI
3447 * devices are grouped together to provide MC functionality, we need
3448 * to use a different method for releasing the devices
3449 */
3450
3451 mutex_lock(&sbridge_edac_lock);
3452
3453 if (unlikely(!probed)) {
3454 mutex_unlock(&sbridge_edac_lock);
3455 return;
3456 }
3457
3458 list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
3459 sbridge_unregister_mci(sbridge_dev);
3460
3461 /* Release PCI resources */
3462 sbridge_put_all_devices();
3463
3464 probed--;
3465
3466 mutex_unlock(&sbridge_edac_lock);
3467}
3468
3469MODULE_DEVICE_TABLE(pci, sbridge_pci_tbl);
3470
3471/*
3472 * sbridge_driver pci_driver structure for this module
3473 *
3474 */
3475static struct pci_driver sbridge_driver = {
3476 .name = "sbridge_edac",
3477 .probe = sbridge_probe,
9b3c6e85 3478 .remove = sbridge_remove,
eebf11a0
MCC		 3479 .id_table = sbridge_pci_tbl,
3480};
3481
3482/*
3483 * sbridge_init Module entry function
3484 * Try to initialize this module for its devices
3485 */
3486static int __init sbridge_init(void)
3487{
3488 int pci_rc;
3489
956b9ba1 3490 edac_dbg(2, "\n");
eebf11a0
MCC		 3491
3492 /* Ensure that the OPSTATE is set correctly for POLL or NMI */
3493 opstate_init();
3494
3495 pci_rc = pci_register_driver(&sbridge_driver);
e35fca47
CG		 3496 if (pci_rc >= 0) {
3497 mce_register_decode_chain(&sbridge_mce_dec);
fd521039
CG		 3498 if (get_edac_report_status() == EDAC_REPORTING_DISABLED)
3499 sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n");
eebf11a0 3500 return 0;
e35fca47 3501 }
eebf11a0
MCC		 3502
3503 sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n",
3504 pci_rc);
3505
3506 return pci_rc;
3507}
3508
3509/*
3510 * sbridge_exit() Module exit function
3511 * Unregister the driver
3512 */
3513static void __exit sbridge_exit(void)
3514{
956b9ba1 3515 edac_dbg(2, "\n");
eebf11a0 3516 pci_unregister_driver(&sbridge_driver);
e35fca47 3517 mce_unregister_decode_chain(&sbridge_mce_dec);
eebf11a0
MCC		 3518}
3519
3520module_init(sbridge_init);
3521module_exit(sbridge_exit);
3522
3523module_param(edac_op_state, int, 0444);
3524MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
3525
3526MODULE_LICENSE("GPL");
37e59f87 3527MODULE_AUTHOR("Mauro Carvalho Chehab");
eebf11a0 3528MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
4d715a80 3529MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - "
eebf11a0 3530 SBRIDGE_REVISION);
Linux kernel - Deliverables
This page took 0.380501 seconds and 5 git commands to generate.