projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
mtd: nand: sunxi: check ecc->size values
[deliverable/linux.git] / drivers / mtd / nand / sunxi_nand.c
1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/of_gpio.h>
33 #include <linux/mtd/mtd.h>
34 #include <linux/mtd/nand.h>
35 #include <linux/mtd/partitions.h>
36 #include <linux/clk.h>
37 #include <linux/delay.h>
38 #include <linux/dmaengine.h>
39 #include <linux/gpio.h>
40 #include <linux/interrupt.h>
41 #include <linux/iopoll.h>
42
43 #define NFC_REG_CTL 0x0000
44 #define NFC_REG_ST 0x0004
45 #define NFC_REG_INT 0x0008
46 #define NFC_REG_TIMING_CTL 0x000C
47 #define NFC_REG_TIMING_CFG 0x0010
48 #define NFC_REG_ADDR_LOW 0x0014
49 #define NFC_REG_ADDR_HIGH 0x0018
50 #define NFC_REG_SECTOR_NUM 0x001C
51 #define NFC_REG_CNT 0x0020
52 #define NFC_REG_CMD 0x0024
53 #define NFC_REG_RCMD_SET 0x0028
54 #define NFC_REG_WCMD_SET 0x002C
55 #define NFC_REG_IO_DATA 0x0030
56 #define NFC_REG_ECC_CTL 0x0034
57 #define NFC_REG_ECC_ST 0x0038
58 #define NFC_REG_DEBUG 0x003C
59 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
60 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
61 #define NFC_REG_SPARE_AREA 0x00A0
62 #define NFC_REG_PAT_ID 0x00A4
63 #define NFC_RAM0_BASE 0x0400
64 #define NFC_RAM1_BASE 0x0800
65
66 /* define bit use in NFC_CTL */
67 #define NFC_EN BIT(0)
68 #define NFC_RESET BIT(1)
69 #define NFC_BUS_WIDTH_MSK BIT(2)
70 #define NFC_BUS_WIDTH_8 (0 << 2)
71 #define NFC_BUS_WIDTH_16 (1 << 2)
72 #define NFC_RB_SEL_MSK BIT(3)
73 #define NFC_RB_SEL(x) ((x) << 3)
74 #define NFC_CE_SEL_MSK GENMASK(26, 24)
75 #define NFC_CE_SEL(x) ((x) << 24)
76 #define NFC_CE_CTL BIT(6)
77 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
78 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
79 #define NFC_SAM BIT(12)
80 #define NFC_RAM_METHOD BIT(14)
81 #define NFC_DEBUG_CTL BIT(31)
82
83 /* define bit use in NFC_ST */
84 #define NFC_RB_B2R BIT(0)
85 #define NFC_CMD_INT_FLAG BIT(1)
86 #define NFC_DMA_INT_FLAG BIT(2)
87 #define NFC_CMD_FIFO_STATUS BIT(3)
88 #define NFC_STA BIT(4)
89 #define NFC_NATCH_INT_FLAG BIT(5)
90 #define NFC_RB_STATE(x) BIT(x + 8)
91
92 /* define bit use in NFC_INT */
93 #define NFC_B2R_INT_ENABLE BIT(0)
94 #define NFC_CMD_INT_ENABLE BIT(1)
95 #define NFC_DMA_INT_ENABLE BIT(2)
96 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
97 NFC_CMD_INT_ENABLE | \
98 NFC_DMA_INT_ENABLE)
99
100 /* define bit use in NFC_TIMING_CTL */
101 #define NFC_TIMING_CTL_EDO BIT(8)
102
103 /* define NFC_TIMING_CFG register layout */
104 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
105 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
106 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
107 (((tCAD) & 0x7) << 8))
108
109 /* define bit use in NFC_CMD */
110 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
111 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
112 #define NFC_CMD(x) (x)
113 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
114 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
115 #define NFC_SEND_ADR BIT(19)
116 #define NFC_ACCESS_DIR BIT(20)
117 #define NFC_DATA_TRANS BIT(21)
118 #define NFC_SEND_CMD1 BIT(22)
119 #define NFC_WAIT_FLAG BIT(23)
120 #define NFC_SEND_CMD2 BIT(24)
121 #define NFC_SEQ BIT(25)
122 #define NFC_DATA_SWAP_METHOD BIT(26)
123 #define NFC_ROW_AUTO_INC BIT(27)
124 #define NFC_SEND_CMD3 BIT(28)
125 #define NFC_SEND_CMD4 BIT(29)
126 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
127 #define NFC_NORMAL_OP (0 << 30)
128 #define NFC_ECC_OP (1 << 30)
129 #define NFC_PAGE_OP (2 << 30)
130
131 /* define bit use in NFC_RCMD_SET */
132 #define NFC_READ_CMD_MSK GENMASK(7, 0)
133 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
134 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
135
136 /* define bit use in NFC_WCMD_SET */
137 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
138 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
139 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
140 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
141
142 /* define bit use in NFC_ECC_CTL */
143 #define NFC_ECC_EN BIT(0)
144 #define NFC_ECC_PIPELINE BIT(3)
145 #define NFC_ECC_EXCEPTION BIT(4)
146 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
147 #define NFC_RANDOM_EN BIT(9)
148 #define NFC_RANDOM_DIRECTION BIT(10)
149 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
150 #define NFC_ECC_MODE(x) ((x) << 12)
151 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
152 #define NFC_RANDOM_SEED(x) ((x) << 16)
153
154 /* define bit use in NFC_ECC_ST */
155 #define NFC_ECC_ERR(x) BIT(x)
156 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
157 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
158 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
159
160 #define NFC_DEFAULT_TIMEOUT_MS 1000
161
162 #define NFC_SRAM_SIZE 1024
163
164 #define NFC_MAX_CS 7
165
166 /*
167 * Ready/Busy detection type: describes the Ready/Busy detection modes
168 *
169 * @RB_NONE: no external detection available, rely on STATUS command
170 * and software timeouts
171 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
172 * pin of the NAND flash chip must be connected to one of the
173 * native NAND R/B pins (those which can be muxed to the NAND
174 * Controller)
175 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
176 * pin of the NAND flash chip must be connected to a GPIO capable
177 * pin.
178 */
179 enum sunxi_nand_rb_type {
180 RB_NONE,
181 RB_NATIVE,
182 RB_GPIO,
183 };
184
185 /*
186 * Ready/Busy structure: stores information related to Ready/Busy detection
187 *
188 * @type: the Ready/Busy detection mode
189 * @info: information related to the R/B detection mode. Either a gpio
190 * id or a native R/B id (those supported by the NAND controller).
191 */
192 struct sunxi_nand_rb {
193 enum sunxi_nand_rb_type type;
194 union {
195 int gpio;
196 int nativeid;
197 } info;
198 };
199
200 /*
201 * Chip Select structure: stores information related to NAND Chip Select
202 *
203 * @cs: the NAND CS id used to communicate with a NAND Chip
204 * @rb: the Ready/Busy description
205 */
206 struct sunxi_nand_chip_sel {
207 u8 cs;
208 struct sunxi_nand_rb rb;
209 };
210
211 /*
212 * sunxi HW ECC infos: stores information related to HW ECC support
213 *
214 * @mode: the sunxi ECC mode field deduced from ECC requirements
215 */
216 struct sunxi_nand_hw_ecc {
217 int mode;
218 };
219
220 /*
221 * NAND chip structure: stores NAND chip device related information
222 *
223 * @node: used to store NAND chips into a list
224 * @nand: base NAND chip structure
225 * @mtd: base MTD structure
226 * @clk_rate: clk_rate required for this NAND chip
227 * @timing_cfg TIMING_CFG register value for this NAND chip
228 * @selected: current active CS
229 * @nsels: number of CS lines required by the NAND chip
230 * @sels: array of CS lines descriptions
231 */
232 struct sunxi_nand_chip {
233 struct list_head node;
234 struct nand_chip nand;
235 unsigned long clk_rate;
236 u32 timing_cfg;
237 u32 timing_ctl;
238 int selected;
239 int addr_cycles;
240 u32 addr[2];
241 int cmd_cycles;
242 u8 cmd[2];
243 int nsels;
244 struct sunxi_nand_chip_sel sels[0];
245 };
246
247 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
248 {
249 return container_of(nand, struct sunxi_nand_chip, nand);
250 }
251
252 /*
253 * NAND Controller structure: stores sunxi NAND controller information
254 *
255 * @controller: base controller structure
256 * @dev: parent device (used to print error messages)
257 * @regs: NAND controller registers
258 * @ahb_clk: NAND Controller AHB clock
259 * @mod_clk: NAND Controller mod clock
260 * @assigned_cs: bitmask describing already assigned CS lines
261 * @clk_rate: NAND controller current clock rate
262 * @chips: a list containing all the NAND chips attached to
263 * this NAND controller
264 * @complete: a completion object used to wait for NAND
265 * controller events
266 */
267 struct sunxi_nfc {
268 struct nand_hw_control controller;
269 struct device *dev;
270 void __iomem *regs;
271 struct clk *ahb_clk;
272 struct clk *mod_clk;
273 unsigned long assigned_cs;
274 unsigned long clk_rate;
275 struct list_head chips;
276 struct completion complete;
277 struct dma_chan *dmac;
278 };
279
280 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
281 {
282 return container_of(ctrl, struct sunxi_nfc, controller);
283 }
284
285 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
286 {
287 struct sunxi_nfc *nfc = dev_id;
288 u32 st = readl(nfc->regs + NFC_REG_ST);
289 u32 ien = readl(nfc->regs + NFC_REG_INT);
290
291 if (!(ien & st))
292 return IRQ_NONE;
293
294 if ((ien & st) == ien)
295 complete(&nfc->complete);
296
297 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
298 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
299
300 return IRQ_HANDLED;
301 }
302
303 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
304 bool use_polling, unsigned int timeout_ms)
305 {
306 int ret;
307
308 if (events & ~NFC_INT_MASK)
309 return -EINVAL;
310
311 if (!timeout_ms)
312 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
313
314 if (!use_polling) {
315 init_completion(&nfc->complete);
316
317 writel(events, nfc->regs + NFC_REG_INT);
318
319 ret = wait_for_completion_timeout(&nfc->complete,
320 msecs_to_jiffies(timeout_ms));
321
322 writel(0, nfc->regs + NFC_REG_INT);
323 } else {
324 u32 status;
325
326 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
327 (status & events) == events, 1,
328 timeout_ms * 1000);
329 }
330
331 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
332
333 if (ret)
334 dev_err(nfc->dev, "wait interrupt timedout\n");
335
336 return ret;
337 }
338
339 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
340 {
341 u32 status;
342 int ret;
343
344 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
345 !(status & NFC_CMD_FIFO_STATUS), 1,
346 NFC_DEFAULT_TIMEOUT_MS * 1000);
347 if (ret)
348 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
349
350 return ret;
351 }
352
353 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
354 {
355 u32 ctl;
356 int ret;
357
358 writel(0, nfc->regs + NFC_REG_ECC_CTL);
359 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
360
361 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
362 !(ctl & NFC_RESET), 1,
363 NFC_DEFAULT_TIMEOUT_MS * 1000);
364 if (ret)
365 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
366
367 return ret;
368 }
369
370 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
371 int chunksize, int nchunks,
372 enum dma_data_direction ddir,
373 struct scatterlist *sg)
374 {
375 struct nand_chip *nand = mtd_to_nand(mtd);
376 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
377 struct dma_async_tx_descriptor *dmad;
378 enum dma_transfer_direction tdir;
379 dma_cookie_t dmat;
380 int ret;
381
382 if (ddir == DMA_FROM_DEVICE)
383 tdir = DMA_DEV_TO_MEM;
384 else
385 tdir = DMA_MEM_TO_DEV;
386
387 sg_init_one(sg, buf, nchunks * chunksize);
388 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
389 if (!ret)
390 return -ENOMEM;
391
392 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
393 if (!dmad) {
394 ret = -EINVAL;
395 goto err_unmap_buf;
396 }
397
398 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
399 nfc->regs + NFC_REG_CTL);
400 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
401 writel(chunksize, nfc->regs + NFC_REG_CNT);
402 dmat = dmaengine_submit(dmad);
403
404 ret = dma_submit_error(dmat);
405 if (ret)
406 goto err_clr_dma_flag;
407
408 return 0;
409
410 err_clr_dma_flag:
411 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
412 nfc->regs + NFC_REG_CTL);
413
414 err_unmap_buf:
415 dma_unmap_sg(nfc->dev, sg, 1, ddir);
416 return ret;
417 }
418
419 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
420 enum dma_data_direction ddir,
421 struct scatterlist *sg)
422 {
423 struct nand_chip *nand = mtd_to_nand(mtd);
424 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
425
426 dma_unmap_sg(nfc->dev, sg, 1, ddir);
427 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
428 nfc->regs + NFC_REG_CTL);
429 }
430
431 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
432 {
433 struct nand_chip *nand = mtd_to_nand(mtd);
434 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
435 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
436 struct sunxi_nand_rb *rb;
437 int ret;
438
439 if (sunxi_nand->selected < 0)
440 return 0;
441
442 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
443
444 switch (rb->type) {
445 case RB_NATIVE:
446 ret = !!(readl(nfc->regs + NFC_REG_ST) &
447 NFC_RB_STATE(rb->info.nativeid));
448 break;
449 case RB_GPIO:
450 ret = gpio_get_value(rb->info.gpio);
451 break;
452 case RB_NONE:
453 default:
454 ret = 0;
455 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
456 break;
457 }
458
459 return ret;
460 }
461
462 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
463 {
464 struct nand_chip *nand = mtd_to_nand(mtd);
465 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
466 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
467 struct sunxi_nand_chip_sel *sel;
468 u32 ctl;
469
470 if (chip > 0 && chip >= sunxi_nand->nsels)
471 return;
472
473 if (chip == sunxi_nand->selected)
474 return;
475
476 ctl = readl(nfc->regs + NFC_REG_CTL) &
477 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
478
479 if (chip >= 0) {
480 sel = &sunxi_nand->sels[chip];
481
482 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
483 NFC_PAGE_SHIFT(nand->page_shift);
484 if (sel->rb.type == RB_NONE) {
485 nand->dev_ready = NULL;
486 } else {
487 nand->dev_ready = sunxi_nfc_dev_ready;
488 if (sel->rb.type == RB_NATIVE)
489 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
490 }
491
492 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
493
494 if (nfc->clk_rate != sunxi_nand->clk_rate) {
495 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
496 nfc->clk_rate = sunxi_nand->clk_rate;
497 }
498 }
499
500 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
501 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
502 writel(ctl, nfc->regs + NFC_REG_CTL);
503
504 sunxi_nand->selected = chip;
505 }
506
507 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
508 {
509 struct nand_chip *nand = mtd_to_nand(mtd);
510 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
511 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
512 int ret;
513 int cnt;
514 int offs = 0;
515 u32 tmp;
516
517 while (len > offs) {
518 cnt = min(len - offs, NFC_SRAM_SIZE);
519
520 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
521 if (ret)
522 break;
523
524 writel(cnt, nfc->regs + NFC_REG_CNT);
525 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
526 writel(tmp, nfc->regs + NFC_REG_CMD);
527
528 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
529 if (ret)
530 break;
531
532 if (buf)
533 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
534 cnt);
535 offs += cnt;
536 }
537 }
538
539 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
540 int len)
541 {
542 struct nand_chip *nand = mtd_to_nand(mtd);
543 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
544 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
545 int ret;
546 int cnt;
547 int offs = 0;
548 u32 tmp;
549
550 while (len > offs) {
551 cnt = min(len - offs, NFC_SRAM_SIZE);
552
553 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
554 if (ret)
555 break;
556
557 writel(cnt, nfc->regs + NFC_REG_CNT);
558 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
559 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
560 NFC_ACCESS_DIR;
561 writel(tmp, nfc->regs + NFC_REG_CMD);
562
563 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
564 if (ret)
565 break;
566
567 offs += cnt;
568 }
569 }
570
571 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
572 {
573 uint8_t ret;
574
575 sunxi_nfc_read_buf(mtd, &ret, 1);
576
577 return ret;
578 }
579
580 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
581 unsigned int ctrl)
582 {
583 struct nand_chip *nand = mtd_to_nand(mtd);
584 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
585 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
586 int ret;
587
588 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
589 if (ret)
590 return;
591
592 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
593 !(ctrl & (NAND_CLE | NAND_ALE))) {
594 u32 cmd = 0;
595
596 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
597 return;
598
599 if (sunxi_nand->cmd_cycles--)
600 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
601
602 if (sunxi_nand->cmd_cycles--) {
603 cmd |= NFC_SEND_CMD2;
604 writel(sunxi_nand->cmd[1],
605 nfc->regs + NFC_REG_RCMD_SET);
606 }
607
608 sunxi_nand->cmd_cycles = 0;
609
610 if (sunxi_nand->addr_cycles) {
611 cmd |= NFC_SEND_ADR |
612 NFC_ADR_NUM(sunxi_nand->addr_cycles);
613 writel(sunxi_nand->addr[0],
614 nfc->regs + NFC_REG_ADDR_LOW);
615 }
616
617 if (sunxi_nand->addr_cycles > 4)
618 writel(sunxi_nand->addr[1],
619 nfc->regs + NFC_REG_ADDR_HIGH);
620
621 writel(cmd, nfc->regs + NFC_REG_CMD);
622 sunxi_nand->addr[0] = 0;
623 sunxi_nand->addr[1] = 0;
624 sunxi_nand->addr_cycles = 0;
625 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
626 }
627
628 if (ctrl & NAND_CLE) {
629 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
630 } else if (ctrl & NAND_ALE) {
631 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
632 dat << ((sunxi_nand->addr_cycles % 4) * 8);
633 sunxi_nand->addr_cycles++;
634 }
635 }
636
637 /* These seed values have been extracted from Allwinner's BSP */
638 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
639 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
640 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
641 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
642 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
643 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
644 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
645 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
646 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
647 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
648 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
649 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
650 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
651 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
652 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
653 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
654 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
655 };
656
657 /*
658 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
659 * have been generated using
660 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
661 * the randomizer engine does internally before de/scrambling OOB data.
662 *
663 * Those tables are statically defined to avoid calculating randomizer state
664 * at runtime.
665 */
666 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
667 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
668 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
669 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
670 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
671 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
672 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
673 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
674 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
675 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
676 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
677 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
678 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
679 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
680 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
681 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
682 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
683 };
684
685 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
686 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
687 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
688 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
689 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
690 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
691 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
692 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
693 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
694 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
695 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
696 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
697 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
698 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
699 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
700 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
701 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
702 };
703
704 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
705 {
706 state &= 0x7fff;
707
708 /*
709 * This loop is just a simple implementation of a Fibonacci LFSR using
710 * the x16 + x15 + 1 polynomial.
711 */
712 while (count--)
713 state = ((state >> 1) |
714 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
715
716 return state;
717 }
718
719 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
720 {
721 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
722 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
723
724 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
725 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
726
727 if (ecc) {
728 if (mtd->ecc_step_size == 512)
729 seeds = sunxi_nfc_randomizer_ecc512_seeds;
730 else
731 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
732 }
733
734 return seeds[page % mod];
735 }
736
737 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
738 int page, bool ecc)
739 {
740 struct nand_chip *nand = mtd_to_nand(mtd);
741 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
742 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
743 u16 state;
744
745 if (!(nand->options & NAND_NEED_SCRAMBLING))
746 return;
747
748 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
749 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
750 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
751 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
752 }
753
754 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
755 {
756 struct nand_chip *nand = mtd_to_nand(mtd);
757 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
758
759 if (!(nand->options & NAND_NEED_SCRAMBLING))
760 return;
761
762 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
763 nfc->regs + NFC_REG_ECC_CTL);
764 }
765
766 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
767 {
768 struct nand_chip *nand = mtd_to_nand(mtd);
769 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
770
771 if (!(nand->options & NAND_NEED_SCRAMBLING))
772 return;
773
774 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
775 nfc->regs + NFC_REG_ECC_CTL);
776 }
777
778 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
779 {
780 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
781
782 bbm[0] ^= state;
783 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
784 }
785
786 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
787 const uint8_t *buf, int len,
788 bool ecc, int page)
789 {
790 sunxi_nfc_randomizer_config(mtd, page, ecc);
791 sunxi_nfc_randomizer_enable(mtd);
792 sunxi_nfc_write_buf(mtd, buf, len);
793 sunxi_nfc_randomizer_disable(mtd);
794 }
795
796 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
797 int len, bool ecc, int page)
798 {
799 sunxi_nfc_randomizer_config(mtd, page, ecc);
800 sunxi_nfc_randomizer_enable(mtd);
801 sunxi_nfc_read_buf(mtd, buf, len);
802 sunxi_nfc_randomizer_disable(mtd);
803 }
804
805 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
806 {
807 struct nand_chip *nand = mtd_to_nand(mtd);
808 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
809 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
810 u32 ecc_ctl;
811
812 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
813 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
814 NFC_ECC_BLOCK_SIZE_MSK);
815 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
816 NFC_ECC_PIPELINE;
817
818 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
819 }
820
821 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
822 {
823 struct nand_chip *nand = mtd_to_nand(mtd);
824 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
825
826 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
827 nfc->regs + NFC_REG_ECC_CTL);
828 }
829
830 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
831 {
832 buf[0] = user_data;
833 buf[1] = user_data >> 8;
834 buf[2] = user_data >> 16;
835 buf[3] = user_data >> 24;
836 }
837
838 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
839 {
840 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
841 }
842
843 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
844 int step, bool bbm, int page)
845 {
846 struct nand_chip *nand = mtd_to_nand(mtd);
847 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
848
849 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
850 oob);
851
852 /* De-randomize the Bad Block Marker. */
853 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
854 sunxi_nfc_randomize_bbm(mtd, page, oob);
855 }
856
857 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
858 const u8 *oob, int step,
859 bool bbm, int page)
860 {
861 struct nand_chip *nand = mtd_to_nand(mtd);
862 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
863 u8 user_data[4];
864
865 /* Randomize the Bad Block Marker. */
866 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
867 memcpy(user_data, oob, sizeof(user_data));
868 sunxi_nfc_randomize_bbm(mtd, page, user_data);
869 oob = user_data;
870 }
871
872 writel(sunxi_nfc_buf_to_user_data(oob),
873 nfc->regs + NFC_REG_USER_DATA(step));
874 }
875
876 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
877 unsigned int *max_bitflips, int ret)
878 {
879 if (ret < 0) {
880 mtd->ecc_stats.failed++;
881 } else {
882 mtd->ecc_stats.corrected += ret;
883 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
884 }
885 }
886
887 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
888 int step, u32 status, bool *erased)
889 {
890 struct nand_chip *nand = mtd_to_nand(mtd);
891 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
892 struct nand_ecc_ctrl *ecc = &nand->ecc;
893 u32 tmp;
894
895 *erased = false;
896
897 if (status & NFC_ECC_ERR(step))
898 return -EBADMSG;
899
900 if (status & NFC_ECC_PAT_FOUND(step)) {
901 u8 pattern;
902
903 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
904 pattern = 0x0;
905 } else {
906 pattern = 0xff;
907 *erased = true;
908 }
909
910 if (data)
911 memset(data, pattern, ecc->size);
912
913 if (oob)
914 memset(oob, pattern, ecc->bytes + 4);
915
916 return 0;
917 }
918
919 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
920
921 return NFC_ECC_ERR_CNT(step, tmp);
922 }
923
924 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
925 u8 *data, int data_off,
926 u8 *oob, int oob_off,
927 int *cur_off,
928 unsigned int *max_bitflips,
929 bool bbm, bool oob_required, int page)
930 {
931 struct nand_chip *nand = mtd_to_nand(mtd);
932 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
933 struct nand_ecc_ctrl *ecc = &nand->ecc;
934 int raw_mode = 0;
935 bool erased;
936 int ret;
937
938 if (*cur_off != data_off)
939 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
940
941 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
942
943 if (data_off + ecc->size != oob_off)
944 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
945
946 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
947 if (ret)
948 return ret;
949
950 sunxi_nfc_randomizer_enable(mtd);
951 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
952 nfc->regs + NFC_REG_CMD);
953
954 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
955 sunxi_nfc_randomizer_disable(mtd);
956 if (ret)
957 return ret;
958
959 *cur_off = oob_off + ecc->bytes + 4;
960
961 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
962 readl(nfc->regs + NFC_REG_ECC_ST),
963 &erased);
964 if (erased)
965 return 1;
966
967 if (ret < 0) {
968 /*
969 * Re-read the data with the randomizer disabled to identify
970 * bitflips in erased pages.
971 */
972 if (nand->options & NAND_NEED_SCRAMBLING) {
973 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
974 nand->read_buf(mtd, data, ecc->size);
975 } else {
976 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
977 ecc->size);
978 }
979
980 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
981 nand->read_buf(mtd, oob, ecc->bytes + 4);
982
983 ret = nand_check_erased_ecc_chunk(data, ecc->size,
984 oob, ecc->bytes + 4,
985 NULL, 0, ecc->strength);
986 if (ret >= 0)
987 raw_mode = 1;
988 } else {
989 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
990
991 if (oob_required) {
992 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
993 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
994 true, page);
995
996 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
997 bbm, page);
998 }
999 }
1000
1001 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1002
1003 return raw_mode;
1004 }
1005
1006 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1007 u8 *oob, int *cur_off,
1008 bool randomize, int page)
1009 {
1010 struct nand_chip *nand = mtd_to_nand(mtd);
1011 struct nand_ecc_ctrl *ecc = &nand->ecc;
1012 int offset = ((ecc->bytes + 4) * ecc->steps);
1013 int len = mtd->oobsize - offset;
1014
1015 if (len <= 0)
1016 return;
1017
1018 if (!cur_off || *cur_off != offset)
1019 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1020 offset + mtd->writesize, -1);
1021
1022 if (!randomize)
1023 sunxi_nfc_read_buf(mtd, oob + offset, len);
1024 else
1025 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1026 false, page);
1027
1028 if (cur_off)
1029 *cur_off = mtd->oobsize + mtd->writesize;
1030 }
1031
1032 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1033 int oob_required, int page,
1034 int nchunks)
1035 {
1036 struct nand_chip *nand = mtd_to_nand(mtd);
1037 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1038 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1039 struct nand_ecc_ctrl *ecc = &nand->ecc;
1040 unsigned int max_bitflips = 0;
1041 int ret, i, raw_mode = 0;
1042 struct scatterlist sg;
1043 u32 status;
1044
1045 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1046 if (ret)
1047 return ret;
1048
1049 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1050 DMA_FROM_DEVICE, &sg);
1051 if (ret)
1052 return ret;
1053
1054 sunxi_nfc_hw_ecc_enable(mtd);
1055 sunxi_nfc_randomizer_config(mtd, page, false);
1056 sunxi_nfc_randomizer_enable(mtd);
1057
1058 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1059 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1060
1061 dma_async_issue_pending(nfc->dmac);
1062
1063 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1064 nfc->regs + NFC_REG_CMD);
1065
1066 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1067 if (ret)
1068 dmaengine_terminate_all(nfc->dmac);
1069
1070 sunxi_nfc_randomizer_disable(mtd);
1071 sunxi_nfc_hw_ecc_disable(mtd);
1072
1073 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1074
1075 if (ret)
1076 return ret;
1077
1078 status = readl(nfc->regs + NFC_REG_ECC_ST);
1079
1080 for (i = 0; i < nchunks; i++) {
1081 int data_off = i * ecc->size;
1082 int oob_off = i * (ecc->bytes + 4);
1083 u8 *data = buf + data_off;
1084 u8 *oob = nand->oob_poi + oob_off;
1085 bool erased;
1086
1087 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1088 oob_required ? oob : NULL,
1089 i, status, &erased);
1090
1091 /* ECC errors are handled in the second loop. */
1092 if (ret < 0)
1093 continue;
1094
1095 if (oob_required && !erased) {
1096 /* TODO: use DMA to retrieve OOB */
1097 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1098 mtd->writesize + oob_off, -1);
1099 nand->read_buf(mtd, oob, ecc->bytes + 4);
1100
1101 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1102 !i, page);
1103 }
1104
1105 if (erased)
1106 raw_mode = 1;
1107
1108 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1109 }
1110
1111 if (status & NFC_ECC_ERR_MSK) {
1112 for (i = 0; i < nchunks; i++) {
1113 int data_off = i * ecc->size;
1114 int oob_off = i * (ecc->bytes + 4);
1115 u8 *data = buf + data_off;
1116 u8 *oob = nand->oob_poi + oob_off;
1117
1118 if (!(status & NFC_ECC_ERR(i)))
1119 continue;
1120
1121 /*
1122 * Re-read the data with the randomizer disabled to
1123 * identify bitflips in erased pages.
1124 */
1125 if (randomized) {
1126 /* TODO: use DMA to read page in raw mode */
1127 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1128 data_off, -1);
1129 nand->read_buf(mtd, data, ecc->size);
1130 }
1131
1132 /* TODO: use DMA to retrieve OOB */
1133 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1134 mtd->writesize + oob_off, -1);
1135 nand->read_buf(mtd, oob, ecc->bytes + 4);
1136
1137 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1138 oob, ecc->bytes + 4,
1139 NULL, 0,
1140 ecc->strength);
1141 if (ret >= 0)
1142 raw_mode = 1;
1143
1144 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1145 }
1146 }
1147
1148 if (oob_required)
1149 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1150 NULL, !raw_mode,
1151 page);
1152
1153 return max_bitflips;
1154 }
1155
1156 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1157 const u8 *data, int data_off,
1158 const u8 *oob, int oob_off,
1159 int *cur_off, bool bbm,
1160 int page)
1161 {
1162 struct nand_chip *nand = mtd_to_nand(mtd);
1163 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1164 struct nand_ecc_ctrl *ecc = &nand->ecc;
1165 int ret;
1166
1167 if (data_off != *cur_off)
1168 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1169
1170 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1171
1172 if (data_off + ecc->size != oob_off)
1173 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1174
1175 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1176 if (ret)
1177 return ret;
1178
1179 sunxi_nfc_randomizer_enable(mtd);
1180 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1181
1182 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1183 NFC_ACCESS_DIR | NFC_ECC_OP,
1184 nfc->regs + NFC_REG_CMD);
1185
1186 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1187 sunxi_nfc_randomizer_disable(mtd);
1188 if (ret)
1189 return ret;
1190
1191 *cur_off = oob_off + ecc->bytes + 4;
1192
1193 return 0;
1194 }
1195
1196 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1197 u8 *oob, int *cur_off,
1198 int page)
1199 {
1200 struct nand_chip *nand = mtd_to_nand(mtd);
1201 struct nand_ecc_ctrl *ecc = &nand->ecc;
1202 int offset = ((ecc->bytes + 4) * ecc->steps);
1203 int len = mtd->oobsize - offset;
1204
1205 if (len <= 0)
1206 return;
1207
1208 if (!cur_off || *cur_off != offset)
1209 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1210 offset + mtd->writesize, -1);
1211
1212 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1213
1214 if (cur_off)
1215 *cur_off = mtd->oobsize + mtd->writesize;
1216 }
1217
1218 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1219 struct nand_chip *chip, uint8_t *buf,
1220 int oob_required, int page)
1221 {
1222 struct nand_ecc_ctrl *ecc = &chip->ecc;
1223 unsigned int max_bitflips = 0;
1224 int ret, i, cur_off = 0;
1225 bool raw_mode = false;
1226
1227 sunxi_nfc_hw_ecc_enable(mtd);
1228
1229 for (i = 0; i < ecc->steps; i++) {
1230 int data_off = i * ecc->size;
1231 int oob_off = i * (ecc->bytes + 4);
1232 u8 *data = buf + data_off;
1233 u8 *oob = chip->oob_poi + oob_off;
1234
1235 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1236 oob_off + mtd->writesize,
1237 &cur_off, &max_bitflips,
1238 !i, oob_required, page);
1239 if (ret < 0)
1240 return ret;
1241 else if (ret)
1242 raw_mode = true;
1243 }
1244
1245 if (oob_required)
1246 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1247 !raw_mode, page);
1248
1249 sunxi_nfc_hw_ecc_disable(mtd);
1250
1251 return max_bitflips;
1252 }
1253
1254 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1255 struct nand_chip *chip, u8 *buf,
1256 int oob_required, int page)
1257 {
1258 int ret;
1259
1260 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1261 chip->ecc.steps);
1262 if (ret >= 0)
1263 return ret;
1264
1265 /* Fallback to PIO mode */
1266 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1267
1268 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1269 }
1270
1271 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1272 struct nand_chip *chip,
1273 u32 data_offs, u32 readlen,
1274 u8 *bufpoi, int page)
1275 {
1276 struct nand_ecc_ctrl *ecc = &chip->ecc;
1277 int ret, i, cur_off = 0;
1278 unsigned int max_bitflips = 0;
1279
1280 sunxi_nfc_hw_ecc_enable(mtd);
1281
1282 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1283 for (i = data_offs / ecc->size;
1284 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1285 int data_off = i * ecc->size;
1286 int oob_off = i * (ecc->bytes + 4);
1287 u8 *data = bufpoi + data_off;
1288 u8 *oob = chip->oob_poi + oob_off;
1289
1290 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1291 oob,
1292 oob_off + mtd->writesize,
1293 &cur_off, &max_bitflips, !i,
1294 false, page);
1295 if (ret < 0)
1296 return ret;
1297 }
1298
1299 sunxi_nfc_hw_ecc_disable(mtd);
1300
1301 return max_bitflips;
1302 }
1303
1304 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1305 struct nand_chip *chip,
1306 u32 data_offs, u32 readlen,
1307 u8 *buf, int page)
1308 {
1309 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1310 int ret;
1311
1312 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1313 if (ret >= 0)
1314 return ret;
1315
1316 /* Fallback to PIO mode */
1317 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1318
1319 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1320 buf, page);
1321 }
1322
1323 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1324 struct nand_chip *chip,
1325 const uint8_t *buf, int oob_required,
1326 int page)
1327 {
1328 struct nand_ecc_ctrl *ecc = &chip->ecc;
1329 int ret, i, cur_off = 0;
1330
1331 sunxi_nfc_hw_ecc_enable(mtd);
1332
1333 for (i = 0; i < ecc->steps; i++) {
1334 int data_off = i * ecc->size;
1335 int oob_off = i * (ecc->bytes + 4);
1336 const u8 *data = buf + data_off;
1337 const u8 *oob = chip->oob_poi + oob_off;
1338
1339 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1340 oob_off + mtd->writesize,
1341 &cur_off, !i, page);
1342 if (ret)
1343 return ret;
1344 }
1345
1346 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1347 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1348 &cur_off, page);
1349
1350 sunxi_nfc_hw_ecc_disable(mtd);
1351
1352 return 0;
1353 }
1354
1355 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1356 struct nand_chip *chip,
1357 const u8 *buf,
1358 int oob_required,
1359 int page)
1360 {
1361 struct nand_chip *nand = mtd_to_nand(mtd);
1362 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1363 struct nand_ecc_ctrl *ecc = &nand->ecc;
1364 struct scatterlist sg;
1365 int ret, i;
1366
1367 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1368 if (ret)
1369 return ret;
1370
1371 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1372 DMA_TO_DEVICE, &sg);
1373 if (ret)
1374 goto pio_fallback;
1375
1376 for (i = 0; i < ecc->steps; i++) {
1377 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1378
1379 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1380 }
1381
1382 sunxi_nfc_hw_ecc_enable(mtd);
1383 sunxi_nfc_randomizer_config(mtd, page, false);
1384 sunxi_nfc_randomizer_enable(mtd);
1385
1386 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1387 nfc->regs + NFC_REG_RCMD_SET);
1388
1389 dma_async_issue_pending(nfc->dmac);
1390
1391 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1392 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1393 nfc->regs + NFC_REG_CMD);
1394
1395 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1396 if (ret)
1397 dmaengine_terminate_all(nfc->dmac);
1398
1399 sunxi_nfc_randomizer_disable(mtd);
1400 sunxi_nfc_hw_ecc_disable(mtd);
1401
1402 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1403
1404 if (ret)
1405 return ret;
1406
1407 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1408 /* TODO: use DMA to transfer extra OOB bytes ? */
1409 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1410 NULL, page);
1411
1412 return 0;
1413
1414 pio_fallback:
1415 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1416 }
1417
1418 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1419 struct nand_chip *chip,
1420 uint8_t *buf, int oob_required,
1421 int page)
1422 {
1423 struct nand_ecc_ctrl *ecc = &chip->ecc;
1424 unsigned int max_bitflips = 0;
1425 int ret, i, cur_off = 0;
1426 bool raw_mode = false;
1427
1428 sunxi_nfc_hw_ecc_enable(mtd);
1429
1430 for (i = 0; i < ecc->steps; i++) {
1431 int data_off = i * (ecc->size + ecc->bytes + 4);
1432 int oob_off = data_off + ecc->size;
1433 u8 *data = buf + (i * ecc->size);
1434 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1435
1436 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1437 oob_off, &cur_off,
1438 &max_bitflips, !i,
1439 oob_required,
1440 page);
1441 if (ret < 0)
1442 return ret;
1443 else if (ret)
1444 raw_mode = true;
1445 }
1446
1447 if (oob_required)
1448 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1449 !raw_mode, page);
1450
1451 sunxi_nfc_hw_ecc_disable(mtd);
1452
1453 return max_bitflips;
1454 }
1455
1456 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1457 struct nand_chip *chip,
1458 const uint8_t *buf,
1459 int oob_required, int page)
1460 {
1461 struct nand_ecc_ctrl *ecc = &chip->ecc;
1462 int ret, i, cur_off = 0;
1463
1464 sunxi_nfc_hw_ecc_enable(mtd);
1465
1466 for (i = 0; i < ecc->steps; i++) {
1467 int data_off = i * (ecc->size + ecc->bytes + 4);
1468 int oob_off = data_off + ecc->size;
1469 const u8 *data = buf + (i * ecc->size);
1470 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1471
1472 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1473 oob, oob_off, &cur_off,
1474 false, page);
1475 if (ret)
1476 return ret;
1477 }
1478
1479 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1480 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1481 &cur_off, page);
1482
1483 sunxi_nfc_hw_ecc_disable(mtd);
1484
1485 return 0;
1486 }
1487
1488 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1489 struct nand_chip *chip,
1490 int page)
1491 {
1492 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1493
1494 chip->pagebuf = -1;
1495
1496 return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1497 }
1498
1499 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1500 struct nand_chip *chip,
1501 int page)
1502 {
1503 int ret, status;
1504
1505 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1506
1507 chip->pagebuf = -1;
1508
1509 memset(chip->buffers->databuf, 0xff, mtd->writesize);
1510 ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1511 if (ret)
1512 return ret;
1513
1514 /* Send command to program the OOB data */
1515 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1516
1517 status = chip->waitfunc(mtd, chip);
1518
1519 return status & NAND_STATUS_FAIL ? -EIO : 0;
1520 }
1521
1522 static const s32 tWB_lut[] = {6, 12, 16, 20};
1523 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1524
1525 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1526 u32 clk_period)
1527 {
1528 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1529 int i;
1530
1531 for (i = 0; i < lut_size; i++) {
1532 if (clk_cycles <= lut[i])
1533 return i;
1534 }
1535
1536 /* Doesn't fit */
1537 return -EINVAL;
1538 }
1539
1540 #define sunxi_nand_lookup_timing(l, p, c) \
1541 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1542
1543 static int sunxi_nand_chip_set_timings(struct sunxi_nand_chip *chip,
1544 const struct nand_sdr_timings *timings)
1545 {
1546 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1547 u32 min_clk_period = 0;
1548 s32 tWB, tADL, tWHR, tRHW, tCAD;
1549 long real_clk_rate;
1550
1551 /* T1 <=> tCLS */
1552 if (timings->tCLS_min > min_clk_period)
1553 min_clk_period = timings->tCLS_min;
1554
1555 /* T2 <=> tCLH */
1556 if (timings->tCLH_min > min_clk_period)
1557 min_clk_period = timings->tCLH_min;
1558
1559 /* T3 <=> tCS */
1560 if (timings->tCS_min > min_clk_period)
1561 min_clk_period = timings->tCS_min;
1562
1563 /* T4 <=> tCH */
1564 if (timings->tCH_min > min_clk_period)
1565 min_clk_period = timings->tCH_min;
1566
1567 /* T5 <=> tWP */
1568 if (timings->tWP_min > min_clk_period)
1569 min_clk_period = timings->tWP_min;
1570
1571 /* T6 <=> tWH */
1572 if (timings->tWH_min > min_clk_period)
1573 min_clk_period = timings->tWH_min;
1574
1575 /* T7 <=> tALS */
1576 if (timings->tALS_min > min_clk_period)
1577 min_clk_period = timings->tALS_min;
1578
1579 /* T8 <=> tDS */
1580 if (timings->tDS_min > min_clk_period)
1581 min_clk_period = timings->tDS_min;
1582
1583 /* T9 <=> tDH */
1584 if (timings->tDH_min > min_clk_period)
1585 min_clk_period = timings->tDH_min;
1586
1587 /* T10 <=> tRR */
1588 if (timings->tRR_min > (min_clk_period * 3))
1589 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1590
1591 /* T11 <=> tALH */
1592 if (timings->tALH_min > min_clk_period)
1593 min_clk_period = timings->tALH_min;
1594
1595 /* T12 <=> tRP */
1596 if (timings->tRP_min > min_clk_period)
1597 min_clk_period = timings->tRP_min;
1598
1599 /* T13 <=> tREH */
1600 if (timings->tREH_min > min_clk_period)
1601 min_clk_period = timings->tREH_min;
1602
1603 /* T14 <=> tRC */
1604 if (timings->tRC_min > (min_clk_period * 2))
1605 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1606
1607 /* T15 <=> tWC */
1608 if (timings->tWC_min > (min_clk_period * 2))
1609 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1610
1611 /* T16 - T19 + tCAD */
1612 if (timings->tWB_max > (min_clk_period * 20))
1613 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1614
1615 if (timings->tADL_min > (min_clk_period * 32))
1616 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1617
1618 if (timings->tWHR_min > (min_clk_period * 32))
1619 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1620
1621 if (timings->tRHW_min > (min_clk_period * 20))
1622 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1623
1624 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1625 min_clk_period);
1626 if (tWB < 0) {
1627 dev_err(nfc->dev, "unsupported tWB\n");
1628 return tWB;
1629 }
1630
1631 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1632 if (tADL > 3) {
1633 dev_err(nfc->dev, "unsupported tADL\n");
1634 return -EINVAL;
1635 }
1636
1637 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1638 if (tWHR > 3) {
1639 dev_err(nfc->dev, "unsupported tWHR\n");
1640 return -EINVAL;
1641 }
1642
1643 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1644 min_clk_period);
1645 if (tRHW < 0) {
1646 dev_err(nfc->dev, "unsupported tRHW\n");
1647 return tRHW;
1648 }
1649
1650 /*
1651 * TODO: according to ONFI specs this value only applies for DDR NAND,
1652 * but Allwinner seems to set this to 0x7. Mimic them for now.
1653 */
1654 tCAD = 0x7;
1655
1656 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1657 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1658
1659 /* Convert min_clk_period from picoseconds to nanoseconds */
1660 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1661
1662 /*
1663 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1664 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1665 * This new formula was verified with a scope and validated by
1666 * Allwinner engineers.
1667 */
1668 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1669 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1670
1671 /*
1672 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1673 * output cycle timings shall be used if the host drives tRC less than
1674 * 30 ns.
1675 */
1676 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1677 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1678 NFC_TIMING_CTL_EDO : 0;
1679
1680 return 0;
1681 }
1682
1683 static int sunxi_nand_chip_init_timings(struct sunxi_nand_chip *chip,
1684 struct device_node *np)
1685 {
1686 struct mtd_info *mtd = nand_to_mtd(&chip->nand);
1687 const struct nand_sdr_timings *timings;
1688 int ret;
1689 int mode;
1690
1691 mode = onfi_get_async_timing_mode(&chip->nand);
1692 if (mode == ONFI_TIMING_MODE_UNKNOWN) {
1693 mode = chip->nand.onfi_timing_mode_default;
1694 } else {
1695 uint8_t feature[ONFI_SUBFEATURE_PARAM_LEN] = {};
1696 int i;
1697
1698 mode = fls(mode) - 1;
1699 if (mode < 0)
1700 mode = 0;
1701
1702 feature[0] = mode;
1703 for (i = 0; i < chip->nsels; i++) {
1704 chip->nand.select_chip(mtd, i);
1705 ret = chip->nand.onfi_set_features(mtd, &chip->nand,
1706 ONFI_FEATURE_ADDR_TIMING_MODE,
1707 feature);
1708 chip->nand.select_chip(mtd, -1);
1709 if (ret)
1710 return ret;
1711 }
1712 }
1713
1714 timings = onfi_async_timing_mode_to_sdr_timings(mode);
1715 if (IS_ERR(timings))
1716 return PTR_ERR(timings);
1717
1718 return sunxi_nand_chip_set_timings(chip, timings);
1719 }
1720
1721 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1722 struct mtd_oob_region *oobregion)
1723 {
1724 struct nand_chip *nand = mtd_to_nand(mtd);
1725 struct nand_ecc_ctrl *ecc = &nand->ecc;
1726
1727 if (section >= ecc->steps)
1728 return -ERANGE;
1729
1730 oobregion->offset = section * (ecc->bytes + 4) + 4;
1731 oobregion->length = ecc->bytes;
1732
1733 return 0;
1734 }
1735
1736 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1737 struct mtd_oob_region *oobregion)
1738 {
1739 struct nand_chip *nand = mtd_to_nand(mtd);
1740 struct nand_ecc_ctrl *ecc = &nand->ecc;
1741
1742 if (section > ecc->steps)
1743 return -ERANGE;
1744
1745 /*
1746 * The first 2 bytes are used for BB markers, hence we
1747 * only have 2 bytes available in the first user data
1748 * section.
1749 */
1750 if (!section && ecc->mode == NAND_ECC_HW) {
1751 oobregion->offset = 2;
1752 oobregion->length = 2;
1753
1754 return 0;
1755 }
1756
1757 oobregion->offset = section * (ecc->bytes + 4);
1758
1759 if (section < ecc->steps)
1760 oobregion->length = 4;
1761 else
1762 oobregion->offset = mtd->oobsize - oobregion->offset;
1763
1764 return 0;
1765 }
1766
1767 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1768 .ecc = sunxi_nand_ooblayout_ecc,
1769 .free = sunxi_nand_ooblayout_free,
1770 };
1771
1772 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1773 struct nand_ecc_ctrl *ecc,
1774 struct device_node *np)
1775 {
1776 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1777 struct nand_chip *nand = mtd_to_nand(mtd);
1778 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1779 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1780 struct sunxi_nand_hw_ecc *data;
1781 int nsectors;
1782 int ret;
1783 int i;
1784
1785 data = kzalloc(sizeof(*data), GFP_KERNEL);
1786 if (!data)
1787 return -ENOMEM;
1788
1789 if (ecc->size != 512 && ecc->size != 1024)
1790 return -EINVAL;
1791
1792 /* Prefer 1k ECC chunk over 512 ones */
1793 if (ecc->size == 512 && mtd->writesize > 512) {
1794 ecc->size = 1024;
1795 ecc->strength *= 2;
1796 }
1797
1798 /* Add ECC info retrieval from DT */
1799 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1800 if (ecc->strength <= strengths[i])
1801 break;
1802 }
1803
1804 if (i >= ARRAY_SIZE(strengths)) {
1805 dev_err(nfc->dev, "unsupported strength\n");
1806 ret = -ENOTSUPP;
1807 goto err;
1808 }
1809
1810 data->mode = i;
1811
1812 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1813 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1814
1815 /* HW ECC always work with even numbers of ECC bytes */
1816 ecc->bytes = ALIGN(ecc->bytes, 2);
1817
1818 nsectors = mtd->writesize / ecc->size;
1819
1820 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1821 ret = -EINVAL;
1822 goto err;
1823 }
1824
1825 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1826 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1827 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1828 ecc->priv = data;
1829
1830 return 0;
1831
1832 err:
1833 kfree(data);
1834
1835 return ret;
1836 }
1837
1838 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1839 {
1840 kfree(ecc->priv);
1841 }
1842
1843 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1844 struct nand_ecc_ctrl *ecc,
1845 struct device_node *np)
1846 {
1847 struct nand_chip *nand = mtd_to_nand(mtd);
1848 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1849 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1850 int ret;
1851
1852 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1853 if (ret)
1854 return ret;
1855
1856 if (nfc->dmac) {
1857 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1858 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1859 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1860 nand->options |= NAND_USE_BOUNCE_BUFFER;
1861 } else {
1862 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1863 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1864 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1865 }
1866
1867 /* TODO: support DMA for raw accesses */
1868 ecc->read_oob_raw = nand_read_oob_std;
1869 ecc->write_oob_raw = nand_write_oob_std;
1870 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1871
1872 return 0;
1873 }
1874
1875 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1876 struct nand_ecc_ctrl *ecc,
1877 struct device_node *np)
1878 {
1879 int ret;
1880
1881 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1882 if (ret)
1883 return ret;
1884
1885 ecc->prepad = 4;
1886 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1887 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1888 ecc->read_oob_raw = nand_read_oob_syndrome;
1889 ecc->write_oob_raw = nand_write_oob_syndrome;
1890
1891 return 0;
1892 }
1893
1894 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1895 {
1896 switch (ecc->mode) {
1897 case NAND_ECC_HW:
1898 case NAND_ECC_HW_SYNDROME:
1899 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1900 break;
1901 case NAND_ECC_NONE:
1902 default:
1903 break;
1904 }
1905 }
1906
1907 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1908 struct device_node *np)
1909 {
1910 struct nand_chip *nand = mtd_to_nand(mtd);
1911 int ret;
1912
1913 if (!ecc->size) {
1914 ecc->size = nand->ecc_step_ds;
1915 ecc->strength = nand->ecc_strength_ds;
1916 }
1917
1918 if (!ecc->size || !ecc->strength)
1919 return -EINVAL;
1920
1921 switch (ecc->mode) {
1922 case NAND_ECC_HW:
1923 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1924 if (ret)
1925 return ret;
1926 break;
1927 case NAND_ECC_HW_SYNDROME:
1928 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1929 if (ret)
1930 return ret;
1931 break;
1932 case NAND_ECC_NONE:
1933 case NAND_ECC_SOFT:
1934 break;
1935 default:
1936 return -EINVAL;
1937 }
1938
1939 return 0;
1940 }
1941
1942 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1943 struct device_node *np)
1944 {
1945 const struct nand_sdr_timings *timings;
1946 struct sunxi_nand_chip *chip;
1947 struct mtd_info *mtd;
1948 struct nand_chip *nand;
1949 int nsels;
1950 int ret;
1951 int i;
1952 u32 tmp;
1953
1954 if (!of_get_property(np, "reg", &nsels))
1955 return -EINVAL;
1956
1957 nsels /= sizeof(u32);
1958 if (!nsels) {
1959 dev_err(dev, "invalid reg property size\n");
1960 return -EINVAL;
1961 }
1962
1963 chip = devm_kzalloc(dev,
1964 sizeof(*chip) +
1965 (nsels * sizeof(struct sunxi_nand_chip_sel)),
1966 GFP_KERNEL);
1967 if (!chip) {
1968 dev_err(dev, "could not allocate chip\n");
1969 return -ENOMEM;
1970 }
1971
1972 chip->nsels = nsels;
1973 chip->selected = -1;
1974
1975 for (i = 0; i < nsels; i++) {
1976 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1977 if (ret) {
1978 dev_err(dev, "could not retrieve reg property: %d\n",
1979 ret);
1980 return ret;
1981 }
1982
1983 if (tmp > NFC_MAX_CS) {
1984 dev_err(dev,
1985 "invalid reg value: %u (max CS = 7)\n",
1986 tmp);
1987 return -EINVAL;
1988 }
1989
1990 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1991 dev_err(dev, "CS %d already assigned\n", tmp);
1992 return -EINVAL;
1993 }
1994
1995 chip->sels[i].cs = tmp;
1996
1997 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1998 tmp < 2) {
1999 chip->sels[i].rb.type = RB_NATIVE;
2000 chip->sels[i].rb.info.nativeid = tmp;
2001 } else {
2002 ret = of_get_named_gpio(np, "rb-gpios", i);
2003 if (ret >= 0) {
2004 tmp = ret;
2005 chip->sels[i].rb.type = RB_GPIO;
2006 chip->sels[i].rb.info.gpio = tmp;
2007 ret = devm_gpio_request(dev, tmp, "nand-rb");
2008 if (ret)
2009 return ret;
2010
2011 ret = gpio_direction_input(tmp);
2012 if (ret)
2013 return ret;
2014 } else {
2015 chip->sels[i].rb.type = RB_NONE;
2016 }
2017 }
2018 }
2019
2020 nand = &chip->nand;
2021 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2022 nand->chip_delay = 200;
2023 nand->controller = &nfc->controller;
2024 /*
2025 * Set the ECC mode to the default value in case nothing is specified
2026 * in the DT.
2027 */
2028 nand->ecc.mode = NAND_ECC_HW;
2029 nand_set_flash_node(nand, np);
2030 nand->select_chip = sunxi_nfc_select_chip;
2031 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2032 nand->read_buf = sunxi_nfc_read_buf;
2033 nand->write_buf = sunxi_nfc_write_buf;
2034 nand->read_byte = sunxi_nfc_read_byte;
2035
2036 mtd = nand_to_mtd(nand);
2037 mtd->dev.parent = dev;
2038
2039 timings = onfi_async_timing_mode_to_sdr_timings(0);
2040 if (IS_ERR(timings)) {
2041 ret = PTR_ERR(timings);
2042 dev_err(dev,
2043 "could not retrieve timings for ONFI mode 0: %d\n",
2044 ret);
2045 return ret;
2046 }
2047
2048 ret = sunxi_nand_chip_set_timings(chip, timings);
2049 if (ret) {
2050 dev_err(dev, "could not configure chip timings: %d\n", ret);
2051 return ret;
2052 }
2053
2054 ret = nand_scan_ident(mtd, nsels, NULL);
2055 if (ret)
2056 return ret;
2057
2058 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2059 nand->bbt_options |= NAND_BBT_NO_OOB;
2060
2061 if (nand->options & NAND_NEED_SCRAMBLING)
2062 nand->options |= NAND_NO_SUBPAGE_WRITE;
2063
2064 nand->options |= NAND_SUBPAGE_READ;
2065
2066 ret = sunxi_nand_chip_init_timings(chip, np);
2067 if (ret) {
2068 dev_err(dev, "could not configure chip timings: %d\n", ret);
2069 return ret;
2070 }
2071
2072 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2073 if (ret) {
2074 dev_err(dev, "ECC init failed: %d\n", ret);
2075 return ret;
2076 }
2077
2078 ret = nand_scan_tail(mtd);
2079 if (ret) {
2080 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2081 return ret;
2082 }
2083
2084 ret = mtd_device_register(mtd, NULL, 0);
2085 if (ret) {
2086 dev_err(dev, "failed to register mtd device: %d\n", ret);
2087 nand_release(mtd);
2088 return ret;
2089 }
2090
2091 list_add_tail(&chip->node, &nfc->chips);
2092
2093 return 0;
2094 }
2095
2096 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2097 {
2098 struct device_node *np = dev->of_node;
2099 struct device_node *nand_np;
2100 int nchips = of_get_child_count(np);
2101 int ret;
2102
2103 if (nchips > 8) {
2104 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2105 return -EINVAL;
2106 }
2107
2108 for_each_child_of_node(np, nand_np) {
2109 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2110 if (ret) {
2111 of_node_put(nand_np);
2112 return ret;
2113 }
2114 }
2115
2116 return 0;
2117 }
2118
2119 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2120 {
2121 struct sunxi_nand_chip *chip;
2122
2123 while (!list_empty(&nfc->chips)) {
2124 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2125 node);
2126 nand_release(nand_to_mtd(&chip->nand));
2127 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2128 list_del(&chip->node);
2129 }
2130 }
2131
2132 static int sunxi_nfc_probe(struct platform_device *pdev)
2133 {
2134 struct device *dev = &pdev->dev;
2135 struct resource *r;
2136 struct sunxi_nfc *nfc;
2137 int irq;
2138 int ret;
2139
2140 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2141 if (!nfc)
2142 return -ENOMEM;
2143
2144 nfc->dev = dev;
2145 spin_lock_init(&nfc->controller.lock);
2146 init_waitqueue_head(&nfc->controller.wq);
2147 INIT_LIST_HEAD(&nfc->chips);
2148
2149 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2150 nfc->regs = devm_ioremap_resource(dev, r);
2151 if (IS_ERR(nfc->regs))
2152 return PTR_ERR(nfc->regs);
2153
2154 irq = platform_get_irq(pdev, 0);
2155 if (irq < 0) {
2156 dev_err(dev, "failed to retrieve irq\n");
2157 return irq;
2158 }
2159
2160 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2161 if (IS_ERR(nfc->ahb_clk)) {
2162 dev_err(dev, "failed to retrieve ahb clk\n");
2163 return PTR_ERR(nfc->ahb_clk);
2164 }
2165
2166 ret = clk_prepare_enable(nfc->ahb_clk);
2167 if (ret)
2168 return ret;
2169
2170 nfc->mod_clk = devm_clk_get(dev, "mod");
2171 if (IS_ERR(nfc->mod_clk)) {
2172 dev_err(dev, "failed to retrieve mod clk\n");
2173 ret = PTR_ERR(nfc->mod_clk);
2174 goto out_ahb_clk_unprepare;
2175 }
2176
2177 ret = clk_prepare_enable(nfc->mod_clk);
2178 if (ret)
2179 goto out_ahb_clk_unprepare;
2180
2181 ret = sunxi_nfc_rst(nfc);
2182 if (ret)
2183 goto out_mod_clk_unprepare;
2184
2185 writel(0, nfc->regs + NFC_REG_INT);
2186 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2187 0, "sunxi-nand", nfc);
2188 if (ret)
2189 goto out_mod_clk_unprepare;
2190
2191 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2192 if (nfc->dmac) {
2193 struct dma_slave_config dmac_cfg = { };
2194
2195 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2196 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2197 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2198 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2199 dmac_cfg.src_maxburst = 4;
2200 dmac_cfg.dst_maxburst = 4;
2201 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2202 } else {
2203 dev_warn(dev, "failed to request rxtx DMA channel\n");
2204 }
2205
2206 platform_set_drvdata(pdev, nfc);
2207
2208 ret = sunxi_nand_chips_init(dev, nfc);
2209 if (ret) {
2210 dev_err(dev, "failed to init nand chips\n");
2211 goto out_release_dmac;
2212 }
2213
2214 return 0;
2215
2216 out_release_dmac:
2217 if (nfc->dmac)
2218 dma_release_channel(nfc->dmac);
2219 out_mod_clk_unprepare:
2220 clk_disable_unprepare(nfc->mod_clk);
2221 out_ahb_clk_unprepare:
2222 clk_disable_unprepare(nfc->ahb_clk);
2223
2224 return ret;
2225 }
2226
2227 static int sunxi_nfc_remove(struct platform_device *pdev)
2228 {
2229 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2230
2231 sunxi_nand_chips_cleanup(nfc);
2232 if (nfc->dmac)
2233 dma_release_channel(nfc->dmac);
2234 clk_disable_unprepare(nfc->mod_clk);
2235 clk_disable_unprepare(nfc->ahb_clk);
2236
2237 return 0;
2238 }
2239
2240 static const struct of_device_id sunxi_nfc_ids[] = {
2241 { .compatible = "allwinner,sun4i-a10-nand" },
2242 { /* sentinel */ }
2243 };
2244 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2245
2246 static struct platform_driver sunxi_nfc_driver = {
2247 .driver = {
2248 .name = "sunxi_nand",
2249 .of_match_table = sunxi_nfc_ids,
2250 },
2251 .probe = sunxi_nfc_probe,
2252 .remove = sunxi_nfc_remove,
2253 };
2254 module_platform_driver(sunxi_nfc_driver);
2255
2256 MODULE_LICENSE("GPL v2");
2257 MODULE_AUTHOR("Boris BREZILLON");
2258 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2259 MODULE_ALIAS("platform:sunxi_nand");
Linux kernel - Deliverables
This page took 0.173041 seconds and 6 git commands to generate.