projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlxsw: cmd: Introduce FID-offset flooding tables
[deliverable/linux.git] / drivers / net / ethernet / mellanox / mlxsw / pci.c
CommitLineData
eda6500a
JP		 1/*
2 * drivers/net/ethernet/mellanox/mlxsw/pci.c
3 * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2015 Jiri Pirko <jiri@mellanox.com>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the names of the copyright holders nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * Alternatively, this software may be distributed under the terms of the
19 * GNU General Public License ("GPL") version 2 as published by the Free
20 * Software Foundation.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#include <linux/kernel.h>
36#include <linux/module.h>
37#include <linux/export.h>
38#include <linux/err.h>
39#include <linux/device.h>
40#include <linux/pci.h>
41#include <linux/interrupt.h>
42#include <linux/wait.h>
43#include <linux/types.h>
44#include <linux/skbuff.h>
45#include <linux/if_vlan.h>
46#include <linux/log2.h>
47#include <linux/debugfs.h>
48#include <linux/seq_file.h>
1e81779a 49#include <linux/string.h>
eda6500a
JP		 50
51#include "pci.h"
52#include "core.h"
53#include "cmd.h"
54#include "port.h"
55
56static const char mlxsw_pci_driver_name[] = "mlxsw_pci";
57
58static const struct pci_device_id mlxsw_pci_id_table[] = {
31557f0f 59 {PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SWITCHX2), 0},
eda6500a
JP		 60 {0, }
61};
62
63static struct dentry *mlxsw_pci_dbg_root;
64
65static const char *mlxsw_pci_device_kind_get(const struct pci_device_id *id)
66{
67 switch (id->device) {
31557f0f
JP		 68 case PCI_DEVICE_ID_MELLANOX_SWITCHX2:
69 return MLXSW_DEVICE_KIND_SWITCHX2;
eda6500a
JP		 70 default:
71 BUG();
72 }
73}
74
75#define mlxsw_pci_write32(mlxsw_pci, reg, val) \
76 iowrite32be(val, (mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
77#define mlxsw_pci_read32(mlxsw_pci, reg) \
78 ioread32be((mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
79
80enum mlxsw_pci_queue_type {
81 MLXSW_PCI_QUEUE_TYPE_SDQ,
82 MLXSW_PCI_QUEUE_TYPE_RDQ,
83 MLXSW_PCI_QUEUE_TYPE_CQ,
84 MLXSW_PCI_QUEUE_TYPE_EQ,
85};
86
87static const char *mlxsw_pci_queue_type_str(enum mlxsw_pci_queue_type q_type)
88{
89 switch (q_type) {
90 case MLXSW_PCI_QUEUE_TYPE_SDQ:
91 return "sdq";
92 case MLXSW_PCI_QUEUE_TYPE_RDQ:
93 return "rdq";
94 case MLXSW_PCI_QUEUE_TYPE_CQ:
95 return "cq";
96 case MLXSW_PCI_QUEUE_TYPE_EQ:
97 return "eq";
98 }
99 BUG();
100}
101
102#define MLXSW_PCI_QUEUE_TYPE_COUNT 4
103
104static const u16 mlxsw_pci_doorbell_type_offset[] = {
105 MLXSW_PCI_DOORBELL_SDQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_SDQ */
106 MLXSW_PCI_DOORBELL_RDQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_RDQ */
107 MLXSW_PCI_DOORBELL_CQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_CQ */
108 MLXSW_PCI_DOORBELL_EQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_EQ */
109};
110
111static const u16 mlxsw_pci_doorbell_arm_type_offset[] = {
112 0, /* unused */
113 0, /* unused */
114 MLXSW_PCI_DOORBELL_ARM_CQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_CQ */
115 MLXSW_PCI_DOORBELL_ARM_EQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_EQ */
116};
117
118struct mlxsw_pci_mem_item {
119 char *buf;
120 dma_addr_t mapaddr;
121 size_t size;
122};
123
124struct mlxsw_pci_queue_elem_info {
125 char *elem; /* pointer to actual dma mapped element mem chunk */
126 union {
127 struct {
128 struct sk_buff *skb;
129 } sdq;
130 struct {
131 struct sk_buff *skb;
132 } rdq;
133 } u;
134};
135
136struct mlxsw_pci_queue {
137 spinlock_t lock; /* for queue accesses */
138 struct mlxsw_pci_mem_item mem_item;
139 struct mlxsw_pci_queue_elem_info *elem_info;
140 u16 producer_counter;
141 u16 consumer_counter;
142 u16 count; /* number of elements in queue */
143 u8 num; /* queue number */
144 u8 elem_size; /* size of one element */
145 enum mlxsw_pci_queue_type type;
146 struct tasklet_struct tasklet; /* queue processing tasklet */
147 struct mlxsw_pci *pci;
148 union {
149 struct {
150 u32 comp_sdq_count;
151 u32 comp_rdq_count;
152 } cq;
153 struct {
154 u32 ev_cmd_count;
155 u32 ev_comp_count;
156 u32 ev_other_count;
157 } eq;
158 } u;
159};
160
161struct mlxsw_pci_queue_type_group {
162 struct mlxsw_pci_queue *q;
163 u8 count; /* number of queues in group */
164};
165
166struct mlxsw_pci {
167 struct pci_dev *pdev;
168 u8 __iomem *hw_addr;
169 struct mlxsw_pci_queue_type_group queues[MLXSW_PCI_QUEUE_TYPE_COUNT];
170 u32 doorbell_offset;
171 struct msix_entry msix_entry;
172 struct mlxsw_core *core;
173 struct {
eda6500a 174 struct mlxsw_pci_mem_item *items;
3e2206da 175 unsigned int count;
eda6500a
JP		 176 } fw_area;
177 struct {
1e81779a
IS		 178 struct mlxsw_pci_mem_item out_mbox;
179 struct mlxsw_pci_mem_item in_mbox;
eda6500a
JP		 180 struct mutex lock; /* Lock access to command registers */
181 bool nopoll;
182 wait_queue_head_t wait;
183 bool wait_done;
184 struct {
185 u8 status;
186 u64 out_param;
187 } comp;
188 } cmd;
189 struct mlxsw_bus_info bus_info;
190 struct dentry *dbg_dir;
191};
192
193static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
194{
195 tasklet_schedule(&q->tasklet);
196}
197
198static char *__mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q,
199 size_t elem_size, int elem_index)
200{
201 return q->mem_item.buf + (elem_size * elem_index);
202}
203
204static struct mlxsw_pci_queue_elem_info *
205mlxsw_pci_queue_elem_info_get(struct mlxsw_pci_queue *q, int elem_index)
206{
207 return &q->elem_info[elem_index];
208}
209
210static struct mlxsw_pci_queue_elem_info *
211mlxsw_pci_queue_elem_info_producer_get(struct mlxsw_pci_queue *q)
212{
213 int index = q->producer_counter & (q->count - 1);
214
215 if ((q->producer_counter - q->consumer_counter) == q->count)
216 return NULL;
217 return mlxsw_pci_queue_elem_info_get(q, index);
218}
219
220static struct mlxsw_pci_queue_elem_info *
221mlxsw_pci_queue_elem_info_consumer_get(struct mlxsw_pci_queue *q)
222{
223 int index = q->consumer_counter & (q->count - 1);
224
225 return mlxsw_pci_queue_elem_info_get(q, index);
226}
227
228static char *mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q, int elem_index)
229{
230 return mlxsw_pci_queue_elem_info_get(q, elem_index)->elem;
231}
232
233static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)
234{
235 return owner_bit != !!(q->consumer_counter & q->count);
236}
237
238static char *mlxsw_pci_queue_sw_elem_get(struct mlxsw_pci_queue *q,
239 u32 (*get_elem_owner_func)(char *))
240{
241 struct mlxsw_pci_queue_elem_info *elem_info;
242 char *elem;
243 bool owner_bit;
244
245 elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
246 elem = elem_info->elem;
247 owner_bit = get_elem_owner_func(elem);
248 if (mlxsw_pci_elem_hw_owned(q, owner_bit))
249 return NULL;
250 q->consumer_counter++;
251 rmb(); /* make sure we read owned bit before the rest of elem */
252 return elem;
253}
254
255static struct mlxsw_pci_queue_type_group *
256mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,
257 enum mlxsw_pci_queue_type q_type)
258{
259 return &mlxsw_pci->queues[q_type];
260}
261
262static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci,
263 enum mlxsw_pci_queue_type q_type)
264{
265 struct mlxsw_pci_queue_type_group *queue_group;
266
267 queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type);
268 return queue_group->count;
269}
270
271static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci)
272{
273 return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ);
274}
275
276static u8 mlxsw_pci_rdq_count(struct mlxsw_pci *mlxsw_pci)
277{
278 return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_RDQ);
279}
280
281static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci)
282{
283 return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ);
284}
285
286static u8 mlxsw_pci_eq_count(struct mlxsw_pci *mlxsw_pci)
287{
288 return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ);
289}
290
291static struct mlxsw_pci_queue *
292__mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci,
293 enum mlxsw_pci_queue_type q_type, u8 q_num)
294{
295 return &mlxsw_pci->queues[q_type].q[q_num];
296}
297
298static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci,
299 u8 q_num)
300{
301 return __mlxsw_pci_queue_get(mlxsw_pci,
302 MLXSW_PCI_QUEUE_TYPE_SDQ, q_num);
303}
304
305static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci,
306 u8 q_num)
307{
308 return __mlxsw_pci_queue_get(mlxsw_pci,
309 MLXSW_PCI_QUEUE_TYPE_RDQ, q_num);
310}
311
312static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci,
313 u8 q_num)
314{
315 return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num);
316}
317
318static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci,
319 u8 q_num)
320{
321 return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num);
322}
323
324static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci,
325 struct mlxsw_pci_queue *q,
326 u16 val)
327{
328 mlxsw_pci_write32(mlxsw_pci,
329 DOORBELL(mlxsw_pci->doorbell_offset,
330 mlxsw_pci_doorbell_type_offset[q->type],
331 q->num), val);
332}
333
334static void __mlxsw_pci_queue_doorbell_arm_set(struct mlxsw_pci *mlxsw_pci,
335 struct mlxsw_pci_queue *q,
336 u16 val)
337{
338 mlxsw_pci_write32(mlxsw_pci,
339 DOORBELL(mlxsw_pci->doorbell_offset,
340 mlxsw_pci_doorbell_arm_type_offset[q->type],
341 q->num), val);
342}
343
344static void mlxsw_pci_queue_doorbell_producer_ring(struct mlxsw_pci *mlxsw_pci,
345 struct mlxsw_pci_queue *q)
346{
347 wmb(); /* ensure all writes are done before we ring a bell */
348 __mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);
349}
350
351static void mlxsw_pci_queue_doorbell_consumer_ring(struct mlxsw_pci *mlxsw_pci,
352 struct mlxsw_pci_queue *q)
353{
354 wmb(); /* ensure all writes are done before we ring a bell */
355 __mlxsw_pci_queue_doorbell_set(mlxsw_pci, q,
356 q->consumer_counter + q->count);
357}
358
359static void
360mlxsw_pci_queue_doorbell_arm_consumer_ring(struct mlxsw_pci *mlxsw_pci,
361 struct mlxsw_pci_queue *q)
362{
363 wmb(); /* ensure all writes are done before we ring a bell */
364 __mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);
365}
366
367static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q,
368 int page_index)
369{
370 return q->mem_item.mapaddr + MLXSW_PCI_PAGE_SIZE * page_index;
371}
372
373static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
374 struct mlxsw_pci_queue *q)
375{
376 int i;
377 int err;
378
379 q->producer_counter = 0;
380 q->consumer_counter = 0;
381
382 /* Set CQ of same number of this SDQ. */
383 mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);
384 mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, 7);
385 mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
386 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
387 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
388
389 mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
390 }
391
392 err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);
393 if (err)
394 return err;
395 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
396 return 0;
397}
398
399static void mlxsw_pci_sdq_fini(struct mlxsw_pci *mlxsw_pci,
400 struct mlxsw_pci_queue *q)
401{
402 mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);
403}
404
405static int mlxsw_pci_sdq_dbg_read(struct seq_file *file, void *data)
406{
407 struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
408 struct mlxsw_pci_queue *q;
409 int i;
410 static const char hdr[] =
411 "NUM PROD_COUNT CONS_COUNT COUNT\n";
412
413 seq_printf(file, hdr);
414 for (i = 0; i < mlxsw_pci_sdq_count(mlxsw_pci); i++) {
415 q = mlxsw_pci_sdq_get(mlxsw_pci, i);
416 spin_lock_bh(&q->lock);
417 seq_printf(file, "%3d %10d %10d %5d\n",
418 i, q->producer_counter, q->consumer_counter,
419 q->count);
420 spin_unlock_bh(&q->lock);
421 }
422 return 0;
423}
424
425static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe,
426 int index, char *frag_data, size_t frag_len,
427 int direction)
428{
429 struct pci_dev *pdev = mlxsw_pci->pdev;
430 dma_addr_t mapaddr;
431
432 mapaddr = pci_map_single(pdev, frag_data, frag_len, direction);
433 if (unlikely(pci_dma_mapping_error(pdev, mapaddr))) {
6cf9dc8b 434 dev_err_ratelimited(&pdev->dev, "failed to dma map tx frag\n");
eda6500a
JP		 435 return -EIO;
436 }
437 mlxsw_pci_wqe_address_set(wqe, index, mapaddr);
438 mlxsw_pci_wqe_byte_count_set(wqe, index, frag_len);
439 return 0;
440}
441
442static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe,
443 int index, int direction)
444{
445 struct pci_dev *pdev = mlxsw_pci->pdev;
446 size_t frag_len = mlxsw_pci_wqe_byte_count_get(wqe, index);
447 dma_addr_t mapaddr = mlxsw_pci_wqe_address_get(wqe, index);
448
449 if (!frag_len)
450 return;
451 pci_unmap_single(pdev, mapaddr, frag_len, direction);
452}
453
454static int mlxsw_pci_rdq_skb_alloc(struct mlxsw_pci *mlxsw_pci,
455 struct mlxsw_pci_queue_elem_info *elem_info)
456{
457 size_t buf_len = MLXSW_PORT_MAX_MTU;
458 char *wqe = elem_info->elem;
459 struct sk_buff *skb;
460 int err;
461
462 elem_info->u.rdq.skb = NULL;
463 skb = netdev_alloc_skb_ip_align(NULL, buf_len);
464 if (!skb)
465 return -ENOMEM;
466
467 /* Assume that wqe was previously zeroed. */
468
469 err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
470 buf_len, DMA_FROM_DEVICE);
471 if (err)
472 goto err_frag_map;
473
474 elem_info->u.rdq.skb = skb;
475 return 0;
476
477err_frag_map:
478 dev_kfree_skb_any(skb);
479 return err;
480}
481
482static void mlxsw_pci_rdq_skb_free(struct mlxsw_pci *mlxsw_pci,
483 struct mlxsw_pci_queue_elem_info *elem_info)
484{
485 struct sk_buff *skb;
486 char *wqe;
487
488 skb = elem_info->u.rdq.skb;
489 wqe = elem_info->elem;
490
491 mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
492 dev_kfree_skb_any(skb);
493}
494
495static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
496 struct mlxsw_pci_queue *q)
497{
498 struct mlxsw_pci_queue_elem_info *elem_info;
424e1114 499 u8 sdq_count = mlxsw_pci_sdq_count(mlxsw_pci);
eda6500a
JP		 500 int i;
501 int err;
502
503 q->producer_counter = 0;
504 q->consumer_counter = 0;
505
506 /* Set CQ of same number of this RDQ with base
424e1114 507 * above SDQ count as the lower ones are assigned to SDQs.
eda6500a 508 */
424e1114 509 mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num);
eda6500a
JP		 510 mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
511 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
512 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
513
514 mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
515 }
516
517 err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);
518 if (err)
519 return err;
520
521 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
522
523 for (i = 0; i < q->count; i++) {
524 elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
525 BUG_ON(!elem_info);
526 err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
527 if (err)
528 goto rollback;
529 /* Everything is set up, ring doorbell to pass elem to HW */
530 q->producer_counter++;
531 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
532 }
533
534 return 0;
535
536rollback:
537 for (i--; i >= 0; i--) {
538 elem_info = mlxsw_pci_queue_elem_info_get(q, i);
539 mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
540 }
541 mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
542
543 return err;
544}
545
546static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,
547 struct mlxsw_pci_queue *q)
548{
549 struct mlxsw_pci_queue_elem_info *elem_info;
550 int i;
551
552 mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
553 for (i = 0; i < q->count; i++) {
554 elem_info = mlxsw_pci_queue_elem_info_get(q, i);
555 mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
556 }
557}
558
559static int mlxsw_pci_rdq_dbg_read(struct seq_file *file, void *data)
560{
561 struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
562 struct mlxsw_pci_queue *q;
563 int i;
564 static const char hdr[] =
565 "NUM PROD_COUNT CONS_COUNT COUNT\n";
566
567 seq_printf(file, hdr);
568 for (i = 0; i < mlxsw_pci_rdq_count(mlxsw_pci); i++) {
569 q = mlxsw_pci_rdq_get(mlxsw_pci, i);
570 spin_lock_bh(&q->lock);
571 seq_printf(file, "%3d %10d %10d %5d\n",
572 i, q->producer_counter, q->consumer_counter,
573 q->count);
574 spin_unlock_bh(&q->lock);
575 }
576 return 0;
577}
578
579static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
580 struct mlxsw_pci_queue *q)
581{
582 int i;
583 int err;
584
585 q->consumer_counter = 0;
586
587 for (i = 0; i < q->count; i++) {
588 char *elem = mlxsw_pci_queue_elem_get(q, i);
589
590 mlxsw_pci_cqe_owner_set(elem, 1);
591 }
592
593 mlxsw_cmd_mbox_sw2hw_cq_cv_set(mbox, 0); /* CQE ver 0 */
594 mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);
595 mlxsw_cmd_mbox_sw2hw_cq_oi_set(mbox, 0);
596 mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);
597 mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));
598 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
599 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
600
601 mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr);
602 }
603 err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);
604 if (err)
605 return err;
606 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
607 mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
608 return 0;
609}
610
611static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci,
612 struct mlxsw_pci_queue *q)
613{
614 mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);
615}
616
617static int mlxsw_pci_cq_dbg_read(struct seq_file *file, void *data)
618{
619 struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
620
621 struct mlxsw_pci_queue *q;
622 int i;
623 static const char hdr[] =
624 "NUM CONS_INDEX SDQ_COUNT RDQ_COUNT COUNT\n";
625
626 seq_printf(file, hdr);
627 for (i = 0; i < mlxsw_pci_cq_count(mlxsw_pci); i++) {
628 q = mlxsw_pci_cq_get(mlxsw_pci, i);
629 spin_lock_bh(&q->lock);
630 seq_printf(file, "%3d %10d %10d %10d %5d\n",
631 i, q->consumer_counter, q->u.cq.comp_sdq_count,
632 q->u.cq.comp_rdq_count, q->count);
633 spin_unlock_bh(&q->lock);
634 }
635 return 0;
636}
637
638static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,
639 struct mlxsw_pci_queue *q,
640 u16 consumer_counter_limit,
641 char *cqe)
642{
643 struct pci_dev *pdev = mlxsw_pci->pdev;
644 struct mlxsw_pci_queue_elem_info *elem_info;
645 char *wqe;
646 struct sk_buff *skb;
647 int i;
648
649 spin_lock(&q->lock);
650 elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
651 skb = elem_info->u.sdq.skb;
652 wqe = elem_info->elem;
653 for (i = 0; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
654 mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
655 dev_kfree_skb_any(skb);
656 elem_info->u.sdq.skb = NULL;
657
658 if (q->consumer_counter++ != consumer_counter_limit)
659 dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in SDQ\n");
660 spin_unlock(&q->lock);
661}
662
663static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
664 struct mlxsw_pci_queue *q,
665 u16 consumer_counter_limit,
666 char *cqe)
667{
668 struct pci_dev *pdev = mlxsw_pci->pdev;
669 struct mlxsw_pci_queue_elem_info *elem_info;
670 char *wqe;
671 struct sk_buff *skb;
672 struct mlxsw_rx_info rx_info;
7b7b9cff 673 u16 byte_count;
eda6500a
JP		 674 int err;
675
676 elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
677 skb = elem_info->u.sdq.skb;
678 if (!skb)
679 return;
680 wqe = elem_info->elem;
681 mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
682
683 if (q->consumer_counter++ != consumer_counter_limit)
684 dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in RDQ\n");
685
686 /* We do not support lag now */
687 if (mlxsw_pci_cqe_lag_get(cqe))
688 goto drop;
689
690 rx_info.sys_port = mlxsw_pci_cqe_system_port_get(cqe);
691 rx_info.trap_id = mlxsw_pci_cqe_trap_id_get(cqe);
692
7b7b9cff
JP		 693 byte_count = mlxsw_pci_cqe_byte_count_get(cqe);
694 if (mlxsw_pci_cqe_crc_get(cqe))
695 byte_count -= ETH_FCS_LEN;
696 skb_put(skb, byte_count);
eda6500a
JP		 697 mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);
698
699put_new_skb:
700 memset(wqe, 0, q->elem_size);
701 err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
6cf9dc8b
JP		 702 if (err)
703 dev_dbg_ratelimited(&pdev->dev, "Failed to alloc skb for RDQ\n");
eda6500a
JP		 704 /* Everything is set up, ring doorbell to pass elem to HW */
705 q->producer_counter++;
706 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
707 return;
708
709drop:
710 dev_kfree_skb_any(skb);
711 goto put_new_skb;
712}
713
714static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)
715{
716 return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_cqe_owner_get);
717}
718
719static void mlxsw_pci_cq_tasklet(unsigned long data)
720{
721 struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;
722 struct mlxsw_pci *mlxsw_pci = q->pci;
723 char *cqe;
724 int items = 0;
725 int credits = q->count >> 1;
726
727 while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) {
728 u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe);
729 u8 sendq = mlxsw_pci_cqe_sr_get(cqe);
730 u8 dqn = mlxsw_pci_cqe_dqn_get(cqe);
731
732 if (sendq) {
733 struct mlxsw_pci_queue *sdq;
734
735 sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn);
736 mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq,
737 wqe_counter, cqe);
738 q->u.cq.comp_sdq_count++;
739 } else {
740 struct mlxsw_pci_queue *rdq;
741
742 rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn);
743 mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq,
744 wqe_counter, cqe);
745 q->u.cq.comp_rdq_count++;
746 }
747 if (++items == credits)
748 break;
749 }
750 if (items) {
751 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
752 mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
753 }
754}
755
756static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
757 struct mlxsw_pci_queue *q)
758{
759 int i;
760 int err;
761
762 q->consumer_counter = 0;
763
764 for (i = 0; i < q->count; i++) {
765 char *elem = mlxsw_pci_queue_elem_get(q, i);
766
767 mlxsw_pci_eqe_owner_set(elem, 1);
768 }
769
770 mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */
771 mlxsw_cmd_mbox_sw2hw_eq_oi_set(mbox, 0);
772 mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */
773 mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));
774 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
775 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
776
777 mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr);
778 }
779 err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);
780 if (err)
781 return err;
782 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
783 mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
784 return 0;
785}
786
787static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci,
788 struct mlxsw_pci_queue *q)
789{
790 mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);
791}
792
793static int mlxsw_pci_eq_dbg_read(struct seq_file *file, void *data)
794{
795 struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
796 struct mlxsw_pci_queue *q;
797 int i;
798 static const char hdr[] =
799 "NUM CONS_COUNT EV_CMD EV_COMP EV_OTHER COUNT\n";
800
801 seq_printf(file, hdr);
802 for (i = 0; i < mlxsw_pci_eq_count(mlxsw_pci); i++) {
803 q = mlxsw_pci_eq_get(mlxsw_pci, i);
804 spin_lock_bh(&q->lock);
805 seq_printf(file, "%3d %10d %10d %10d %10d %5d\n",
806 i, q->consumer_counter, q->u.eq.ev_cmd_count,
807 q->u.eq.ev_comp_count, q->u.eq.ev_other_count,
808 q->count);
809 spin_unlock_bh(&q->lock);
810 }
811 return 0;
812}
813
814static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe)
815{
816 mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe);
817 mlxsw_pci->cmd.comp.out_param =
818 ((u64) mlxsw_pci_eqe_cmd_out_param_h_get(eqe)) << 32 |
819 mlxsw_pci_eqe_cmd_out_param_l_get(eqe);
820 mlxsw_pci->cmd.wait_done = true;
821 wake_up(&mlxsw_pci->cmd.wait);
822}
823
824static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)
825{
826 return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_eqe_owner_get);
827}
828
829static void mlxsw_pci_eq_tasklet(unsigned long data)
830{
831 struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;
832 struct mlxsw_pci *mlxsw_pci = q->pci;
e4c870b1
JP		 833 u8 cq_count = mlxsw_pci_cq_count(mlxsw_pci);
834 unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)];
eda6500a
JP		 835 char *eqe;
836 u8 cqn;
837 bool cq_handle = false;
838 int items = 0;
839 int credits = q->count >> 1;
840
841 memset(&active_cqns, 0, sizeof(active_cqns));
842
843 while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) {
844 u8 event_type = mlxsw_pci_eqe_event_type_get(eqe);
845
846 switch (event_type) {
847 case MLXSW_PCI_EQE_EVENT_TYPE_CMD:
848 mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe);
849 q->u.eq.ev_cmd_count++;
850 break;
851 case MLXSW_PCI_EQE_EVENT_TYPE_COMP:
852 cqn = mlxsw_pci_eqe_cqn_get(eqe);
853 set_bit(cqn, active_cqns);
854 cq_handle = true;
855 q->u.eq.ev_comp_count++;
856 break;
857 default:
858 q->u.eq.ev_other_count++;
859 }
860 if (++items == credits)
861 break;
862 }
863 if (items) {
864 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
865 mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
866 }
867
868 if (!cq_handle)
869 return;
e4c870b1 870 for_each_set_bit(cqn, active_cqns, cq_count) {
eda6500a
JP		 871 q = mlxsw_pci_cq_get(mlxsw_pci, cqn);
872 mlxsw_pci_queue_tasklet_schedule(q);
873 }
874}
875
876struct mlxsw_pci_queue_ops {
877 const char *name;
878 enum mlxsw_pci_queue_type type;
879 int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
880 struct mlxsw_pci_queue *q);
881 void (*fini)(struct mlxsw_pci *mlxsw_pci,
882 struct mlxsw_pci_queue *q);
883 void (*tasklet)(unsigned long data);
884 int (*dbg_read)(struct seq_file *s, void *data);
885 u16 elem_count;
886 u8 elem_size;
887};
888
889static const struct mlxsw_pci_queue_ops mlxsw_pci_sdq_ops = {
890 .type = MLXSW_PCI_QUEUE_TYPE_SDQ,
891 .init = mlxsw_pci_sdq_init,
892 .fini = mlxsw_pci_sdq_fini,
893 .dbg_read = mlxsw_pci_sdq_dbg_read,
894 .elem_count = MLXSW_PCI_WQE_COUNT,
895 .elem_size = MLXSW_PCI_WQE_SIZE,
896};
897
898static const struct mlxsw_pci_queue_ops mlxsw_pci_rdq_ops = {
899 .type = MLXSW_PCI_QUEUE_TYPE_RDQ,
900 .init = mlxsw_pci_rdq_init,
901 .fini = mlxsw_pci_rdq_fini,
902 .dbg_read = mlxsw_pci_rdq_dbg_read,
903 .elem_count = MLXSW_PCI_WQE_COUNT,
904 .elem_size = MLXSW_PCI_WQE_SIZE
905};
906
907static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = {
908 .type = MLXSW_PCI_QUEUE_TYPE_CQ,
909 .init = mlxsw_pci_cq_init,
910 .fini = mlxsw_pci_cq_fini,
911 .tasklet = mlxsw_pci_cq_tasklet,
912 .dbg_read = mlxsw_pci_cq_dbg_read,
913 .elem_count = MLXSW_PCI_CQE_COUNT,
914 .elem_size = MLXSW_PCI_CQE_SIZE
915};
916
917static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = {
918 .type = MLXSW_PCI_QUEUE_TYPE_EQ,
919 .init = mlxsw_pci_eq_init,
920 .fini = mlxsw_pci_eq_fini,
921 .tasklet = mlxsw_pci_eq_tasklet,
922 .dbg_read = mlxsw_pci_eq_dbg_read,
923 .elem_count = MLXSW_PCI_EQE_COUNT,
924 .elem_size = MLXSW_PCI_EQE_SIZE
925};
926
927static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
928 const struct mlxsw_pci_queue_ops *q_ops,
929 struct mlxsw_pci_queue *q, u8 q_num)
930{
931 struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
932 int i;
933 int err;
934
935 spin_lock_init(&q->lock);
936 q->num = q_num;
937 q->count = q_ops->elem_count;
938 q->elem_size = q_ops->elem_size;
939 q->type = q_ops->type;
940 q->pci = mlxsw_pci;
941
942 if (q_ops->tasklet)
943 tasklet_init(&q->tasklet, q_ops->tasklet, (unsigned long) q);
944
945 mem_item->size = MLXSW_PCI_AQ_SIZE;
946 mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,
947 mem_item->size,
948 &mem_item->mapaddr);
949 if (!mem_item->buf)
950 return -ENOMEM;
951 memset(mem_item->buf, 0, mem_item->size);
952
953 q->elem_info = kcalloc(q->count, sizeof(*q->elem_info), GFP_KERNEL);
954 if (!q->elem_info) {
955 err = -ENOMEM;
956 goto err_elem_info_alloc;
957 }
958
959 /* Initialize dma mapped elements info elem_info for
960 * future easy access.
961 */
962 for (i = 0; i < q->count; i++) {
963 struct mlxsw_pci_queue_elem_info *elem_info;
964
965 elem_info = mlxsw_pci_queue_elem_info_get(q, i);
966 elem_info->elem =
967 __mlxsw_pci_queue_elem_get(q, q_ops->elem_size, i);
968 }
969
970 mlxsw_cmd_mbox_zero(mbox);
971 err = q_ops->init(mlxsw_pci, mbox, q);
972 if (err)
973 goto err_q_ops_init;
974 return 0;
975
976err_q_ops_init:
977 kfree(q->elem_info);
978err_elem_info_alloc:
979 pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
980 mem_item->buf, mem_item->mapaddr);
981 return err;
982}
983
984static void mlxsw_pci_queue_fini(struct mlxsw_pci *mlxsw_pci,
985 const struct mlxsw_pci_queue_ops *q_ops,
986 struct mlxsw_pci_queue *q)
987{
988 struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
989
990 q_ops->fini(mlxsw_pci, q);
991 kfree(q->elem_info);
992 pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
993 mem_item->buf, mem_item->mapaddr);
994}
995
996static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
997 const struct mlxsw_pci_queue_ops *q_ops,
998 u8 num_qs)
999{
1000 struct pci_dev *pdev = mlxsw_pci->pdev;
1001 struct mlxsw_pci_queue_type_group *queue_group;
1002 char tmp[16];
1003 int i;
1004 int err;
1005
1006 queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1007 queue_group->q = kcalloc(num_qs, sizeof(*queue_group->q), GFP_KERNEL);
1008 if (!queue_group->q)
1009 return -ENOMEM;
1010
1011 for (i = 0; i < num_qs; i++) {
1012 err = mlxsw_pci_queue_init(mlxsw_pci, mbox, q_ops,
1013 &queue_group->q[i], i);
1014 if (err)
1015 goto err_queue_init;
1016 }
1017 queue_group->count = num_qs;
1018
1019 sprintf(tmp, "%s_stats", mlxsw_pci_queue_type_str(q_ops->type));
1020 debugfs_create_devm_seqfile(&pdev->dev, tmp, mlxsw_pci->dbg_dir,
1021 q_ops->dbg_read);
1022
1023 return 0;
1024
1025err_queue_init:
1026 for (i--; i >= 0; i--)
1027 mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1028 kfree(queue_group->q);
1029 return err;
1030}
1031
1032static void mlxsw_pci_queue_group_fini(struct mlxsw_pci *mlxsw_pci,
1033 const struct mlxsw_pci_queue_ops *q_ops)
1034{
1035 struct mlxsw_pci_queue_type_group *queue_group;
1036 int i;
1037
1038 queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1039 for (i = 0; i < queue_group->count; i++)
1040 mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1041 kfree(queue_group->q);
1042}
1043
1044static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
1045{
1046 struct pci_dev *pdev = mlxsw_pci->pdev;
1047 u8 num_sdqs;
1048 u8 sdq_log2sz;
1049 u8 num_rdqs;
1050 u8 rdq_log2sz;
1051 u8 num_cqs;
1052 u8 cq_log2sz;
1053 u8 num_eqs;
1054 u8 eq_log2sz;
1055 int err;
1056
1057 mlxsw_cmd_mbox_zero(mbox);
1058 err = mlxsw_cmd_query_aq_cap(mlxsw_pci->core, mbox);
1059 if (err)
1060 return err;
1061
1062 num_sdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_sdqs_get(mbox);
1063 sdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_sdq_sz_get(mbox);
1064 num_rdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_rdqs_get(mbox);
1065 rdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_rdq_sz_get(mbox);
1066 num_cqs = mlxsw_cmd_mbox_query_aq_cap_max_num_cqs_get(mbox);
1067 cq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cq_sz_get(mbox);
1068 num_eqs = mlxsw_cmd_mbox_query_aq_cap_max_num_eqs_get(mbox);
1069 eq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_eq_sz_get(mbox);
1070
c85c3882 1071 if (num_sdqs + num_rdqs > num_cqs ||
e4c870b1 1072 num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_COUNT) {
eda6500a
JP		 1073 dev_err(&pdev->dev, "Unsupported number of queues\n");
1074 return -EINVAL;
1075 }
1076
1077 if ((1 << sdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1078 (1 << rdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1079 (1 << cq_log2sz != MLXSW_PCI_CQE_COUNT) ||
1080 (1 << eq_log2sz != MLXSW_PCI_EQE_COUNT)) {
1081 dev_err(&pdev->dev, "Unsupported number of async queue descriptors\n");
1082 return -EINVAL;
1083 }
1084
1085 err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,
1086 num_eqs);
1087 if (err) {
1088 dev_err(&pdev->dev, "Failed to initialize event queues\n");
1089 return err;
1090 }
1091
1092 err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_cq_ops,
1093 num_cqs);
1094 if (err) {
1095 dev_err(&pdev->dev, "Failed to initialize completion queues\n");
1096 goto err_cqs_init;
1097 }
1098
1099 err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_sdq_ops,
1100 num_sdqs);
1101 if (err) {
1102 dev_err(&pdev->dev, "Failed to initialize send descriptor queues\n");
1103 goto err_sdqs_init;
1104 }
1105
1106 err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_rdq_ops,
1107 num_rdqs);
1108 if (err) {
1109 dev_err(&pdev->dev, "Failed to initialize receive descriptor queues\n");
1110 goto err_rdqs_init;
1111 }
1112
1113 /* We have to poll in command interface until queues are initialized */
1114 mlxsw_pci->cmd.nopoll = true;
1115 return 0;
1116
1117err_rdqs_init:
1118 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1119err_sdqs_init:
1120 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1121err_cqs_init:
1122 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1123 return err;
1124}
1125
1126static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci)
1127{
1128 mlxsw_pci->cmd.nopoll = false;
1129 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops);
1130 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1131 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1132 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1133}
1134
1135static void
1136mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,
1137 char *mbox, int index,
1138 const struct mlxsw_swid_config *swid)
1139{
1140 u8 mask = 0;
1141
1142 if (swid->used_type) {
1143 mlxsw_cmd_mbox_config_profile_swid_config_type_set(
1144 mbox, index, swid->type);
1145 mask |= 1;
1146 }
1147 if (swid->used_properties) {
1148 mlxsw_cmd_mbox_config_profile_swid_config_properties_set(
1149 mbox, index, swid->properties);
1150 mask |= 2;
1151 }
1152 mlxsw_cmd_mbox_config_profile_swid_config_mask_set(mbox, index, mask);
1153}
1154
1155static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
1156 const struct mlxsw_config_profile *profile)
1157{
1158 int i;
1159
1160 mlxsw_cmd_mbox_zero(mbox);
1161
1162 if (profile->used_max_vepa_channels) {
1163 mlxsw_cmd_mbox_config_profile_set_max_vepa_channels_set(
1164 mbox, 1);
1165 mlxsw_cmd_mbox_config_profile_max_vepa_channels_set(
1166 mbox, profile->max_vepa_channels);
1167 }
1168 if (profile->used_max_lag) {
1169 mlxsw_cmd_mbox_config_profile_set_max_lag_set(
1170 mbox, 1);
1171 mlxsw_cmd_mbox_config_profile_max_lag_set(
1172 mbox, profile->max_lag);
1173 }
1174 if (profile->used_max_port_per_lag) {
1175 mlxsw_cmd_mbox_config_profile_set_max_port_per_lag_set(
1176 mbox, 1);
1177 mlxsw_cmd_mbox_config_profile_max_port_per_lag_set(
1178 mbox, profile->max_port_per_lag);
1179 }
1180 if (profile->used_max_mid) {
1181 mlxsw_cmd_mbox_config_profile_set_max_mid_set(
1182 mbox, 1);
1183 mlxsw_cmd_mbox_config_profile_max_mid_set(
1184 mbox, profile->max_mid);
1185 }
1186 if (profile->used_max_pgt) {
1187 mlxsw_cmd_mbox_config_profile_set_max_pgt_set(
1188 mbox, 1);
1189 mlxsw_cmd_mbox_config_profile_max_pgt_set(
1190 mbox, profile->max_pgt);
1191 }
1192 if (profile->used_max_system_port) {
1193 mlxsw_cmd_mbox_config_profile_set_max_system_port_set(
1194 mbox, 1);
1195 mlxsw_cmd_mbox_config_profile_max_system_port_set(
1196 mbox, profile->max_system_port);
1197 }
1198 if (profile->used_max_vlan_groups) {
1199 mlxsw_cmd_mbox_config_profile_set_max_vlan_groups_set(
1200 mbox, 1);
1201 mlxsw_cmd_mbox_config_profile_max_vlan_groups_set(
1202 mbox, profile->max_vlan_groups);
1203 }
1204 if (profile->used_max_regions) {
1205 mlxsw_cmd_mbox_config_profile_set_max_regions_set(
1206 mbox, 1);
1207 mlxsw_cmd_mbox_config_profile_max_regions_set(
1208 mbox, profile->max_regions);
1209 }
1210 if (profile->used_flood_tables) {
1211 mlxsw_cmd_mbox_config_profile_set_flood_tables_set(
1212 mbox, 1);
1213 mlxsw_cmd_mbox_config_profile_max_flood_tables_set(
1214 mbox, profile->max_flood_tables);
1215 mlxsw_cmd_mbox_config_profile_max_vid_flood_tables_set(
1216 mbox, profile->max_vid_flood_tables);
12fd35ab
IS		 1217 mlxsw_cmd_mbox_config_profile_max_fid_offset_flood_tables_set(
1218 mbox, profile->max_fid_offset_flood_tables);
1219 mlxsw_cmd_mbox_config_profile_fid_offset_flood_table_size_set(
1220 mbox, profile->fid_offset_flood_table_size);
453b6a8d
IS		 1221 mlxsw_cmd_mbox_config_profile_max_fid_flood_tables_set(
1222 mbox, profile->max_fid_flood_tables);
1223 mlxsw_cmd_mbox_config_profile_fid_flood_table_size_set(
1224 mbox, profile->fid_flood_table_size);
eda6500a
JP		 1225 }
1226 if (profile->used_flood_mode) {
1227 mlxsw_cmd_mbox_config_profile_set_flood_mode_set(
1228 mbox, 1);
1229 mlxsw_cmd_mbox_config_profile_flood_mode_set(
1230 mbox, profile->flood_mode);
1231 }
1232 if (profile->used_max_ib_mc) {
1233 mlxsw_cmd_mbox_config_profile_set_max_ib_mc_set(
1234 mbox, 1);
1235 mlxsw_cmd_mbox_config_profile_max_ib_mc_set(
1236 mbox, profile->max_ib_mc);
1237 }
1238 if (profile->used_max_pkey) {
1239 mlxsw_cmd_mbox_config_profile_set_max_pkey_set(
1240 mbox, 1);
1241 mlxsw_cmd_mbox_config_profile_max_pkey_set(
1242 mbox, profile->max_pkey);
1243 }
1244 if (profile->used_ar_sec) {
1245 mlxsw_cmd_mbox_config_profile_set_ar_sec_set(
1246 mbox, 1);
1247 mlxsw_cmd_mbox_config_profile_ar_sec_set(
1248 mbox, profile->ar_sec);
1249 }
1250 if (profile->used_adaptive_routing_group_cap) {
1251 mlxsw_cmd_mbox_config_profile_set_adaptive_routing_group_cap_set(
1252 mbox, 1);
1253 mlxsw_cmd_mbox_config_profile_adaptive_routing_group_cap_set(
1254 mbox, profile->adaptive_routing_group_cap);
1255 }
1256
1257 for (i = 0; i < MLXSW_CONFIG_PROFILE_SWID_COUNT; i++)
1258 mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,
1259 &profile->swid_config[i]);
1260
1261 return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);
1262}
1263
1264static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)
1265{
1266 struct mlxsw_bus_info *bus_info = &mlxsw_pci->bus_info;
1267 int err;
1268
1269 mlxsw_cmd_mbox_zero(mbox);
1270 err = mlxsw_cmd_boardinfo(mlxsw_pci->core, mbox);
1271 if (err)
1272 return err;
1273 mlxsw_cmd_mbox_boardinfo_vsd_memcpy_from(mbox, bus_info->vsd);
1274 mlxsw_cmd_mbox_boardinfo_psid_memcpy_from(mbox, bus_info->psid);
1275 return 0;
1276}
1277
1278static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
1279 u16 num_pages)
1280{
1281 struct mlxsw_pci_mem_item *mem_item;
3e2206da 1282 int nent = 0;
eda6500a
JP		 1283 int i;
1284 int err;
1285
1286 mlxsw_pci->fw_area.items = kcalloc(num_pages, sizeof(*mem_item),
1287 GFP_KERNEL);
1288 if (!mlxsw_pci->fw_area.items)
1289 return -ENOMEM;
3e2206da 1290 mlxsw_pci->fw_area.count = num_pages;
eda6500a
JP		 1291
1292 mlxsw_cmd_mbox_zero(mbox);
1293 for (i = 0; i < num_pages; i++) {
1294 mem_item = &mlxsw_pci->fw_area.items[i];
1295
1296 mem_item->size = MLXSW_PCI_PAGE_SIZE;
1297 mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,
1298 mem_item->size,
1299 &mem_item->mapaddr);
1300 if (!mem_item->buf) {
1301 err = -ENOMEM;
1302 goto err_alloc;
1303 }
3e2206da
JP		 1304 mlxsw_cmd_mbox_map_fa_pa_set(mbox, nent, mem_item->mapaddr);
1305 mlxsw_cmd_mbox_map_fa_log2size_set(mbox, nent, 0); /* 1 page */
1306 if (++nent == MLXSW_CMD_MAP_FA_VPM_ENTRIES_MAX) {
1307 err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);
1308 if (err)
1309 goto err_cmd_map_fa;
1310 nent = 0;
1311 mlxsw_cmd_mbox_zero(mbox);
1312 }
eda6500a
JP		 1313 }
1314
3e2206da
JP		 1315 if (nent) {
1316 err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);
1317 if (err)
1318 goto err_cmd_map_fa;
1319 }
eda6500a
JP		 1320
1321 return 0;
1322
1323err_cmd_map_fa:
1324err_alloc:
1325 for (i--; i >= 0; i--) {
1326 mem_item = &mlxsw_pci->fw_area.items[i];
1327
1328 pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
1329 mem_item->buf, mem_item->mapaddr);
1330 }
1331 kfree(mlxsw_pci->fw_area.items);
1332 return err;
1333}
1334
1335static void mlxsw_pci_fw_area_fini(struct mlxsw_pci *mlxsw_pci)
1336{
1337 struct mlxsw_pci_mem_item *mem_item;
1338 int i;
1339
1340 mlxsw_cmd_unmap_fa(mlxsw_pci->core);
1341
3e2206da 1342 for (i = 0; i < mlxsw_pci->fw_area.count; i++) {
eda6500a
JP		 1343 mem_item = &mlxsw_pci->fw_area.items[i];
1344
1345 pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
1346 mem_item->buf, mem_item->mapaddr);
1347 }
1348 kfree(mlxsw_pci->fw_area.items);
1349}
1350
1351static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id)
1352{
1353 struct mlxsw_pci *mlxsw_pci = dev_id;
1354 struct mlxsw_pci_queue *q;
1355 int i;
1356
1357 for (i = 0; i < MLXSW_PCI_EQS_COUNT; i++) {
1358 q = mlxsw_pci_eq_get(mlxsw_pci, i);
1359 mlxsw_pci_queue_tasklet_schedule(q);
1360 }
1361 return IRQ_HANDLED;
1362}
1363
1e81779a
IS		 1364static int mlxsw_pci_mbox_alloc(struct mlxsw_pci *mlxsw_pci,
1365 struct mlxsw_pci_mem_item *mbox)
1366{
1367 struct pci_dev *pdev = mlxsw_pci->pdev;
1368 int err = 0;
1369
1370 mbox->size = MLXSW_CMD_MBOX_SIZE;
1371 mbox->buf = pci_alloc_consistent(pdev, MLXSW_CMD_MBOX_SIZE,
1372 &mbox->mapaddr);
1373 if (!mbox->buf) {
1374 dev_err(&pdev->dev, "Failed allocating memory for mailbox\n");
1375 err = -ENOMEM;
1376 }
1377
1378 return err;
1379}
1380
1381static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,
1382 struct mlxsw_pci_mem_item *mbox)
1383{
1384 struct pci_dev *pdev = mlxsw_pci->pdev;
1385
1386 pci_free_consistent(pdev, MLXSW_CMD_MBOX_SIZE, mbox->buf,
1387 mbox->mapaddr);
1388}
1389
eda6500a
JP		 1390static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
1391 const struct mlxsw_config_profile *profile)
1392{
1393 struct mlxsw_pci *mlxsw_pci = bus_priv;
1394 struct pci_dev *pdev = mlxsw_pci->pdev;
1395 char *mbox;
1396 u16 num_pages;
1397 int err;
1398
1399 mutex_init(&mlxsw_pci->cmd.lock);
1400 init_waitqueue_head(&mlxsw_pci->cmd.wait);
1401
1402 mlxsw_pci->core = mlxsw_core;
1403
1404 mbox = mlxsw_cmd_mbox_alloc();
1405 if (!mbox)
1406 return -ENOMEM;
1e81779a
IS		 1407
1408 err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1409 if (err)
1410 goto mbox_put;
1411
1412 err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1413 if (err)
1414 goto err_out_mbox_alloc;
1415
eda6500a
JP		 1416 err = mlxsw_cmd_query_fw(mlxsw_core, mbox);
1417 if (err)
1418 goto err_query_fw;
1419
1420 mlxsw_pci->bus_info.fw_rev.major =
1421 mlxsw_cmd_mbox_query_fw_fw_rev_major_get(mbox);
1422 mlxsw_pci->bus_info.fw_rev.minor =
1423 mlxsw_cmd_mbox_query_fw_fw_rev_minor_get(mbox);
1424 mlxsw_pci->bus_info.fw_rev.subminor =
1425 mlxsw_cmd_mbox_query_fw_fw_rev_subminor_get(mbox);
1426
1427 if (mlxsw_cmd_mbox_query_fw_cmd_interface_rev_get(mbox) != 1) {
1428 dev_err(&pdev->dev, "Unsupported cmd interface revision ID queried from hw\n");
1429 err = -EINVAL;
1430 goto err_iface_rev;
1431 }
1432 if (mlxsw_cmd_mbox_query_fw_doorbell_page_bar_get(mbox) != 0) {
1433 dev_err(&pdev->dev, "Unsupported doorbell page bar queried from hw\n");
1434 err = -EINVAL;
1435 goto err_doorbell_page_bar;
1436 }
1437
1438 mlxsw_pci->doorbell_offset =
1439 mlxsw_cmd_mbox_query_fw_doorbell_page_offset_get(mbox);
1440
1441 num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox);
1442 err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);
1443 if (err)
1444 goto err_fw_area_init;
1445
1446 err = mlxsw_pci_boardinfo(mlxsw_pci, mbox);
1447 if (err)
1448 goto err_boardinfo;
1449
1450 err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile);
1451 if (err)
1452 goto err_config_profile;
1453
1454 err = mlxsw_pci_aqs_init(mlxsw_pci, mbox);
1455 if (err)
1456 goto err_aqs_init;
1457
1458 err = request_irq(mlxsw_pci->msix_entry.vector,
1459 mlxsw_pci_eq_irq_handler, 0,
1460 mlxsw_pci_driver_name, mlxsw_pci);
1461 if (err) {
1462 dev_err(&pdev->dev, "IRQ request failed\n");
1463 goto err_request_eq_irq;
1464 }
1465
1466 goto mbox_put;
1467
1468err_request_eq_irq:
1469 mlxsw_pci_aqs_fini(mlxsw_pci);
1470err_aqs_init:
1471err_config_profile:
1472err_boardinfo:
1473 mlxsw_pci_fw_area_fini(mlxsw_pci);
1474err_fw_area_init:
1475err_doorbell_page_bar:
1476err_iface_rev:
1477err_query_fw:
1e81779a
IS		 1478 mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1479err_out_mbox_alloc:
1480 mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
eda6500a
JP		 1481mbox_put:
1482 mlxsw_cmd_mbox_free(mbox);
1483 return err;
1484}
1485
1486static void mlxsw_pci_fini(void *bus_priv)
1487{
1488 struct mlxsw_pci *mlxsw_pci = bus_priv;
1489
1490 free_irq(mlxsw_pci->msix_entry.vector, mlxsw_pci);
1491 mlxsw_pci_aqs_fini(mlxsw_pci);
1492 mlxsw_pci_fw_area_fini(mlxsw_pci);
1e81779a
IS		 1493 mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1494 mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
eda6500a
JP		 1495}
1496
1497static struct mlxsw_pci_queue *
1498mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci,
1499 const struct mlxsw_tx_info *tx_info)
1500{
1501 u8 sdqn = tx_info->local_port % mlxsw_pci_sdq_count(mlxsw_pci);
1502
1503 return mlxsw_pci_sdq_get(mlxsw_pci, sdqn);
1504}
1505
d003462a
IS		 1506static bool mlxsw_pci_skb_transmit_busy(void *bus_priv,
1507 const struct mlxsw_tx_info *tx_info)
1508{
1509 struct mlxsw_pci *mlxsw_pci = bus_priv;
1510 struct mlxsw_pci_queue *q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1511
1512 return !mlxsw_pci_queue_elem_info_producer_get(q);
1513}
1514
eda6500a
JP		 1515static int mlxsw_pci_skb_transmit(void *bus_priv, struct sk_buff *skb,
1516 const struct mlxsw_tx_info *tx_info)
1517{
1518 struct mlxsw_pci *mlxsw_pci = bus_priv;
1519 struct mlxsw_pci_queue *q;
1520 struct mlxsw_pci_queue_elem_info *elem_info;
1521 char *wqe;
1522 int i;
1523 int err;
1524
1525 if (skb_shinfo(skb)->nr_frags > MLXSW_PCI_WQE_SG_ENTRIES - 1) {
1526 err = skb_linearize(skb);
1527 if (err)
1528 return err;
1529 }
1530
1531 q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1532 spin_lock_bh(&q->lock);
1533 elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
1534 if (!elem_info) {
1535 /* queue is full */
1536 err = -EAGAIN;
1537 goto unlock;
1538 }
1539 elem_info->u.sdq.skb = skb;
1540
1541 wqe = elem_info->elem;
1542 mlxsw_pci_wqe_c_set(wqe, 1); /* always report completion */
1543 mlxsw_pci_wqe_lp_set(wqe, !!tx_info->is_emad);
1544 mlxsw_pci_wqe_type_set(wqe, MLXSW_PCI_WQE_TYPE_ETHERNET);
1545
1546 err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
1547 skb_headlen(skb), DMA_TO_DEVICE);
1548 if (err)
1549 goto unlock;
1550
1551 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1552 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1553
1554 err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, i + 1,
1555 skb_frag_address(frag),
1556 skb_frag_size(frag),
1557 DMA_TO_DEVICE);
1558 if (err)
1559 goto unmap_frags;
1560 }
1561
1562 /* Set unused sq entries byte count to zero. */
1563 for (i++; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
1564 mlxsw_pci_wqe_byte_count_set(wqe, i, 0);
1565
1566 /* Everything is set up, ring producer doorbell to get HW going */
1567 q->producer_counter++;
1568 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
1569
1570 goto unlock;
1571
1572unmap_frags:
1573 for (; i >= 0; i--)
1574 mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
1575unlock:
1576 spin_unlock_bh(&q->lock);
1577 return err;
1578}
1579
1580static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
1581 u32 in_mod, bool out_mbox_direct,
1582 char *in_mbox, size_t in_mbox_size,
1583 char *out_mbox, size_t out_mbox_size,
1584 u8 *p_status)
1585{
1586 struct mlxsw_pci *mlxsw_pci = bus_priv;
1e81779a
IS		 1587 dma_addr_t in_mapaddr = mlxsw_pci->cmd.in_mbox.mapaddr;
1588 dma_addr_t out_mapaddr = mlxsw_pci->cmd.out_mbox.mapaddr;
eda6500a
JP		 1589 bool evreq = mlxsw_pci->cmd.nopoll;
1590 unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS);
1591 bool *p_wait_done = &mlxsw_pci->cmd.wait_done;
1592 int err;
1593
1594 *p_status = MLXSW_CMD_STATUS_OK;
1595
1596 err = mutex_lock_interruptible(&mlxsw_pci->cmd.lock);
1597 if (err)
1598 return err;
1599
1e81779a
IS		 1600 if (in_mbox)
1601 memcpy(mlxsw_pci->cmd.in_mbox.buf, in_mbox, in_mbox_size);
eda6500a
JP		 1602 mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_HI, in_mapaddr >> 32);
1603 mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_LO, in_mapaddr);
1604
eda6500a
JP		 1605 mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_HI, out_mapaddr >> 32);
1606 mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_LO, out_mapaddr);
1607
1608 mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod);
1609 mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0);
1610
1611 *p_wait_done = false;
1612
1613 wmb(); /* all needs to be written before we write control register */
1614 mlxsw_pci_write32(mlxsw_pci, CIR_CTRL,
1615 MLXSW_PCI_CIR_CTRL_GO_BIT |
1616 (evreq ? MLXSW_PCI_CIR_CTRL_EVREQ_BIT : 0) |
1617 (opcode_mod << MLXSW_PCI_CIR_CTRL_OPCODE_MOD_SHIFT) |
1618 opcode);
1619
1620 if (!evreq) {
1621 unsigned long end;
1622
1623 end = jiffies + timeout;
1624 do {
1625 u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL);
1626
1627 if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) {
1628 *p_wait_done = true;
1629 *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT;
1630 break;
1631 }
1632 cond_resched();
1633 } while (time_before(jiffies, end));
1634 } else {
1635 wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout);
1636 *p_status = mlxsw_pci->cmd.comp.status;
1637 }
1638
1639 err = 0;
1640 if (*p_wait_done) {
1641 if (*p_status)
1642 err = -EIO;
1643 } else {
1644 err = -ETIMEDOUT;
1645 }
1646
1647 if (!err && out_mbox && out_mbox_direct) {
1e81779a 1648 /* Some commands don't use output param as address to mailbox
eda6500a
JP		 1649 * but they store output directly into registers. In that case,
1650 * copy registers into mbox buffer.
1651 */
1652 __be32 tmp;
1653
1654 if (!evreq) {
1655 tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1656 CIR_OUT_PARAM_HI));
1657 memcpy(out_mbox, &tmp, sizeof(tmp));
1658 tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1659 CIR_OUT_PARAM_LO));
1660 memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp));
1661 }
1e81779a
IS		 1662 } else if (!err && out_mbox)
1663 memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size);
eda6500a 1664
eda6500a
JP		 1665 mutex_unlock(&mlxsw_pci->cmd.lock);
1666
1667 return err;
1668}
1669
1670static const struct mlxsw_bus mlxsw_pci_bus = {
d003462a
IS		 1671 .kind = "pci",
1672 .init = mlxsw_pci_init,
1673 .fini = mlxsw_pci_fini,
1674 .skb_transmit_busy = mlxsw_pci_skb_transmit_busy,
1675 .skb_transmit = mlxsw_pci_skb_transmit,
1676 .cmd_exec = mlxsw_pci_cmd_exec,
eda6500a
JP		 1677};
1678
1679static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci)
1680{
1681 mlxsw_pci_write32(mlxsw_pci, SW_RESET, MLXSW_PCI_SW_RESET_RST_BIT);
1682 /* Current firware does not let us know when the reset is done.
1683 * So we just wait here for constant time and hope for the best.
1684 */
1685 msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
1686 return 0;
1687}
1688
1689static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1690{
1691 struct mlxsw_pci *mlxsw_pci;
1692 int err;
1693
1694 mlxsw_pci = kzalloc(sizeof(*mlxsw_pci), GFP_KERNEL);
1695 if (!mlxsw_pci)
1696 return -ENOMEM;
1697
1698 err = pci_enable_device(pdev);
1699 if (err) {
1700 dev_err(&pdev->dev, "pci_enable_device failed\n");
1701 goto err_pci_enable_device;
1702 }
1703
1704 err = pci_request_regions(pdev, mlxsw_pci_driver_name);
1705 if (err) {
1706 dev_err(&pdev->dev, "pci_request_regions failed\n");
1707 goto err_pci_request_regions;
1708 }
1709
1710 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1711 if (!err) {
1712 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1713 if (err) {
1714 dev_err(&pdev->dev, "pci_set_consistent_dma_mask failed\n");
1715 goto err_pci_set_dma_mask;
1716 }
1717 } else {
1718 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1719 if (err) {
1720 dev_err(&pdev->dev, "pci_set_dma_mask failed\n");
1721 goto err_pci_set_dma_mask;
1722 }
1723 }
1724
1725 if (pci_resource_len(pdev, 0) < MLXSW_PCI_BAR0_SIZE) {
1726 dev_err(&pdev->dev, "invalid PCI region size\n");
1727 err = -EINVAL;
1728 goto err_pci_resource_len_check;
1729 }
1730
1731 mlxsw_pci->hw_addr = ioremap(pci_resource_start(pdev, 0),
1732 pci_resource_len(pdev, 0));
1733 if (!mlxsw_pci->hw_addr) {
1734 dev_err(&pdev->dev, "ioremap failed\n");
1735 err = -EIO;
1736 goto err_ioremap;
1737 }
1738 pci_set_master(pdev);
1739
1740 mlxsw_pci->pdev = pdev;
1741 pci_set_drvdata(pdev, mlxsw_pci);
1742
1743 err = mlxsw_pci_sw_reset(mlxsw_pci);
1744 if (err) {
1745 dev_err(&pdev->dev, "Software reset failed\n");
1746 goto err_sw_reset;
1747 }
1748
1749 err = pci_enable_msix_exact(pdev, &mlxsw_pci->msix_entry, 1);
1750 if (err) {
1751 dev_err(&pdev->dev, "MSI-X init failed\n");
1752 goto err_msix_init;
1753 }
1754
1755 mlxsw_pci->bus_info.device_kind = mlxsw_pci_device_kind_get(id);
1756 mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);
1757 mlxsw_pci->bus_info.dev = &pdev->dev;
1758
1759 mlxsw_pci->dbg_dir = debugfs_create_dir(mlxsw_pci->bus_info.device_name,
1760 mlxsw_pci_dbg_root);
1761 if (!mlxsw_pci->dbg_dir) {
1762 dev_err(&pdev->dev, "Failed to create debugfs dir\n");
5c121979 1763 err = -ENOMEM;
eda6500a
JP		 1764 goto err_dbg_create_dir;
1765 }
1766
1767 err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,
1768 &mlxsw_pci_bus, mlxsw_pci);
1769 if (err) {
1770 dev_err(&pdev->dev, "cannot register bus device\n");
1771 goto err_bus_device_register;
1772 }
1773
1774 return 0;
1775
1776err_bus_device_register:
1777 debugfs_remove_recursive(mlxsw_pci->dbg_dir);
1778err_dbg_create_dir:
1779 pci_disable_msix(mlxsw_pci->pdev);
1780err_msix_init:
1781err_sw_reset:
1782 iounmap(mlxsw_pci->hw_addr);
1783err_ioremap:
1784err_pci_resource_len_check:
1785err_pci_set_dma_mask:
1786 pci_release_regions(pdev);
1787err_pci_request_regions:
1788 pci_disable_device(pdev);
1789err_pci_enable_device:
1790 kfree(mlxsw_pci);
1791 return err;
1792}
1793
1794static void mlxsw_pci_remove(struct pci_dev *pdev)
1795{
1796 struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
1797
1798 mlxsw_core_bus_device_unregister(mlxsw_pci->core);
1799 debugfs_remove_recursive(mlxsw_pci->dbg_dir);
1800 pci_disable_msix(mlxsw_pci->pdev);
1801 iounmap(mlxsw_pci->hw_addr);
1802 pci_release_regions(mlxsw_pci->pdev);
1803 pci_disable_device(mlxsw_pci->pdev);
1804 kfree(mlxsw_pci);
1805}
1806
1807static struct pci_driver mlxsw_pci_driver = {
1808 .name = mlxsw_pci_driver_name,
1809 .id_table = mlxsw_pci_id_table,
1810 .probe = mlxsw_pci_probe,
1811 .remove = mlxsw_pci_remove,
1812};
1813
1814static int __init mlxsw_pci_module_init(void)
1815{
1816 int err;
1817
1818 mlxsw_pci_dbg_root = debugfs_create_dir(mlxsw_pci_driver_name, NULL);
1819 if (!mlxsw_pci_dbg_root)
1820 return -ENOMEM;
1821 err = pci_register_driver(&mlxsw_pci_driver);
1822 if (err)
1823 goto err_register_driver;
1824 return 0;
1825
1826err_register_driver:
1827 debugfs_remove_recursive(mlxsw_pci_dbg_root);
1828 return err;
1829}
1830
1831static void __exit mlxsw_pci_module_exit(void)
1832{
1833 pci_unregister_driver(&mlxsw_pci_driver);
1834 debugfs_remove_recursive(mlxsw_pci_dbg_root);
1835}
1836
1837module_init(mlxsw_pci_module_init);
1838module_exit(mlxsw_pci_module_exit);
1839
1840MODULE_LICENSE("Dual BSD/GPL");
1841MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
1842MODULE_DESCRIPTION("Mellanox switch PCI interface driver");
1843MODULE_DEVICE_TABLE(pci, mlxsw_pci_id_table);
Linux kernel - Deliverables
This page took 0.21925 seconds and 5 git commands to generate.