Commit | Line | Data |
---|---|---|
f21fb3ed RV |
1 | /********************************************************************** |
2 | * Author: Cavium, Inc. | |
3 | * | |
4 | * Contact: support@cavium.com | |
5 | * Please include "LiquidIO" in the subject. | |
6 | * | |
7 | * Copyright (c) 2003-2015 Cavium, Inc. | |
8 | * | |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more | |
17 | * details. | |
18 | * | |
19 | * This file may also be available under a different license from Cavium. | |
20 | * Contact Cavium, Inc. for more information | |
21 | **********************************************************************/ | |
f21fb3ed | 22 | #include <linux/pci.h> |
f21fb3ed | 23 | #include <linux/netdevice.h> |
5b173cf9 | 24 | #include <linux/vmalloc.h> |
f21fb3ed RV |
25 | #include "liquidio_common.h" |
26 | #include "octeon_droq.h" | |
27 | #include "octeon_iq.h" | |
28 | #include "response_manager.h" | |
29 | #include "octeon_device.h" | |
f21fb3ed RV |
30 | #include "octeon_main.h" |
31 | #include "octeon_network.h" | |
32 | #include "cn66xx_regs.h" | |
33 | #include "cn66xx_device.h" | |
5b823514 | 34 | #include "cn23xx_pf_device.h" |
f21fb3ed RV |
35 | |
36 | #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2)) | |
37 | #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2)) | |
38 | ||
39 | struct niclist { | |
40 | struct list_head list; | |
41 | void *ptr; | |
42 | }; | |
43 | ||
44 | struct __dispatch { | |
45 | struct list_head list; | |
46 | struct octeon_recv_info *rinfo; | |
47 | octeon_dispatch_fn_t disp_fn; | |
48 | }; | |
49 | ||
50 | /** Get the argument that the user set when registering dispatch | |
51 | * function for a given opcode/subcode. | |
52 | * @param octeon_dev - the octeon device pointer. | |
53 | * @param opcode - the opcode for which the dispatch argument | |
54 | * is to be checked. | |
55 | * @param subcode - the subcode for which the dispatch argument | |
56 | * is to be checked. | |
57 | * @return Success: void * (argument to the dispatch function) | |
58 | * @return Failure: NULL | |
59 | * | |
60 | */ | |
61 | static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, | |
62 | u16 opcode, u16 subcode) | |
63 | { | |
64 | int idx; | |
65 | struct list_head *dispatch; | |
66 | void *fn_arg = NULL; | |
67 | u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); | |
68 | ||
69 | idx = combined_opcode & OCTEON_OPCODE_MASK; | |
70 | ||
71 | spin_lock_bh(&octeon_dev->dispatch.lock); | |
72 | ||
73 | if (octeon_dev->dispatch.count == 0) { | |
74 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
75 | return NULL; | |
76 | } | |
77 | ||
78 | if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { | |
79 | fn_arg = octeon_dev->dispatch.dlist[idx].arg; | |
80 | } else { | |
81 | list_for_each(dispatch, | |
82 | &octeon_dev->dispatch.dlist[idx].list) { | |
83 | if (((struct octeon_dispatch *)dispatch)->opcode == | |
84 | combined_opcode) { | |
85 | fn_arg = ((struct octeon_dispatch *) | |
86 | dispatch)->arg; | |
87 | break; | |
88 | } | |
89 | } | |
90 | } | |
91 | ||
92 | spin_unlock_bh(&octeon_dev->dispatch.lock); | |
93 | return fn_arg; | |
94 | } | |
95 | ||
cd8b1eb4 | 96 | /** Check for packets on Droq. This function should be called with lock held. |
a2c64b67 RV |
97 | * @param droq - Droq on which count is checked. |
98 | * @return Returns packet count. | |
99 | */ | |
a7d5a3dc | 100 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) |
f21fb3ed RV |
101 | { |
102 | u32 pkt_count = 0; | |
cd8b1eb4 | 103 | u32 last_count; |
f21fb3ed RV |
104 | |
105 | pkt_count = readl(droq->pkts_sent_reg); | |
f21fb3ed | 106 | |
cd8b1eb4 RV |
107 | last_count = pkt_count - droq->pkt_count; |
108 | droq->pkt_count = pkt_count; | |
109 | ||
110 | /* we shall write to cnts at napi irq enable or end of droq tasklet */ | |
111 | if (last_count) | |
112 | atomic_add(last_count, &droq->pkts_pending); | |
113 | ||
114 | return last_count; | |
f21fb3ed RV |
115 | } |
116 | ||
117 | static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) | |
118 | { | |
119 | u32 count = 0; | |
120 | ||
121 | /* max_empty_descs is the max. no. of descs that can have no buffers. | |
122 | * If the empty desc count goes beyond this value, we cannot safely | |
123 | * read in a 64K packet sent by Octeon | |
124 | * (64K is max pkt size from Octeon) | |
125 | */ | |
126 | droq->max_empty_descs = 0; | |
127 | ||
128 | do { | |
129 | droq->max_empty_descs++; | |
130 | count += droq->buffer_size; | |
131 | } while (count < (64 * 1024)); | |
132 | ||
133 | droq->max_empty_descs = droq->max_count - droq->max_empty_descs; | |
134 | } | |
135 | ||
136 | static void octeon_droq_reset_indices(struct octeon_droq *droq) | |
137 | { | |
138 | droq->read_idx = 0; | |
139 | droq->write_idx = 0; | |
140 | droq->refill_idx = 0; | |
141 | droq->refill_count = 0; | |
142 | atomic_set(&droq->pkts_pending, 0); | |
143 | } | |
144 | ||
145 | static void | |
146 | octeon_droq_destroy_ring_buffers(struct octeon_device *oct, | |
147 | struct octeon_droq *droq) | |
148 | { | |
149 | u32 i; | |
cabeb13b | 150 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
151 | |
152 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b RV |
153 | pg_info = &droq->recv_buf_list[i].pg_info; |
154 | ||
155 | if (pg_info->dma) | |
156 | lio_unmap_ring(oct->pci_dev, | |
157 | (u64)pg_info->dma); | |
158 | pg_info->dma = 0; | |
159 | ||
160 | if (pg_info->page) | |
161 | recv_buffer_destroy(droq->recv_buf_list[i].buffer, | |
162 | pg_info); | |
163 | ||
164 | if (droq->desc_ring && droq->desc_ring[i].info_ptr) | |
165 | lio_unmap_ring_info(oct->pci_dev, | |
166 | (u64)droq-> | |
167 | desc_ring[i].info_ptr, | |
168 | OCT_DROQ_INFO_SIZE); | |
169 | droq->recv_buf_list[i].buffer = NULL; | |
f21fb3ed RV |
170 | } |
171 | ||
172 | octeon_droq_reset_indices(droq); | |
173 | } | |
174 | ||
175 | static int | |
176 | octeon_droq_setup_ring_buffers(struct octeon_device *oct, | |
177 | struct octeon_droq *droq) | |
178 | { | |
179 | u32 i; | |
180 | void *buf; | |
181 | struct octeon_droq_desc *desc_ring = droq->desc_ring; | |
182 | ||
183 | for (i = 0; i < droq->max_count; i++) { | |
cabeb13b | 184 | buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); |
f21fb3ed RV |
185 | |
186 | if (!buf) { | |
187 | dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", | |
188 | __func__); | |
cabeb13b | 189 | droq->stats.rx_alloc_failure++; |
f21fb3ed RV |
190 | return -ENOMEM; |
191 | } | |
192 | ||
193 | droq->recv_buf_list[i].buffer = buf; | |
194 | droq->recv_buf_list[i].data = get_rbd(buf); | |
f21fb3ed RV |
195 | droq->info_list[i].length = 0; |
196 | ||
197 | /* map ring buffers into memory */ | |
198 | desc_ring[i].info_ptr = lio_map_ring_info(droq, i); | |
199 | desc_ring[i].buffer_ptr = | |
cabeb13b | 200 | lio_map_ring(droq->recv_buf_list[i].buffer); |
f21fb3ed RV |
201 | } |
202 | ||
203 | octeon_droq_reset_indices(droq); | |
204 | ||
205 | octeon_droq_compute_max_packet_bufs(droq); | |
206 | ||
207 | return 0; | |
208 | } | |
209 | ||
210 | int octeon_delete_droq(struct octeon_device *oct, u32 q_no) | |
211 | { | |
212 | struct octeon_droq *droq = oct->droq[q_no]; | |
213 | ||
214 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
215 | ||
216 | octeon_droq_destroy_ring_buffers(oct, droq); | |
9686f310 | 217 | vfree(droq->recv_buf_list); |
f21fb3ed RV |
218 | |
219 | if (droq->info_base_addr) | |
220 | cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, | |
221 | droq->info_alloc_size, | |
222 | droq->info_base_addr, | |
223 | droq->info_list_dma); | |
224 | ||
225 | if (droq->desc_ring) | |
226 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), | |
227 | droq->desc_ring, droq->desc_ring_dma); | |
228 | ||
229 | memset(droq, 0, OCT_DROQ_SIZE); | |
230 | ||
231 | return 0; | |
232 | } | |
233 | ||
234 | int octeon_init_droq(struct octeon_device *oct, | |
235 | u32 q_no, | |
236 | u32 num_descs, | |
237 | u32 desc_size, | |
238 | void *app_ctx) | |
239 | { | |
240 | struct octeon_droq *droq; | |
241 | u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; | |
242 | u32 c_pkts_per_intr = 0, c_refill_threshold = 0; | |
96ae48b7 RV |
243 | int orig_node = dev_to_node(&oct->pci_dev->dev); |
244 | int numa_node = cpu_to_node(q_no % num_online_cpus()); | |
f21fb3ed RV |
245 | |
246 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); | |
247 | ||
248 | droq = oct->droq[q_no]; | |
249 | memset(droq, 0, OCT_DROQ_SIZE); | |
250 | ||
251 | droq->oct_dev = oct; | |
252 | droq->q_no = q_no; | |
253 | if (app_ctx) | |
254 | droq->app_ctx = app_ctx; | |
255 | else | |
256 | droq->app_ctx = (void *)(size_t)q_no; | |
257 | ||
258 | c_num_descs = num_descs; | |
259 | c_buf_size = desc_size; | |
260 | if (OCTEON_CN6XXX(oct)) { | |
261 | struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); | |
262 | ||
263 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); | |
96ae48b7 RV |
264 | c_refill_threshold = |
265 | (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); | |
5b823514 RV |
266 | } else if (OCTEON_CN23XX_PF(oct)) { |
267 | struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf); | |
268 | ||
269 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); | |
270 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); | |
96ae48b7 RV |
271 | } else { |
272 | return 1; | |
f21fb3ed RV |
273 | } |
274 | ||
275 | droq->max_count = c_num_descs; | |
276 | droq->buffer_size = c_buf_size; | |
277 | ||
278 | desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; | |
96ae48b7 | 279 | set_dev_node(&oct->pci_dev->dev, numa_node); |
f21fb3ed RV |
280 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, |
281 | (dma_addr_t *)&droq->desc_ring_dma); | |
96ae48b7 RV |
282 | set_dev_node(&oct->pci_dev->dev, orig_node); |
283 | if (!droq->desc_ring) | |
284 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, | |
285 | (dma_addr_t *)&droq->desc_ring_dma); | |
f21fb3ed RV |
286 | |
287 | if (!droq->desc_ring) { | |
288 | dev_err(&oct->pci_dev->dev, | |
289 | "Output queue %d ring alloc failed\n", q_no); | |
290 | return 1; | |
291 | } | |
292 | ||
293 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", | |
294 | q_no, droq->desc_ring, droq->desc_ring_dma); | |
295 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, | |
296 | droq->max_count); | |
297 | ||
298 | droq->info_list = | |
96ae48b7 RV |
299 | cnnic_numa_alloc_aligned_dma((droq->max_count * |
300 | OCT_DROQ_INFO_SIZE), | |
301 | &droq->info_alloc_size, | |
302 | &droq->info_base_addr, | |
303 | numa_node); | |
f21fb3ed RV |
304 | if (!droq->info_list) { |
305 | dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); | |
306 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), | |
307 | droq->desc_ring, droq->desc_ring_dma); | |
308 | return 1; | |
309 | } | |
310 | ||
311 | droq->recv_buf_list = (struct octeon_recv_buffer *) | |
96ae48b7 RV |
312 | vmalloc_node(droq->max_count * |
313 | OCT_DROQ_RECVBUF_SIZE, | |
314 | numa_node); | |
315 | if (!droq->recv_buf_list) | |
316 | droq->recv_buf_list = (struct octeon_recv_buffer *) | |
317 | vmalloc(droq->max_count * | |
f21fb3ed RV |
318 | OCT_DROQ_RECVBUF_SIZE); |
319 | if (!droq->recv_buf_list) { | |
320 | dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); | |
321 | goto init_droq_fail; | |
322 | } | |
323 | ||
324 | if (octeon_droq_setup_ring_buffers(oct, droq)) | |
325 | goto init_droq_fail; | |
326 | ||
327 | droq->pkts_per_intr = c_pkts_per_intr; | |
328 | droq->refill_threshold = c_refill_threshold; | |
329 | ||
330 | dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", | |
331 | droq->max_empty_descs); | |
332 | ||
333 | spin_lock_init(&droq->lock); | |
334 | ||
335 | INIT_LIST_HEAD(&droq->dispatch_list); | |
336 | ||
337 | /* For 56xx Pass1, this function won't be called, so no checks. */ | |
338 | oct->fn_list.setup_oq_regs(oct, q_no); | |
339 | ||
63da8404 | 340 | oct->io_qmask.oq |= (1ULL << q_no); |
f21fb3ed RV |
341 | |
342 | return 0; | |
343 | ||
344 | init_droq_fail: | |
345 | octeon_delete_droq(oct, q_no); | |
346 | return 1; | |
347 | } | |
348 | ||
349 | /* octeon_create_recv_info | |
350 | * Parameters: | |
351 | * octeon_dev - pointer to the octeon device structure | |
352 | * droq - droq in which the packet arrived. | |
353 | * buf_cnt - no. of buffers used by the packet. | |
354 | * idx - index in the descriptor for the first buffer in the packet. | |
355 | * Description: | |
356 | * Allocates a recv_info_t and copies the buffer addresses for packet data | |
357 | * into the recv_pkt space which starts at an 8B offset from recv_info_t. | |
358 | * Flags the descriptors for refill later. If available descriptors go | |
359 | * below the threshold to receive a 64K pkt, new buffers are first allocated | |
360 | * before the recv_pkt_t is created. | |
361 | * This routine will be called in interrupt context. | |
362 | * Returns: | |
363 | * Success: Pointer to recv_info_t | |
364 | * Failure: NULL. | |
365 | * Locks: | |
366 | * The droq->lock is held when this routine is called. | |
367 | */ | |
368 | static inline struct octeon_recv_info *octeon_create_recv_info( | |
369 | struct octeon_device *octeon_dev, | |
370 | struct octeon_droq *droq, | |
371 | u32 buf_cnt, | |
372 | u32 idx) | |
373 | { | |
374 | struct octeon_droq_info *info; | |
375 | struct octeon_recv_pkt *recv_pkt; | |
376 | struct octeon_recv_info *recv_info; | |
377 | u32 i, bytes_left; | |
cabeb13b | 378 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
379 | |
380 | info = &droq->info_list[idx]; | |
381 | ||
382 | recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); | |
383 | if (!recv_info) | |
384 | return NULL; | |
385 | ||
386 | recv_pkt = recv_info->recv_pkt; | |
387 | recv_pkt->rh = info->rh; | |
388 | recv_pkt->length = (u32)info->length; | |
389 | recv_pkt->buffer_count = (u16)buf_cnt; | |
390 | recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; | |
391 | ||
392 | i = 0; | |
393 | bytes_left = (u32)info->length; | |
394 | ||
395 | while (buf_cnt) { | |
cabeb13b RV |
396 | { |
397 | pg_info = &droq->recv_buf_list[idx].pg_info; | |
398 | ||
399 | lio_unmap_ring(octeon_dev->pci_dev, | |
400 | (u64)pg_info->dma); | |
401 | pg_info->page = NULL; | |
402 | pg_info->dma = 0; | |
403 | } | |
f21fb3ed RV |
404 | |
405 | recv_pkt->buffer_size[i] = | |
406 | (bytes_left >= | |
407 | droq->buffer_size) ? droq->buffer_size : bytes_left; | |
408 | ||
409 | recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; | |
410 | droq->recv_buf_list[idx].buffer = NULL; | |
411 | ||
412 | INCR_INDEX_BY1(idx, droq->max_count); | |
413 | bytes_left -= droq->buffer_size; | |
414 | i++; | |
415 | buf_cnt--; | |
416 | } | |
417 | ||
418 | return recv_info; | |
419 | } | |
420 | ||
421 | /* If we were not able to refill all buffers, try to move around | |
422 | * the buffers that were not dispatched. | |
423 | */ | |
424 | static inline u32 | |
425 | octeon_droq_refill_pullup_descs(struct octeon_droq *droq, | |
426 | struct octeon_droq_desc *desc_ring) | |
427 | { | |
428 | u32 desc_refilled = 0; | |
429 | ||
430 | u32 refill_index = droq->refill_idx; | |
431 | ||
432 | while (refill_index != droq->read_idx) { | |
433 | if (droq->recv_buf_list[refill_index].buffer) { | |
434 | droq->recv_buf_list[droq->refill_idx].buffer = | |
435 | droq->recv_buf_list[refill_index].buffer; | |
436 | droq->recv_buf_list[droq->refill_idx].data = | |
437 | droq->recv_buf_list[refill_index].data; | |
438 | desc_ring[droq->refill_idx].buffer_ptr = | |
439 | desc_ring[refill_index].buffer_ptr; | |
440 | droq->recv_buf_list[refill_index].buffer = NULL; | |
441 | desc_ring[refill_index].buffer_ptr = 0; | |
442 | do { | |
443 | INCR_INDEX_BY1(droq->refill_idx, | |
444 | droq->max_count); | |
445 | desc_refilled++; | |
446 | droq->refill_count--; | |
447 | } while (droq->recv_buf_list[droq->refill_idx]. | |
448 | buffer); | |
449 | } | |
450 | INCR_INDEX_BY1(refill_index, droq->max_count); | |
451 | } /* while */ | |
452 | return desc_refilled; | |
453 | } | |
454 | ||
455 | /* octeon_droq_refill | |
456 | * Parameters: | |
457 | * droq - droq in which descriptors require new buffers. | |
458 | * Description: | |
459 | * Called during normal DROQ processing in interrupt mode or by the poll | |
460 | * thread to refill the descriptors from which buffers were dispatched | |
461 | * to upper layers. Attempts to allocate new buffers. If that fails, moves | |
462 | * up buffers (that were not dispatched) to form a contiguous ring. | |
463 | * Returns: | |
464 | * No of descriptors refilled. | |
465 | * Locks: | |
466 | * This routine is called with droq->lock held. | |
467 | */ | |
468 | static u32 | |
469 | octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) | |
470 | { | |
471 | struct octeon_droq_desc *desc_ring; | |
472 | void *buf = NULL; | |
473 | u8 *data; | |
474 | u32 desc_refilled = 0; | |
cabeb13b | 475 | struct octeon_skb_page_info *pg_info; |
f21fb3ed RV |
476 | |
477 | desc_ring = droq->desc_ring; | |
478 | ||
479 | while (droq->refill_count && (desc_refilled < droq->max_count)) { | |
480 | /* If a valid buffer exists (happens if there is no dispatch), | |
481 | * reuse | |
482 | * the buffer, else allocate. | |
483 | */ | |
484 | if (!droq->recv_buf_list[droq->refill_idx].buffer) { | |
cabeb13b RV |
485 | pg_info = |
486 | &droq->recv_buf_list[droq->refill_idx].pg_info; | |
487 | /* Either recycle the existing pages or go for | |
488 | * new page alloc | |
489 | */ | |
490 | if (pg_info->page) | |
491 | buf = recv_buffer_reuse(octeon_dev, pg_info); | |
492 | else | |
493 | buf = recv_buffer_alloc(octeon_dev, pg_info); | |
f21fb3ed RV |
494 | /* If a buffer could not be allocated, no point in |
495 | * continuing | |
496 | */ | |
cabeb13b RV |
497 | if (!buf) { |
498 | droq->stats.rx_alloc_failure++; | |
f21fb3ed | 499 | break; |
cabeb13b | 500 | } |
f21fb3ed RV |
501 | droq->recv_buf_list[droq->refill_idx].buffer = |
502 | buf; | |
503 | data = get_rbd(buf); | |
504 | } else { | |
505 | data = get_rbd(droq->recv_buf_list | |
506 | [droq->refill_idx].buffer); | |
507 | } | |
508 | ||
509 | droq->recv_buf_list[droq->refill_idx].data = data; | |
510 | ||
511 | desc_ring[droq->refill_idx].buffer_ptr = | |
cabeb13b RV |
512 | lio_map_ring(droq->recv_buf_list[droq-> |
513 | refill_idx].buffer); | |
f21fb3ed RV |
514 | /* Reset any previous values in the length field. */ |
515 | droq->info_list[droq->refill_idx].length = 0; | |
516 | ||
517 | INCR_INDEX_BY1(droq->refill_idx, droq->max_count); | |
518 | desc_refilled++; | |
519 | droq->refill_count--; | |
520 | } | |
521 | ||
522 | if (droq->refill_count) | |
523 | desc_refilled += | |
524 | octeon_droq_refill_pullup_descs(droq, desc_ring); | |
525 | ||
526 | /* if droq->refill_count | |
527 | * The refill count would not change in pass two. We only moved buffers | |
528 | * to close the gap in the ring, but we would still have the same no. of | |
529 | * buffers to refill. | |
530 | */ | |
531 | return desc_refilled; | |
532 | } | |
533 | ||
534 | static inline u32 | |
535 | octeon_droq_get_bufcount(u32 buf_size, u32 total_len) | |
536 | { | |
537 | u32 buf_cnt = 0; | |
538 | ||
539 | while (total_len > (buf_size * buf_cnt)) | |
540 | buf_cnt++; | |
541 | return buf_cnt; | |
542 | } | |
543 | ||
544 | static int | |
545 | octeon_droq_dispatch_pkt(struct octeon_device *oct, | |
546 | struct octeon_droq *droq, | |
547 | union octeon_rh *rh, | |
548 | struct octeon_droq_info *info) | |
549 | { | |
550 | u32 cnt; | |
551 | octeon_dispatch_fn_t disp_fn; | |
552 | struct octeon_recv_info *rinfo; | |
553 | ||
554 | cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); | |
555 | ||
556 | disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, | |
557 | (u16)rh->r.subcode); | |
558 | if (disp_fn) { | |
559 | rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); | |
560 | if (rinfo) { | |
561 | struct __dispatch *rdisp = rinfo->rsvd; | |
562 | ||
563 | rdisp->rinfo = rinfo; | |
564 | rdisp->disp_fn = disp_fn; | |
565 | rinfo->recv_pkt->rh = *rh; | |
566 | list_add_tail(&rdisp->list, | |
567 | &droq->dispatch_list); | |
568 | } else { | |
569 | droq->stats.dropped_nomem++; | |
570 | } | |
571 | } else { | |
a2c64b67 RV |
572 | dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", |
573 | (unsigned int)rh->r.opcode, | |
574 | (unsigned int)rh->r.subcode); | |
f21fb3ed | 575 | droq->stats.dropped_nodispatch++; |
9ded1a51 | 576 | } |
f21fb3ed RV |
577 | |
578 | return cnt; | |
579 | } | |
580 | ||
581 | static inline void octeon_droq_drop_packets(struct octeon_device *oct, | |
582 | struct octeon_droq *droq, | |
583 | u32 cnt) | |
584 | { | |
585 | u32 i = 0, buf_cnt; | |
586 | struct octeon_droq_info *info; | |
587 | ||
588 | for (i = 0; i < cnt; i++) { | |
589 | info = &droq->info_list[droq->read_idx]; | |
590 | octeon_swap_8B_data((u64 *)info, 2); | |
591 | ||
592 | if (info->length) { | |
593 | info->length -= OCT_RH_SIZE; | |
594 | droq->stats.bytes_received += info->length; | |
595 | buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, | |
596 | (u32)info->length); | |
597 | } else { | |
598 | dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); | |
599 | buf_cnt = 1; | |
600 | } | |
601 | ||
602 | INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); | |
603 | droq->refill_count += buf_cnt; | |
604 | } | |
605 | } | |
606 | ||
607 | static u32 | |
608 | octeon_droq_fast_process_packets(struct octeon_device *oct, | |
609 | struct octeon_droq *droq, | |
610 | u32 pkts_to_process) | |
611 | { | |
612 | struct octeon_droq_info *info; | |
613 | union octeon_rh *rh; | |
614 | u32 pkt, total_len = 0, pkt_count; | |
615 | ||
616 | pkt_count = pkts_to_process; | |
617 | ||
618 | for (pkt = 0; pkt < pkt_count; pkt++) { | |
619 | u32 pkt_len = 0; | |
620 | struct sk_buff *nicbuf = NULL; | |
cabeb13b RV |
621 | struct octeon_skb_page_info *pg_info; |
622 | void *buf; | |
f21fb3ed RV |
623 | |
624 | info = &droq->info_list[droq->read_idx]; | |
625 | octeon_swap_8B_data((u64 *)info, 2); | |
626 | ||
627 | if (!info->length) { | |
628 | dev_err(&oct->pci_dev->dev, | |
629 | "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", | |
630 | droq->q_no, droq->read_idx, pkt_count); | |
631 | print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, | |
632 | (u8 *)info, | |
633 | OCT_DROQ_INFO_SIZE); | |
634 | break; | |
635 | } | |
636 | ||
637 | /* Len of resp hdr in included in the received data len. */ | |
638 | info->length -= OCT_RH_SIZE; | |
639 | rh = &info->rh; | |
640 | ||
641 | total_len += (u32)info->length; | |
f21fb3ed RV |
642 | if (OPCODE_SLOW_PATH(rh)) { |
643 | u32 buf_cnt; | |
644 | ||
645 | buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); | |
646 | INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); | |
647 | droq->refill_count += buf_cnt; | |
648 | } else { | |
649 | if (info->length <= droq->buffer_size) { | |
f21fb3ed RV |
650 | pkt_len = (u32)info->length; |
651 | nicbuf = droq->recv_buf_list[ | |
652 | droq->read_idx].buffer; | |
cabeb13b RV |
653 | pg_info = &droq->recv_buf_list[ |
654 | droq->read_idx].pg_info; | |
655 | if (recv_buffer_recycle(oct, pg_info)) | |
656 | pg_info->page = NULL; | |
f21fb3ed RV |
657 | droq->recv_buf_list[droq->read_idx].buffer = |
658 | NULL; | |
a2c64b67 | 659 | |
f21fb3ed | 660 | INCR_INDEX_BY1(droq->read_idx, droq->max_count); |
f21fb3ed RV |
661 | droq->refill_count++; |
662 | } else { | |
cabeb13b | 663 | nicbuf = octeon_fast_packet_alloc((u32) |
f21fb3ed RV |
664 | info->length); |
665 | pkt_len = 0; | |
666 | /* nicbuf allocation can fail. We'll handle it | |
667 | * inside the loop. | |
668 | */ | |
669 | while (pkt_len < info->length) { | |
cabeb13b | 670 | int cpy_len, idx = droq->read_idx; |
f21fb3ed | 671 | |
cabeb13b RV |
672 | cpy_len = ((pkt_len + droq->buffer_size) |
673 | > info->length) ? | |
f21fb3ed RV |
674 | ((u32)info->length - pkt_len) : |
675 | droq->buffer_size; | |
676 | ||
677 | if (nicbuf) { | |
f21fb3ed RV |
678 | octeon_fast_packet_next(droq, |
679 | nicbuf, | |
680 | cpy_len, | |
cabeb13b RV |
681 | idx); |
682 | buf = droq->recv_buf_list[idx]. | |
683 | buffer; | |
684 | recv_buffer_fast_free(buf); | |
685 | droq->recv_buf_list[idx].buffer | |
686 | = NULL; | |
687 | } else { | |
688 | droq->stats.rx_alloc_failure++; | |
f21fb3ed RV |
689 | } |
690 | ||
691 | pkt_len += cpy_len; | |
692 | INCR_INDEX_BY1(droq->read_idx, | |
693 | droq->max_count); | |
694 | droq->refill_count++; | |
695 | } | |
696 | } | |
697 | ||
698 | if (nicbuf) { | |
cabeb13b | 699 | if (droq->ops.fptr) { |
f21fb3ed | 700 | droq->ops.fptr(oct->octeon_id, |
cabeb13b | 701 | nicbuf, pkt_len, |
0cece6c5 RV |
702 | rh, &droq->napi, |
703 | droq->ops.farg); | |
cabeb13b | 704 | } else { |
f21fb3ed | 705 | recv_buffer_free(nicbuf); |
cabeb13b | 706 | } |
f21fb3ed RV |
707 | } |
708 | } | |
709 | ||
710 | if (droq->refill_count >= droq->refill_threshold) { | |
711 | int desc_refilled = octeon_droq_refill(oct, droq); | |
712 | ||
713 | /* Flush the droq descriptor data to memory to be sure | |
cabeb13b RV |
714 | * that when we update the credits the data in memory |
715 | * is accurate. | |
716 | */ | |
f21fb3ed RV |
717 | wmb(); |
718 | writel((desc_refilled), droq->pkts_credit_reg); | |
719 | /* make sure mmio write completes */ | |
720 | mmiowb(); | |
721 | } | |
722 | ||
cabeb13b | 723 | } /* for (each packet)... */ |
f21fb3ed RV |
724 | |
725 | /* Increment refill_count by the number of buffers processed. */ | |
726 | droq->stats.pkts_received += pkt; | |
727 | droq->stats.bytes_received += total_len; | |
728 | ||
729 | if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { | |
730 | octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); | |
731 | ||
732 | droq->stats.dropped_toomany += (pkts_to_process - pkt); | |
733 | return pkts_to_process; | |
734 | } | |
735 | ||
736 | return pkt; | |
737 | } | |
738 | ||
739 | int | |
740 | octeon_droq_process_packets(struct octeon_device *oct, | |
741 | struct octeon_droq *droq, | |
742 | u32 budget) | |
743 | { | |
744 | u32 pkt_count = 0, pkts_processed = 0; | |
745 | struct list_head *tmp, *tmp2; | |
746 | ||
cd8b1eb4 RV |
747 | /* Grab the droq lock */ |
748 | spin_lock(&droq->lock); | |
749 | ||
750 | octeon_droq_check_hw_for_pkts(droq); | |
f21fb3ed | 751 | pkt_count = atomic_read(&droq->pkts_pending); |
cd8b1eb4 RV |
752 | |
753 | if (!pkt_count) { | |
754 | spin_unlock(&droq->lock); | |
f21fb3ed | 755 | return 0; |
cd8b1eb4 | 756 | } |
f21fb3ed RV |
757 | |
758 | if (pkt_count > budget) | |
759 | pkt_count = budget; | |
760 | ||
f21fb3ed RV |
761 | pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); |
762 | ||
763 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
764 | ||
765 | /* Release the spin lock */ | |
766 | spin_unlock(&droq->lock); | |
767 | ||
768 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
769 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
770 | ||
771 | list_del(tmp); | |
772 | rdisp->disp_fn(rdisp->rinfo, | |
773 | octeon_get_dispatch_arg | |
774 | (oct, | |
775 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
776 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
777 | } | |
778 | ||
779 | /* If there are packets pending. schedule tasklet again */ | |
780 | if (atomic_read(&droq->pkts_pending)) | |
781 | return 1; | |
782 | ||
783 | return 0; | |
784 | } | |
785 | ||
786 | /** | |
787 | * Utility function to poll for packets. check_hw_for_packets must be | |
788 | * called before calling this routine. | |
789 | */ | |
790 | ||
791 | static int | |
792 | octeon_droq_process_poll_pkts(struct octeon_device *oct, | |
793 | struct octeon_droq *droq, u32 budget) | |
794 | { | |
795 | struct list_head *tmp, *tmp2; | |
796 | u32 pkts_available = 0, pkts_processed = 0; | |
797 | u32 total_pkts_processed = 0; | |
798 | ||
799 | if (budget > droq->max_count) | |
800 | budget = droq->max_count; | |
801 | ||
802 | spin_lock(&droq->lock); | |
803 | ||
804 | while (total_pkts_processed < budget) { | |
cd8b1eb4 RV |
805 | octeon_droq_check_hw_for_pkts(droq); |
806 | ||
f21fb3ed RV |
807 | pkts_available = |
808 | CVM_MIN((budget - total_pkts_processed), | |
809 | (u32)(atomic_read(&droq->pkts_pending))); | |
810 | ||
811 | if (pkts_available == 0) | |
812 | break; | |
813 | ||
814 | pkts_processed = | |
815 | octeon_droq_fast_process_packets(oct, droq, | |
816 | pkts_available); | |
817 | ||
818 | atomic_sub(pkts_processed, &droq->pkts_pending); | |
819 | ||
820 | total_pkts_processed += pkts_processed; | |
f21fb3ed RV |
821 | } |
822 | ||
823 | spin_unlock(&droq->lock); | |
824 | ||
825 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { | |
826 | struct __dispatch *rdisp = (struct __dispatch *)tmp; | |
827 | ||
828 | list_del(tmp); | |
829 | rdisp->disp_fn(rdisp->rinfo, | |
830 | octeon_get_dispatch_arg | |
831 | (oct, | |
832 | (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, | |
833 | (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); | |
834 | } | |
835 | ||
836 | return total_pkts_processed; | |
837 | } | |
838 | ||
839 | int | |
840 | octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, | |
841 | u32 arg) | |
842 | { | |
843 | struct octeon_droq *droq; | |
f21fb3ed RV |
844 | |
845 | droq = oct->droq[q_no]; | |
846 | ||
847 | if (cmd == POLL_EVENT_PROCESS_PKTS) | |
848 | return octeon_droq_process_poll_pkts(oct, droq, arg); | |
849 | ||
850 | if (cmd == POLL_EVENT_PENDING_PKTS) { | |
851 | u32 pkt_cnt = atomic_read(&droq->pkts_pending); | |
852 | ||
853 | return octeon_droq_process_packets(oct, droq, pkt_cnt); | |
854 | } | |
855 | ||
856 | if (cmd == POLL_EVENT_ENABLE_INTR) { | |
857 | u32 value; | |
858 | unsigned long flags; | |
859 | ||
860 | /* Enable Pkt Interrupt */ | |
861 | switch (oct->chip_id) { | |
862 | case OCTEON_CN66XX: | |
863 | case OCTEON_CN68XX: { | |
864 | struct octeon_cn6xxx *cn6xxx = | |
865 | (struct octeon_cn6xxx *)oct->chip; | |
866 | spin_lock_irqsave | |
867 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
868 | value = | |
869 | octeon_read_csr(oct, | |
870 | CN6XXX_SLI_PKT_TIME_INT_ENB); | |
871 | value |= (1 << q_no); | |
872 | octeon_write_csr(oct, | |
873 | CN6XXX_SLI_PKT_TIME_INT_ENB, | |
874 | value); | |
875 | value = | |
876 | octeon_read_csr(oct, | |
877 | CN6XXX_SLI_PKT_CNT_INT_ENB); | |
878 | value |= (1 << q_no); | |
879 | octeon_write_csr(oct, | |
880 | CN6XXX_SLI_PKT_CNT_INT_ENB, | |
881 | value); | |
882 | ||
883 | /* don't bother flushing the enables */ | |
884 | ||
885 | spin_unlock_irqrestore | |
886 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); | |
887 | return 0; | |
888 | } | |
889 | break; | |
9ded1a51 RV |
890 | case OCTEON_CN23XX_PF_VID: { |
891 | lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); | |
892 | } | |
893 | break; | |
f21fb3ed | 894 | } |
f21fb3ed RV |
895 | return 0; |
896 | } | |
897 | ||
898 | dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); | |
899 | return -EINVAL; | |
900 | } | |
901 | ||
902 | int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, | |
903 | struct octeon_droq_ops *ops) | |
904 | { | |
905 | struct octeon_droq *droq; | |
906 | unsigned long flags; | |
907 | struct octeon_config *oct_cfg = NULL; | |
908 | ||
909 | oct_cfg = octeon_get_conf(oct); | |
910 | ||
911 | if (!oct_cfg) | |
912 | return -EINVAL; | |
913 | ||
914 | if (!(ops)) { | |
915 | dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", | |
916 | __func__); | |
917 | return -EINVAL; | |
918 | } | |
919 | ||
920 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
921 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
922 | __func__, q_no, (oct->num_oqs - 1)); | |
923 | return -EINVAL; | |
924 | } | |
925 | ||
926 | droq = oct->droq[q_no]; | |
927 | ||
928 | spin_lock_irqsave(&droq->lock, flags); | |
929 | ||
930 | memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); | |
931 | ||
932 | spin_unlock_irqrestore(&droq->lock, flags); | |
933 | ||
934 | return 0; | |
935 | } | |
936 | ||
937 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) | |
938 | { | |
939 | unsigned long flags; | |
940 | struct octeon_droq *droq; | |
941 | struct octeon_config *oct_cfg = NULL; | |
942 | ||
943 | oct_cfg = octeon_get_conf(oct); | |
944 | ||
945 | if (!oct_cfg) | |
946 | return -EINVAL; | |
947 | ||
948 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { | |
949 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", | |
950 | __func__, q_no, oct->num_oqs - 1); | |
951 | return -EINVAL; | |
952 | } | |
953 | ||
954 | droq = oct->droq[q_no]; | |
955 | ||
956 | if (!droq) { | |
957 | dev_info(&oct->pci_dev->dev, | |
958 | "Droq id (%d) not available.\n", q_no); | |
959 | return 0; | |
960 | } | |
961 | ||
962 | spin_lock_irqsave(&droq->lock, flags); | |
963 | ||
964 | droq->ops.fptr = NULL; | |
0cece6c5 | 965 | droq->ops.farg = NULL; |
f21fb3ed RV |
966 | droq->ops.drop_on_max = 0; |
967 | ||
968 | spin_unlock_irqrestore(&droq->lock, flags); | |
969 | ||
970 | return 0; | |
971 | } | |
972 | ||
973 | int octeon_create_droq(struct octeon_device *oct, | |
974 | u32 q_no, u32 num_descs, | |
975 | u32 desc_size, void *app_ctx) | |
976 | { | |
977 | struct octeon_droq *droq; | |
96ae48b7 | 978 | int numa_node = cpu_to_node(q_no % num_online_cpus()); |
f21fb3ed RV |
979 | |
980 | if (oct->droq[q_no]) { | |
981 | dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", | |
982 | q_no); | |
983 | return 1; | |
984 | } | |
985 | ||
986 | /* Allocate the DS for the new droq. */ | |
96ae48b7 RV |
987 | droq = vmalloc_node(sizeof(*droq), numa_node); |
988 | if (!droq) | |
989 | droq = vmalloc(sizeof(*droq)); | |
f21fb3ed RV |
990 | if (!droq) |
991 | goto create_droq_fail; | |
992 | memset(droq, 0, sizeof(struct octeon_droq)); | |
993 | ||
994 | /*Disable the pkt o/p for this Q */ | |
995 | octeon_set_droq_pkt_op(oct, q_no, 0); | |
996 | oct->droq[q_no] = droq; | |
997 | ||
998 | /* Initialize the Droq */ | |
999 | octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx); | |
1000 | ||
1001 | oct->num_oqs++; | |
1002 | ||
1003 | dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, | |
1004 | oct->num_oqs); | |
1005 | ||
1006 | /* Global Droq register settings */ | |
1007 | ||
1008 | /* As of now not required, as setting are done for all 32 Droqs at | |
1009 | * the same time. | |
1010 | */ | |
1011 | return 0; | |
1012 | ||
1013 | create_droq_fail: | |
1014 | octeon_delete_droq(oct, q_no); | |
08a965ec | 1015 | return -ENOMEM; |
f21fb3ed | 1016 | } |