Commit | Line | Data |
---|---|---|
63697980 CM |
1 | /* |
2 | * Copyright 2012 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <linux/io.h> | |
16 | #include <linux/atomic.h> | |
17 | #include <linux/module.h> | |
18 | #include <gxio/dma_queue.h> | |
19 | ||
20 | /* Wait for a memory read to complete. */ | |
21 | #define wait_for_value(val) \ | |
22 | __asm__ __volatile__("move %0, %0" :: "r"(val)) | |
23 | ||
24 | /* The index is in the low 16. */ | |
25 | #define DMA_QUEUE_INDEX_MASK ((1 << 16) - 1) | |
26 | ||
27 | /* | |
28 | * The hardware descriptor-ring type. | |
29 | * This matches the types used by mpipe (MPIPE_EDMA_POST_REGION_VAL_t) | |
30 | * and trio (TRIO_PUSH_DMA_REGION_VAL_t or TRIO_PULL_DMA_REGION_VAL_t). | |
31 | * See those types for more documentation on the individual fields. | |
32 | */ | |
33 | typedef union { | |
34 | struct { | |
35 | #ifndef __BIG_ENDIAN__ | |
36 | uint64_t ring_idx:16; | |
37 | uint64_t count:16; | |
38 | uint64_t gen:1; | |
39 | uint64_t __reserved:31; | |
40 | #else | |
41 | uint64_t __reserved:31; | |
42 | uint64_t gen:1; | |
43 | uint64_t count:16; | |
44 | uint64_t ring_idx:16; | |
45 | #endif | |
46 | }; | |
47 | uint64_t word; | |
48 | } __gxio_ring_t; | |
49 | ||
50 | void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue, | |
51 | void *post_region_addr, unsigned int num_entries) | |
52 | { | |
53 | /* | |
54 | * Limit 65536 entry rings to 65535 credits because we only have a | |
55 | * 16 bit completion counter. | |
56 | */ | |
57 | int64_t credits = (num_entries < 65536) ? num_entries : 65535; | |
58 | ||
59 | memset(dma_queue, 0, sizeof(*dma_queue)); | |
60 | ||
61 | dma_queue->post_region_addr = post_region_addr; | |
62 | dma_queue->hw_complete_count = 0; | |
63 | dma_queue->credits_and_next_index = credits << DMA_QUEUE_CREDIT_SHIFT; | |
64 | } | |
65 | ||
66 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_init); | |
67 | ||
68 | void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue) | |
69 | { | |
70 | __gxio_ring_t val; | |
71 | uint64_t count; | |
72 | uint64_t delta; | |
73 | uint64_t new_count; | |
74 | ||
75 | /* | |
76 | * Read the 64-bit completion count without touching the cache, so | |
77 | * we later avoid having to evict any sharers of this cache line | |
78 | * when we update it below. | |
79 | */ | |
80 | uint64_t orig_hw_complete_count = | |
81 | cmpxchg(&dma_queue->hw_complete_count, | |
82 | -1, -1); | |
83 | ||
84 | /* Make sure the load completes before we access the hardware. */ | |
85 | wait_for_value(orig_hw_complete_count); | |
86 | ||
87 | /* Read the 16-bit count of how many packets it has completed. */ | |
88 | val.word = __gxio_mmio_read(dma_queue->post_region_addr); | |
89 | count = val.count; | |
90 | ||
91 | /* | |
92 | * Calculate the number of completions since we last updated the | |
93 | * 64-bit counter. It's safe to ignore the high bits because the | |
94 | * maximum credit value is 65535. | |
95 | */ | |
96 | delta = (count - orig_hw_complete_count) & 0xffff; | |
97 | if (delta == 0) | |
98 | return; | |
99 | ||
100 | /* | |
101 | * Try to write back the count, advanced by delta. If we race with | |
102 | * another thread, this might fail, in which case we return | |
103 | * immediately on the assumption that some credits are (or at least | |
104 | * were) available. | |
105 | */ | |
106 | new_count = orig_hw_complete_count + delta; | |
107 | if (cmpxchg(&dma_queue->hw_complete_count, | |
108 | orig_hw_complete_count, | |
109 | new_count) != orig_hw_complete_count) | |
110 | return; | |
111 | ||
112 | /* | |
113 | * We succeeded in advancing the completion count; add back the | |
114 | * corresponding number of egress credits. | |
115 | */ | |
116 | __insn_fetchadd(&dma_queue->credits_and_next_index, | |
117 | (delta << DMA_QUEUE_CREDIT_SHIFT)); | |
118 | } | |
119 | ||
120 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_update_credits); | |
121 | ||
122 | /* | |
123 | * A separate 'blocked' method for put() so that backtraces and | |
124 | * profiles will clearly indicate that we're wasting time spinning on | |
125 | * egress availability rather than actually posting commands. | |
126 | */ | |
127 | int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue, | |
128 | int64_t modifier) | |
129 | { | |
130 | int backoff = 16; | |
131 | int64_t old; | |
132 | ||
133 | do { | |
134 | int i; | |
135 | /* Back off to avoid spamming memory networks. */ | |
136 | for (i = backoff; i > 0; i--) | |
137 | __insn_mfspr(SPR_PASS); | |
138 | ||
139 | /* Check credits again. */ | |
140 | __gxio_dma_queue_update_credits(dma_queue); | |
141 | old = __insn_fetchaddgez(&dma_queue->credits_and_next_index, | |
142 | modifier); | |
143 | ||
144 | /* Calculate bounded exponential backoff for next iteration. */ | |
145 | if (backoff < 256) | |
146 | backoff *= 2; | |
147 | } while (old + modifier < 0); | |
148 | ||
149 | return old; | |
150 | } | |
151 | ||
152 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_wait_for_credits); | |
153 | ||
154 | int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue, | |
155 | unsigned int num, int wait) | |
156 | { | |
157 | return __gxio_dma_queue_reserve(dma_queue, num, wait != 0, true); | |
158 | } | |
159 | ||
160 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_reserve_aux); | |
161 | ||
162 | int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue, | |
163 | int64_t completion_slot, int update) | |
164 | { | |
165 | if (update) { | |
166 | if (ACCESS_ONCE(dma_queue->hw_complete_count) > | |
167 | completion_slot) | |
168 | return 1; | |
169 | ||
170 | __gxio_dma_queue_update_credits(dma_queue); | |
171 | } | |
172 | ||
173 | return ACCESS_ONCE(dma_queue->hw_complete_count) > completion_slot; | |
174 | } | |
175 | ||
176 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_is_complete); |