Commit | Line | Data |
---|---|---|
ec8aa669 | 1 | /* |
a0497f9f | 2 | * Copyright (C) 2010-2013 Felix Fietkau <nbd@openwrt.org> |
ec8aa669 FF |
3 | * |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | #include <linux/netdevice.h> | |
9 | #include <linux/types.h> | |
10 | #include <linux/skbuff.h> | |
11 | #include <linux/debugfs.h> | |
12 | #include <linux/random.h> | |
9208247d | 13 | #include <linux/moduleparam.h> |
ec8aa669 FF |
14 | #include <linux/ieee80211.h> |
15 | #include <net/mac80211.h> | |
16 | #include "rate.h" | |
17 | #include "rc80211_minstrel.h" | |
18 | #include "rc80211_minstrel_ht.h" | |
19 | ||
d66c2582 | 20 | #define AVG_AMPDU_SIZE 16 |
ec8aa669 | 21 | #define AVG_PKT_SIZE 1200 |
ec8aa669 FF |
22 | |
23 | /* Number of bits for an average sized packet */ | |
d66c2582 | 24 | #define MCS_NBITS ((AVG_PKT_SIZE * AVG_AMPDU_SIZE) << 3) |
ec8aa669 FF |
25 | |
26 | /* Number of symbols for a packet with (bps) bits per symbol */ | |
00591cea | 27 | #define MCS_NSYMS(bps) DIV_ROUND_UP(MCS_NBITS, (bps)) |
ec8aa669 | 28 | |
ed97a13c | 29 | /* Transmission time (nanoseconds) for a packet containing (syms) symbols */ |
ec8aa669 FF |
30 | #define MCS_SYMBOL_TIME(sgi, syms) \ |
31 | (sgi ? \ | |
ed97a13c FF |
32 | ((syms) * 18000 + 4000) / 5 : /* syms * 3.6 us */ \ |
33 | ((syms) * 1000) << 2 /* syms * 4 us */ \ | |
ec8aa669 FF |
34 | ) |
35 | ||
36 | /* Transmit duration for the raw data part of an average sized packet */ | |
d66c2582 FF |
37 | #define MCS_DURATION(streams, sgi, bps) \ |
38 | (MCS_SYMBOL_TIME(sgi, MCS_NSYMS((streams) * (bps))) / AVG_AMPDU_SIZE) | |
ec8aa669 | 39 | |
8a0ee4fe KB |
40 | #define BW_20 0 |
41 | #define BW_40 1 | |
9208247d | 42 | #define BW_80 2 |
8a0ee4fe | 43 | |
a5f69d94 HS |
44 | /* |
45 | * Define group sort order: HT40 -> SGI -> #streams | |
46 | */ | |
47 | #define GROUP_IDX(_streams, _sgi, _ht40) \ | |
8a0ee4fe | 48 | MINSTREL_HT_GROUP_0 + \ |
a5f69d94 | 49 | MINSTREL_MAX_STREAMS * 2 * _ht40 + \ |
8a0ee4fe | 50 | MINSTREL_MAX_STREAMS * _sgi + \ |
a5f69d94 HS |
51 | _streams - 1 |
52 | ||
ec8aa669 | 53 | /* MCS rate information for an MCS group */ |
a5f69d94 HS |
54 | #define MCS_GROUP(_streams, _sgi, _ht40) \ |
55 | [GROUP_IDX(_streams, _sgi, _ht40)] = { \ | |
ec8aa669 FF |
56 | .streams = _streams, \ |
57 | .flags = \ | |
3ec373c4 | 58 | IEEE80211_TX_RC_MCS | \ |
ec8aa669 FF |
59 | (_sgi ? IEEE80211_TX_RC_SHORT_GI : 0) | \ |
60 | (_ht40 ? IEEE80211_TX_RC_40_MHZ_WIDTH : 0), \ | |
61 | .duration = { \ | |
62 | MCS_DURATION(_streams, _sgi, _ht40 ? 54 : 26), \ | |
63 | MCS_DURATION(_streams, _sgi, _ht40 ? 108 : 52), \ | |
64 | MCS_DURATION(_streams, _sgi, _ht40 ? 162 : 78), \ | |
65 | MCS_DURATION(_streams, _sgi, _ht40 ? 216 : 104), \ | |
66 | MCS_DURATION(_streams, _sgi, _ht40 ? 324 : 156), \ | |
67 | MCS_DURATION(_streams, _sgi, _ht40 ? 432 : 208), \ | |
68 | MCS_DURATION(_streams, _sgi, _ht40 ? 486 : 234), \ | |
69 | MCS_DURATION(_streams, _sgi, _ht40 ? 540 : 260) \ | |
70 | } \ | |
71 | } | |
72 | ||
9208247d KB |
73 | #define VHT_GROUP_IDX(_streams, _sgi, _bw) \ |
74 | (MINSTREL_VHT_GROUP_0 + \ | |
75 | MINSTREL_MAX_STREAMS * 2 * (_bw) + \ | |
76 | MINSTREL_MAX_STREAMS * (_sgi) + \ | |
77 | (_streams) - 1) | |
78 | ||
79 | #define BW2VBPS(_bw, r3, r2, r1) \ | |
80 | (_bw == BW_80 ? r3 : _bw == BW_40 ? r2 : r1) | |
81 | ||
82 | #define VHT_GROUP(_streams, _sgi, _bw) \ | |
83 | [VHT_GROUP_IDX(_streams, _sgi, _bw)] = { \ | |
84 | .streams = _streams, \ | |
85 | .flags = \ | |
86 | IEEE80211_TX_RC_VHT_MCS | \ | |
87 | (_sgi ? IEEE80211_TX_RC_SHORT_GI : 0) | \ | |
88 | (_bw == BW_80 ? IEEE80211_TX_RC_80_MHZ_WIDTH : \ | |
89 | _bw == BW_40 ? IEEE80211_TX_RC_40_MHZ_WIDTH : 0), \ | |
90 | .duration = { \ | |
91 | MCS_DURATION(_streams, _sgi, \ | |
92 | BW2VBPS(_bw, 117, 54, 26)), \ | |
93 | MCS_DURATION(_streams, _sgi, \ | |
94 | BW2VBPS(_bw, 234, 108, 52)), \ | |
95 | MCS_DURATION(_streams, _sgi, \ | |
96 | BW2VBPS(_bw, 351, 162, 78)), \ | |
97 | MCS_DURATION(_streams, _sgi, \ | |
98 | BW2VBPS(_bw, 468, 216, 104)), \ | |
99 | MCS_DURATION(_streams, _sgi, \ | |
100 | BW2VBPS(_bw, 702, 324, 156)), \ | |
101 | MCS_DURATION(_streams, _sgi, \ | |
102 | BW2VBPS(_bw, 936, 432, 208)), \ | |
103 | MCS_DURATION(_streams, _sgi, \ | |
104 | BW2VBPS(_bw, 1053, 486, 234)), \ | |
105 | MCS_DURATION(_streams, _sgi, \ | |
106 | BW2VBPS(_bw, 1170, 540, 260)), \ | |
107 | MCS_DURATION(_streams, _sgi, \ | |
108 | BW2VBPS(_bw, 1404, 648, 312)), \ | |
109 | MCS_DURATION(_streams, _sgi, \ | |
110 | BW2VBPS(_bw, 1560, 720, 346)) \ | |
111 | } \ | |
112 | } | |
113 | ||
a0497f9f | 114 | #define CCK_DURATION(_bitrate, _short, _len) \ |
ed97a13c | 115 | (1000 * (10 /* SIFS */ + \ |
f359d3fe | 116 | (_short ? 72 + 24 : 144 + 48) + \ |
ed97a13c | 117 | (8 * (_len + 4) * 10) / (_bitrate))) |
a0497f9f FF |
118 | |
119 | #define CCK_ACK_DURATION(_bitrate, _short) \ | |
120 | (CCK_DURATION((_bitrate > 10 ? 20 : 10), false, 60) + \ | |
121 | CCK_DURATION(_bitrate, _short, AVG_PKT_SIZE)) | |
122 | ||
123 | #define CCK_DURATION_LIST(_short) \ | |
124 | CCK_ACK_DURATION(10, _short), \ | |
125 | CCK_ACK_DURATION(20, _short), \ | |
126 | CCK_ACK_DURATION(55, _short), \ | |
127 | CCK_ACK_DURATION(110, _short) | |
128 | ||
8a0ee4fe KB |
129 | #define CCK_GROUP \ |
130 | [MINSTREL_CCK_GROUP] = { \ | |
131 | .streams = 0, \ | |
3ec373c4 | 132 | .flags = 0, \ |
8a0ee4fe KB |
133 | .duration = { \ |
134 | CCK_DURATION_LIST(false), \ | |
135 | CCK_DURATION_LIST(true) \ | |
136 | } \ | |
a0497f9f FF |
137 | } |
138 | ||
9208247d KB |
139 | #ifdef CONFIG_MAC80211_RC_MINSTREL_VHT |
140 | static bool minstrel_vht_only = true; | |
141 | module_param(minstrel_vht_only, bool, 0644); | |
142 | MODULE_PARM_DESC(minstrel_vht_only, | |
143 | "Use only VHT rates when VHT is supported by sta."); | |
144 | #endif | |
145 | ||
ec8aa669 FF |
146 | /* |
147 | * To enable sufficiently targeted rate sampling, MCS rates are divided into | |
148 | * groups, based on the number of streams and flags (HT40, SGI) that they | |
149 | * use. | |
a5f69d94 HS |
150 | * |
151 | * Sortorder has to be fixed for GROUP_IDX macro to be applicable: | |
8a0ee4fe | 152 | * BW -> SGI -> #streams |
ec8aa669 FF |
153 | */ |
154 | const struct mcs_group minstrel_mcs_groups[] = { | |
8a0ee4fe KB |
155 | MCS_GROUP(1, 0, BW_20), |
156 | MCS_GROUP(2, 0, BW_20), | |
ec8aa669 | 157 | #if MINSTREL_MAX_STREAMS >= 3 |
8a0ee4fe | 158 | MCS_GROUP(3, 0, BW_20), |
ec8aa669 FF |
159 | #endif |
160 | ||
8a0ee4fe KB |
161 | MCS_GROUP(1, 1, BW_20), |
162 | MCS_GROUP(2, 1, BW_20), | |
ec8aa669 | 163 | #if MINSTREL_MAX_STREAMS >= 3 |
8a0ee4fe | 164 | MCS_GROUP(3, 1, BW_20), |
ec8aa669 FF |
165 | #endif |
166 | ||
8a0ee4fe KB |
167 | MCS_GROUP(1, 0, BW_40), |
168 | MCS_GROUP(2, 0, BW_40), | |
ec8aa669 | 169 | #if MINSTREL_MAX_STREAMS >= 3 |
8a0ee4fe | 170 | MCS_GROUP(3, 0, BW_40), |
ec8aa669 FF |
171 | #endif |
172 | ||
8a0ee4fe KB |
173 | MCS_GROUP(1, 1, BW_40), |
174 | MCS_GROUP(2, 1, BW_40), | |
ec8aa669 | 175 | #if MINSTREL_MAX_STREAMS >= 3 |
8a0ee4fe | 176 | MCS_GROUP(3, 1, BW_40), |
ec8aa669 | 177 | #endif |
a0497f9f | 178 | |
9208247d KB |
179 | CCK_GROUP, |
180 | ||
181 | #ifdef CONFIG_MAC80211_RC_MINSTREL_VHT | |
182 | VHT_GROUP(1, 0, BW_20), | |
183 | VHT_GROUP(2, 0, BW_20), | |
184 | #if MINSTREL_MAX_STREAMS >= 3 | |
185 | VHT_GROUP(3, 0, BW_20), | |
186 | #endif | |
187 | ||
188 | VHT_GROUP(1, 1, BW_20), | |
189 | VHT_GROUP(2, 1, BW_20), | |
190 | #if MINSTREL_MAX_STREAMS >= 3 | |
191 | VHT_GROUP(3, 1, BW_20), | |
192 | #endif | |
ec8aa669 | 193 | |
9208247d KB |
194 | VHT_GROUP(1, 0, BW_40), |
195 | VHT_GROUP(2, 0, BW_40), | |
196 | #if MINSTREL_MAX_STREAMS >= 3 | |
197 | VHT_GROUP(3, 0, BW_40), | |
198 | #endif | |
199 | ||
200 | VHT_GROUP(1, 1, BW_40), | |
201 | VHT_GROUP(2, 1, BW_40), | |
202 | #if MINSTREL_MAX_STREAMS >= 3 | |
203 | VHT_GROUP(3, 1, BW_40), | |
204 | #endif | |
205 | ||
206 | VHT_GROUP(1, 0, BW_80), | |
207 | VHT_GROUP(2, 0, BW_80), | |
208 | #if MINSTREL_MAX_STREAMS >= 3 | |
209 | VHT_GROUP(3, 0, BW_80), | |
210 | #endif | |
211 | ||
212 | VHT_GROUP(1, 1, BW_80), | |
213 | VHT_GROUP(2, 1, BW_80), | |
214 | #if MINSTREL_MAX_STREAMS >= 3 | |
215 | VHT_GROUP(3, 1, BW_80), | |
216 | #endif | |
217 | #endif | |
218 | }; | |
a0497f9f | 219 | |
f6e1a73b | 220 | static u8 sample_table[SAMPLE_COLUMNS][MCS_GROUP_RATES] __read_mostly; |
ec8aa669 | 221 | |
a8566662 FF |
222 | static void |
223 | minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi); | |
224 | ||
9208247d KB |
225 | /* |
226 | * Some VHT MCSes are invalid (when Ndbps / Nes is not an integer) | |
227 | * e.g for MCS9@20MHzx1Nss: Ndbps=8x52*(5/6) Nes=1 | |
228 | * | |
229 | * Returns the valid mcs map for struct minstrel_mcs_group_data.supported | |
230 | */ | |
231 | static u16 | |
232 | minstrel_get_valid_vht_rates(int bw, int nss, __le16 mcs_map) | |
233 | { | |
234 | u16 mask = 0; | |
235 | ||
236 | if (bw == BW_20) { | |
237 | if (nss != 3 && nss != 6) | |
238 | mask = BIT(9); | |
239 | } else if (bw == BW_80) { | |
240 | if (nss == 3 || nss == 7) | |
241 | mask = BIT(6); | |
242 | else if (nss == 6) | |
243 | mask = BIT(9); | |
244 | } else { | |
245 | WARN_ON(bw != BW_40); | |
246 | } | |
247 | ||
248 | switch ((le16_to_cpu(mcs_map) >> (2 * (nss - 1))) & 3) { | |
249 | case IEEE80211_VHT_MCS_SUPPORT_0_7: | |
250 | mask |= 0x300; | |
251 | break; | |
252 | case IEEE80211_VHT_MCS_SUPPORT_0_8: | |
253 | mask |= 0x200; | |
254 | break; | |
255 | case IEEE80211_VHT_MCS_SUPPORT_0_9: | |
256 | break; | |
257 | default: | |
258 | mask = 0x3ff; | |
259 | } | |
260 | ||
261 | return 0x3ff & ~mask; | |
262 | } | |
263 | ||
ec8aa669 FF |
264 | /* |
265 | * Look up an MCS group index based on mac80211 rate information | |
266 | */ | |
267 | static int | |
268 | minstrel_ht_get_group_idx(struct ieee80211_tx_rate *rate) | |
269 | { | |
cc61d8df | 270 | return GROUP_IDX((rate->idx / 8) + 1, |
a5f69d94 HS |
271 | !!(rate->flags & IEEE80211_TX_RC_SHORT_GI), |
272 | !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)); | |
ec8aa669 FF |
273 | } |
274 | ||
9208247d KB |
275 | static int |
276 | minstrel_vht_get_group_idx(struct ieee80211_tx_rate *rate) | |
277 | { | |
278 | return VHT_GROUP_IDX(ieee80211_rate_get_vht_nss(rate), | |
279 | !!(rate->flags & IEEE80211_TX_RC_SHORT_GI), | |
280 | !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH) + | |
281 | 2*!!(rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)); | |
282 | } | |
283 | ||
a0497f9f FF |
284 | static struct minstrel_rate_stats * |
285 | minstrel_ht_get_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, | |
286 | struct ieee80211_tx_rate *rate) | |
287 | { | |
288 | int group, idx; | |
289 | ||
290 | if (rate->flags & IEEE80211_TX_RC_MCS) { | |
291 | group = minstrel_ht_get_group_idx(rate); | |
7a5e3fa2 | 292 | idx = rate->idx % 8; |
9208247d KB |
293 | } else if (rate->flags & IEEE80211_TX_RC_VHT_MCS) { |
294 | group = minstrel_vht_get_group_idx(rate); | |
295 | idx = ieee80211_rate_get_vht_mcs(rate); | |
a0497f9f FF |
296 | } else { |
297 | group = MINSTREL_CCK_GROUP; | |
298 | ||
299 | for (idx = 0; idx < ARRAY_SIZE(mp->cck_rates); idx++) | |
300 | if (rate->idx == mp->cck_rates[idx]) | |
301 | break; | |
302 | ||
303 | /* short preamble */ | |
304 | if (!(mi->groups[group].supported & BIT(idx))) | |
305 | idx += 4; | |
306 | } | |
307 | return &mi->groups[group].rates[idx]; | |
308 | } | |
309 | ||
ec8aa669 FF |
310 | static inline struct minstrel_rate_stats * |
311 | minstrel_get_ratestats(struct minstrel_ht_sta *mi, int index) | |
312 | { | |
313 | return &mi->groups[index / MCS_GROUP_RATES].rates[index % MCS_GROUP_RATES]; | |
314 | } | |
315 | ||
ec8aa669 FF |
316 | /* |
317 | * Calculate throughput based on the average A-MPDU length, taking into account | |
318 | * the expected number of retransmissions and their expected length | |
319 | */ | |
320 | static void | |
6048d763 | 321 | minstrel_ht_calc_tp(struct minstrel_ht_sta *mi, int group, int rate) |
ec8aa669 FF |
322 | { |
323 | struct minstrel_rate_stats *mr; | |
ed97a13c FF |
324 | unsigned int nsecs = 0; |
325 | unsigned int tp; | |
3e8b1eb2 | 326 | unsigned int prob; |
ec8aa669 FF |
327 | |
328 | mr = &mi->groups[group].rates[rate]; | |
3e8b1eb2 | 329 | prob = mr->probability; |
ec8aa669 | 330 | |
3e8b1eb2 | 331 | if (prob < MINSTREL_FRAC(1, 10)) { |
ec8aa669 FF |
332 | mr->cur_tp = 0; |
333 | return; | |
334 | } | |
335 | ||
3e8b1eb2 FF |
336 | /* |
337 | * For the throughput calculation, limit the probability value to 90% to | |
338 | * account for collision related packet error rate fluctuation | |
339 | */ | |
340 | if (prob > MINSTREL_FRAC(9, 10)) | |
341 | prob = MINSTREL_FRAC(9, 10); | |
342 | ||
a0497f9f | 343 | if (group != MINSTREL_CCK_GROUP) |
ed97a13c | 344 | nsecs = 1000 * mi->overhead / MINSTREL_TRUNC(mi->avg_ampdu_len); |
a0497f9f | 345 | |
ed97a13c | 346 | nsecs += minstrel_mcs_groups[group].duration[rate]; |
ed97a13c | 347 | |
00591cea JB |
348 | /* prob is scaled - see MINSTREL_FRAC above */ |
349 | tp = 1000000 * ((prob * 1000) / nsecs); | |
ed97a13c | 350 | mr->cur_tp = MINSTREL_TRUNC(tp); |
ec8aa669 FF |
351 | } |
352 | ||
5935839a TH |
353 | /* |
354 | * Find & sort topmost throughput rates | |
355 | * | |
356 | * If multiple rates provide equal throughput the sorting is based on their | |
357 | * current success probability. Higher success probability is preferred among | |
358 | * MCS groups, CCK rates do not provide aggregation and are therefore at last. | |
359 | */ | |
360 | static void | |
d4d141ca KB |
361 | minstrel_ht_sort_best_tp_rates(struct minstrel_ht_sta *mi, u16 index, |
362 | u16 *tp_list) | |
5935839a TH |
363 | { |
364 | int cur_group, cur_idx, cur_thr, cur_prob; | |
365 | int tmp_group, tmp_idx, tmp_thr, tmp_prob; | |
366 | int j = MAX_THR_RATES; | |
367 | ||
368 | cur_group = index / MCS_GROUP_RATES; | |
369 | cur_idx = index % MCS_GROUP_RATES; | |
370 | cur_thr = mi->groups[cur_group].rates[cur_idx].cur_tp; | |
371 | cur_prob = mi->groups[cur_group].rates[cur_idx].probability; | |
372 | ||
280ba51d | 373 | do { |
5935839a TH |
374 | tmp_group = tp_list[j - 1] / MCS_GROUP_RATES; |
375 | tmp_idx = tp_list[j - 1] % MCS_GROUP_RATES; | |
376 | tmp_thr = mi->groups[tmp_group].rates[tmp_idx].cur_tp; | |
377 | tmp_prob = mi->groups[tmp_group].rates[tmp_idx].probability; | |
280ba51d FF |
378 | if (cur_thr < tmp_thr || |
379 | (cur_thr == tmp_thr && cur_prob <= tmp_prob)) | |
380 | break; | |
381 | j--; | |
382 | } while (j > 0); | |
5935839a TH |
383 | |
384 | if (j < MAX_THR_RATES - 1) { | |
385 | memmove(&tp_list[j + 1], &tp_list[j], (sizeof(*tp_list) * | |
386 | (MAX_THR_RATES - (j + 1)))); | |
387 | } | |
388 | if (j < MAX_THR_RATES) | |
389 | tp_list[j] = index; | |
390 | } | |
391 | ||
392 | /* | |
393 | * Find and set the topmost probability rate per sta and per group | |
394 | */ | |
395 | static void | |
d4d141ca | 396 | minstrel_ht_set_best_prob_rate(struct minstrel_ht_sta *mi, u16 index) |
5935839a TH |
397 | { |
398 | struct minstrel_mcs_group_data *mg; | |
399 | struct minstrel_rate_stats *mr; | |
400 | int tmp_group, tmp_idx, tmp_tp, tmp_prob, max_tp_group; | |
401 | ||
402 | mg = &mi->groups[index / MCS_GROUP_RATES]; | |
403 | mr = &mg->rates[index % MCS_GROUP_RATES]; | |
404 | ||
405 | tmp_group = mi->max_prob_rate / MCS_GROUP_RATES; | |
406 | tmp_idx = mi->max_prob_rate % MCS_GROUP_RATES; | |
407 | tmp_tp = mi->groups[tmp_group].rates[tmp_idx].cur_tp; | |
408 | tmp_prob = mi->groups[tmp_group].rates[tmp_idx].probability; | |
409 | ||
410 | /* if max_tp_rate[0] is from MCS_GROUP max_prob_rate get selected from | |
411 | * MCS_GROUP as well as CCK_GROUP rates do not allow aggregation */ | |
412 | max_tp_group = mi->max_tp_rate[0] / MCS_GROUP_RATES; | |
413 | if((index / MCS_GROUP_RATES == MINSTREL_CCK_GROUP) && | |
414 | (max_tp_group != MINSTREL_CCK_GROUP)) | |
415 | return; | |
416 | ||
417 | if (mr->probability > MINSTREL_FRAC(75, 100)) { | |
418 | if (mr->cur_tp > tmp_tp) | |
419 | mi->max_prob_rate = index; | |
420 | if (mr->cur_tp > mg->rates[mg->max_group_prob_rate].cur_tp) | |
421 | mg->max_group_prob_rate = index; | |
422 | } else { | |
423 | if (mr->probability > tmp_prob) | |
424 | mi->max_prob_rate = index; | |
425 | if (mr->probability > mg->rates[mg->max_group_prob_rate].probability) | |
426 | mg->max_group_prob_rate = index; | |
427 | } | |
428 | } | |
429 | ||
430 | ||
431 | /* | |
432 | * Assign new rate set per sta and use CCK rates only if the fastest | |
433 | * rate (max_tp_rate[0]) is from CCK group. This prohibits such sorted | |
434 | * rate sets where MCS and CCK rates are mixed, because CCK rates can | |
435 | * not use aggregation. | |
436 | */ | |
437 | static void | |
438 | minstrel_ht_assign_best_tp_rates(struct minstrel_ht_sta *mi, | |
d4d141ca KB |
439 | u16 tmp_mcs_tp_rate[MAX_THR_RATES], |
440 | u16 tmp_cck_tp_rate[MAX_THR_RATES]) | |
5935839a TH |
441 | { |
442 | unsigned int tmp_group, tmp_idx, tmp_cck_tp, tmp_mcs_tp; | |
443 | int i; | |
444 | ||
445 | tmp_group = tmp_cck_tp_rate[0] / MCS_GROUP_RATES; | |
446 | tmp_idx = tmp_cck_tp_rate[0] % MCS_GROUP_RATES; | |
447 | tmp_cck_tp = mi->groups[tmp_group].rates[tmp_idx].cur_tp; | |
448 | ||
449 | tmp_group = tmp_mcs_tp_rate[0] / MCS_GROUP_RATES; | |
450 | tmp_idx = tmp_mcs_tp_rate[0] % MCS_GROUP_RATES; | |
451 | tmp_mcs_tp = mi->groups[tmp_group].rates[tmp_idx].cur_tp; | |
452 | ||
453 | if (tmp_cck_tp > tmp_mcs_tp) { | |
454 | for(i = 0; i < MAX_THR_RATES; i++) { | |
455 | minstrel_ht_sort_best_tp_rates(mi, tmp_cck_tp_rate[i], | |
456 | tmp_mcs_tp_rate); | |
457 | } | |
458 | } | |
459 | ||
460 | } | |
461 | ||
462 | /* | |
463 | * Try to increase robustness of max_prob rate by decrease number of | |
464 | * streams if possible. | |
465 | */ | |
466 | static inline void | |
467 | minstrel_ht_prob_rate_reduce_streams(struct minstrel_ht_sta *mi) | |
468 | { | |
469 | struct minstrel_mcs_group_data *mg; | |
470 | struct minstrel_rate_stats *mr; | |
471 | int tmp_max_streams, group; | |
472 | int tmp_tp = 0; | |
473 | ||
474 | tmp_max_streams = minstrel_mcs_groups[mi->max_tp_rate[0] / | |
475 | MCS_GROUP_RATES].streams; | |
476 | for (group = 0; group < ARRAY_SIZE(minstrel_mcs_groups); group++) { | |
477 | mg = &mi->groups[group]; | |
478 | if (!mg->supported || group == MINSTREL_CCK_GROUP) | |
479 | continue; | |
480 | mr = minstrel_get_ratestats(mi, mg->max_group_prob_rate); | |
481 | if (tmp_tp < mr->cur_tp && | |
482 | (minstrel_mcs_groups[group].streams < tmp_max_streams)) { | |
483 | mi->max_prob_rate = mg->max_group_prob_rate; | |
484 | tmp_tp = mr->cur_tp; | |
485 | } | |
486 | } | |
487 | } | |
488 | ||
ec8aa669 FF |
489 | /* |
490 | * Update rate statistics and select new primary rates | |
491 | * | |
492 | * Rules for rate selection: | |
493 | * - max_prob_rate must use only one stream, as a tradeoff between delivery | |
494 | * probability and throughput during strong fluctuations | |
5935839a | 495 | * - as long as the max prob rate has a probability of more than 75%, pick |
ec8aa669 FF |
496 | * higher throughput rates, even if the probablity is a bit lower |
497 | */ | |
498 | static void | |
499 | minstrel_ht_update_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) | |
500 | { | |
501 | struct minstrel_mcs_group_data *mg; | |
502 | struct minstrel_rate_stats *mr; | |
5935839a | 503 | int group, i, j; |
d4d141ca KB |
504 | u16 tmp_mcs_tp_rate[MAX_THR_RATES], tmp_group_tp_rate[MAX_THR_RATES]; |
505 | u16 tmp_cck_tp_rate[MAX_THR_RATES], index; | |
ec8aa669 FF |
506 | |
507 | if (mi->ampdu_packets > 0) { | |
508 | mi->avg_ampdu_len = minstrel_ewma(mi->avg_ampdu_len, | |
509 | MINSTREL_FRAC(mi->ampdu_len, mi->ampdu_packets), EWMA_LEVEL); | |
510 | mi->ampdu_len = 0; | |
511 | mi->ampdu_packets = 0; | |
512 | } | |
513 | ||
514 | mi->sample_slow = 0; | |
515 | mi->sample_count = 0; | |
ec8aa669 | 516 | |
5935839a TH |
517 | /* Initialize global rate indexes */ |
518 | for(j = 0; j < MAX_THR_RATES; j++){ | |
519 | tmp_mcs_tp_rate[j] = 0; | |
520 | tmp_cck_tp_rate[j] = 0; | |
521 | } | |
c2eb5b0f | 522 | |
5935839a TH |
523 | /* Find best rate sets within all MCS groups*/ |
524 | for (group = 0; group < ARRAY_SIZE(minstrel_mcs_groups); group++) { | |
ec8aa669 FF |
525 | |
526 | mg = &mi->groups[group]; | |
527 | if (!mg->supported) | |
528 | continue; | |
529 | ||
ec8aa669 FF |
530 | mi->sample_count++; |
531 | ||
5935839a TH |
532 | /* (re)Initialize group rate indexes */ |
533 | for(j = 0; j < MAX_THR_RATES; j++) | |
534 | tmp_group_tp_rate[j] = group; | |
535 | ||
ec8aa669 FF |
536 | for (i = 0; i < MCS_GROUP_RATES; i++) { |
537 | if (!(mg->supported & BIT(i))) | |
538 | continue; | |
539 | ||
351df099 KB |
540 | index = MCS_GROUP_RATES * group + i; |
541 | ||
ec8aa669 FF |
542 | mr = &mg->rates[i]; |
543 | mr->retry_updated = false; | |
f62838bc | 544 | minstrel_calc_rate_stats(mr); |
6048d763 | 545 | minstrel_ht_calc_tp(mi, group, i); |
ec8aa669 FF |
546 | |
547 | if (!mr->cur_tp) | |
548 | continue; | |
549 | ||
5935839a TH |
550 | /* Find max throughput rate set */ |
551 | if (group != MINSTREL_CCK_GROUP) { | |
552 | minstrel_ht_sort_best_tp_rates(mi, index, | |
553 | tmp_mcs_tp_rate); | |
554 | } else if (group == MINSTREL_CCK_GROUP) { | |
555 | minstrel_ht_sort_best_tp_rates(mi, index, | |
556 | tmp_cck_tp_rate); | |
ec8aa669 | 557 | } |
ec8aa669 | 558 | |
5935839a TH |
559 | /* Find max throughput rate set within a group */ |
560 | minstrel_ht_sort_best_tp_rates(mi, index, | |
561 | tmp_group_tp_rate); | |
ec8aa669 | 562 | |
5935839a TH |
563 | /* Find max probability rate per group and global */ |
564 | minstrel_ht_set_best_prob_rate(mi, index); | |
ec8aa669 FF |
565 | } |
566 | ||
5935839a TH |
567 | memcpy(mg->max_group_tp_rate, tmp_group_tp_rate, |
568 | sizeof(mg->max_group_tp_rate)); | |
ec8aa669 FF |
569 | } |
570 | ||
5935839a TH |
571 | /* Assign new rate set per sta */ |
572 | minstrel_ht_assign_best_tp_rates(mi, tmp_mcs_tp_rate, tmp_cck_tp_rate); | |
573 | memcpy(mi->max_tp_rate, tmp_mcs_tp_rate, sizeof(mi->max_tp_rate)); | |
a299c6d5 | 574 | |
5935839a TH |
575 | /* Try to increase robustness of max_prob_rate*/ |
576 | minstrel_ht_prob_rate_reduce_streams(mi); | |
577 | ||
578 | /* try to sample all available rates during each interval */ | |
579 | mi->sample_count *= 8; | |
a299c6d5 | 580 | |
37feb7e2 LB |
581 | #ifdef CONFIG_MAC80211_DEBUGFS |
582 | /* use fixed index if set */ | |
583 | if (mp->fixed_rate_idx != -1) { | |
5935839a TH |
584 | for (i = 0; i < 4; i++) |
585 | mi->max_tp_rate[i] = mp->fixed_rate_idx; | |
37feb7e2 LB |
586 | mi->max_prob_rate = mp->fixed_rate_idx; |
587 | } | |
588 | #endif | |
a299c6d5 | 589 | |
5935839a | 590 | /* Reset update timer */ |
ec8aa669 FF |
591 | mi->stats_update = jiffies; |
592 | } | |
593 | ||
594 | static bool | |
a0497f9f | 595 | minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct ieee80211_tx_rate *rate) |
ec8aa669 | 596 | { |
b79296be | 597 | if (rate->idx < 0) |
ec8aa669 FF |
598 | return false; |
599 | ||
b79296be | 600 | if (!rate->count) |
ec8aa669 FF |
601 | return false; |
602 | ||
9208247d KB |
603 | if (rate->flags & IEEE80211_TX_RC_MCS || |
604 | rate->flags & IEEE80211_TX_RC_VHT_MCS) | |
a0497f9f FF |
605 | return true; |
606 | ||
607 | return rate->idx == mp->cck_rates[0] || | |
608 | rate->idx == mp->cck_rates[1] || | |
609 | rate->idx == mp->cck_rates[2] || | |
610 | rate->idx == mp->cck_rates[3]; | |
ec8aa669 FF |
611 | } |
612 | ||
613 | static void | |
614 | minstrel_next_sample_idx(struct minstrel_ht_sta *mi) | |
615 | { | |
616 | struct minstrel_mcs_group_data *mg; | |
617 | ||
618 | for (;;) { | |
619 | mi->sample_group++; | |
620 | mi->sample_group %= ARRAY_SIZE(minstrel_mcs_groups); | |
621 | mg = &mi->groups[mi->sample_group]; | |
622 | ||
623 | if (!mg->supported) | |
624 | continue; | |
625 | ||
626 | if (++mg->index >= MCS_GROUP_RATES) { | |
627 | mg->index = 0; | |
628 | if (++mg->column >= ARRAY_SIZE(sample_table)) | |
629 | mg->column = 0; | |
630 | } | |
631 | break; | |
632 | } | |
633 | } | |
634 | ||
635 | static void | |
d4d141ca | 636 | minstrel_downgrade_rate(struct minstrel_ht_sta *mi, u16 *idx, bool primary) |
ec8aa669 FF |
637 | { |
638 | int group, orig_group; | |
639 | ||
640 | orig_group = group = *idx / MCS_GROUP_RATES; | |
641 | while (group > 0) { | |
642 | group--; | |
643 | ||
644 | if (!mi->groups[group].supported) | |
645 | continue; | |
646 | ||
647 | if (minstrel_mcs_groups[group].streams > | |
648 | minstrel_mcs_groups[orig_group].streams) | |
649 | continue; | |
650 | ||
651 | if (primary) | |
5935839a | 652 | *idx = mi->groups[group].max_group_tp_rate[0]; |
ec8aa669 | 653 | else |
5935839a | 654 | *idx = mi->groups[group].max_group_tp_rate[1]; |
ec8aa669 FF |
655 | break; |
656 | } | |
657 | } | |
658 | ||
659 | static void | |
6048d763 | 660 | minstrel_aggr_check(struct ieee80211_sta *pubsta, struct sk_buff *skb) |
ec8aa669 FF |
661 | { |
662 | struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data; | |
663 | struct sta_info *sta = container_of(pubsta, struct sta_info, sta); | |
664 | u16 tid; | |
665 | ||
75769c80 FF |
666 | if (skb_get_queue_mapping(skb) == IEEE80211_AC_VO) |
667 | return; | |
668 | ||
ec8aa669 FF |
669 | if (unlikely(!ieee80211_is_data_qos(hdr->frame_control))) |
670 | return; | |
671 | ||
e133fae2 | 672 | if (unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE))) |
ec8aa669 FF |
673 | return; |
674 | ||
675 | tid = *ieee80211_get_qos_ctl(hdr) & IEEE80211_QOS_CTL_TID_MASK; | |
a622ab72 | 676 | if (likely(sta->ampdu_mlme.tid_tx[tid])) |
ec8aa669 FF |
677 | return; |
678 | ||
bd2ce6e4 | 679 | ieee80211_start_tx_ba_session(pubsta, tid, 5000); |
ec8aa669 FF |
680 | } |
681 | ||
682 | static void | |
683 | minstrel_ht_tx_status(void *priv, struct ieee80211_supported_band *sband, | |
684 | struct ieee80211_sta *sta, void *priv_sta, | |
63558a65 | 685 | struct ieee80211_tx_info *info) |
ec8aa669 FF |
686 | { |
687 | struct minstrel_ht_sta_priv *msp = priv_sta; | |
688 | struct minstrel_ht_sta *mi = &msp->ht; | |
ec8aa669 FF |
689 | struct ieee80211_tx_rate *ar = info->status.rates; |
690 | struct minstrel_rate_stats *rate, *rate2; | |
691 | struct minstrel_priv *mp = priv; | |
a8566662 | 692 | bool last, update = false; |
a544ab7f | 693 | int i; |
ec8aa669 FF |
694 | |
695 | if (!msp->is_ht) | |
63558a65 FF |
696 | return mac80211_minstrel.tx_status_noskb(priv, sband, sta, |
697 | &msp->legacy, info); | |
ec8aa669 FF |
698 | |
699 | /* This packet was aggregated but doesn't carry status info */ | |
700 | if ((info->flags & IEEE80211_TX_CTL_AMPDU) && | |
701 | !(info->flags & IEEE80211_TX_STAT_AMPDU)) | |
702 | return; | |
703 | ||
15d46f38 BS |
704 | if (!(info->flags & IEEE80211_TX_STAT_AMPDU)) { |
705 | info->status.ampdu_ack_len = | |
706 | (info->flags & IEEE80211_TX_STAT_ACK ? 1 : 0); | |
ec8aa669 FF |
707 | info->status.ampdu_len = 1; |
708 | } | |
709 | ||
710 | mi->ampdu_packets++; | |
711 | mi->ampdu_len += info->status.ampdu_len; | |
712 | ||
713 | if (!mi->sample_wait && !mi->sample_tries && mi->sample_count > 0) { | |
c7317e41 | 714 | mi->sample_wait = 16 + 2 * MINSTREL_TRUNC(mi->avg_ampdu_len); |
52c00a37 | 715 | mi->sample_tries = 1; |
ec8aa669 FF |
716 | mi->sample_count--; |
717 | } | |
718 | ||
8d5eab5a | 719 | if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE) |
ec8aa669 | 720 | mi->sample_packets += info->status.ampdu_len; |
ec8aa669 | 721 | |
a0497f9f | 722 | last = !minstrel_ht_txstat_valid(mp, &ar[0]); |
ec8aa669 FF |
723 | for (i = 0; !last; i++) { |
724 | last = (i == IEEE80211_TX_MAX_RATES - 1) || | |
a0497f9f | 725 | !minstrel_ht_txstat_valid(mp, &ar[i + 1]); |
ec8aa669 | 726 | |
a0497f9f | 727 | rate = minstrel_ht_get_stats(mp, mi, &ar[i]); |
ec8aa669 | 728 | |
15d46f38 | 729 | if (last) |
ec8aa669 FF |
730 | rate->success += info->status.ampdu_ack_len; |
731 | ||
732 | rate->attempts += ar[i].count * info->status.ampdu_len; | |
733 | } | |
734 | ||
735 | /* | |
736 | * check for sudden death of spatial multiplexing, | |
737 | * downgrade to a lower number of streams if necessary. | |
738 | */ | |
5935839a | 739 | rate = minstrel_get_ratestats(mi, mi->max_tp_rate[0]); |
ec8aa669 FF |
740 | if (rate->attempts > 30 && |
741 | MINSTREL_FRAC(rate->success, rate->attempts) < | |
a8566662 | 742 | MINSTREL_FRAC(20, 100)) { |
5935839a | 743 | minstrel_downgrade_rate(mi, &mi->max_tp_rate[0], true); |
a8566662 FF |
744 | update = true; |
745 | } | |
ec8aa669 | 746 | |
5935839a | 747 | rate2 = minstrel_get_ratestats(mi, mi->max_tp_rate[1]); |
ecc3d5ae ML |
748 | if (rate2->attempts > 30 && |
749 | MINSTREL_FRAC(rate2->success, rate2->attempts) < | |
a8566662 | 750 | MINSTREL_FRAC(20, 100)) { |
5935839a | 751 | minstrel_downgrade_rate(mi, &mi->max_tp_rate[1], false); |
a8566662 FF |
752 | update = true; |
753 | } | |
ec8aa669 FF |
754 | |
755 | if (time_after(jiffies, mi->stats_update + (mp->update_interval / 2 * HZ) / 1000)) { | |
a8566662 | 756 | update = true; |
ec8aa669 | 757 | minstrel_ht_update_stats(mp, mi); |
ec8aa669 | 758 | } |
a8566662 FF |
759 | |
760 | if (update) | |
761 | minstrel_ht_update_rates(mp, mi); | |
ec8aa669 FF |
762 | } |
763 | ||
764 | static void | |
765 | minstrel_calc_retransmit(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, | |
766 | int index) | |
767 | { | |
768 | struct minstrel_rate_stats *mr; | |
769 | const struct mcs_group *group; | |
770 | unsigned int tx_time, tx_time_rtscts, tx_time_data; | |
771 | unsigned int cw = mp->cw_min; | |
8fddddff | 772 | unsigned int ctime = 0; |
ec8aa669 FF |
773 | unsigned int t_slot = 9; /* FIXME */ |
774 | unsigned int ampdu_len = MINSTREL_TRUNC(mi->avg_ampdu_len); | |
a0497f9f | 775 | unsigned int overhead = 0, overhead_rtscts = 0; |
ec8aa669 FF |
776 | |
777 | mr = minstrel_get_ratestats(mi, index); | |
778 | if (mr->probability < MINSTREL_FRAC(1, 10)) { | |
779 | mr->retry_count = 1; | |
780 | mr->retry_count_rtscts = 1; | |
781 | return; | |
782 | } | |
783 | ||
784 | mr->retry_count = 2; | |
785 | mr->retry_count_rtscts = 2; | |
786 | mr->retry_updated = true; | |
787 | ||
788 | group = &minstrel_mcs_groups[index / MCS_GROUP_RATES]; | |
ed97a13c | 789 | tx_time_data = group->duration[index % MCS_GROUP_RATES] * ampdu_len / 1000; |
8fddddff DH |
790 | |
791 | /* Contention time for first 2 tries */ | |
792 | ctime = (t_slot * cw) >> 1; | |
793 | cw = min((cw << 1) | 1, mp->cw_max); | |
794 | ctime += (t_slot * cw) >> 1; | |
795 | cw = min((cw << 1) | 1, mp->cw_max); | |
796 | ||
a0497f9f FF |
797 | if (index / MCS_GROUP_RATES != MINSTREL_CCK_GROUP) { |
798 | overhead = mi->overhead; | |
799 | overhead_rtscts = mi->overhead_rtscts; | |
800 | } | |
801 | ||
8fddddff | 802 | /* Total TX time for data and Contention after first 2 tries */ |
a0497f9f FF |
803 | tx_time = ctime + 2 * (overhead + tx_time_data); |
804 | tx_time_rtscts = ctime + 2 * (overhead_rtscts + tx_time_data); | |
8fddddff DH |
805 | |
806 | /* See how many more tries we can fit inside segment size */ | |
ec8aa669 | 807 | do { |
8fddddff DH |
808 | /* Contention time for this try */ |
809 | ctime = (t_slot * cw) >> 1; | |
810 | cw = min((cw << 1) | 1, mp->cw_max); | |
811 | ||
812 | /* Total TX time after this try */ | |
a0497f9f FF |
813 | tx_time += ctime + overhead + tx_time_data; |
814 | tx_time_rtscts += ctime + overhead_rtscts + tx_time_data; | |
8fddddff | 815 | |
ec8aa669 FF |
816 | if (tx_time_rtscts < mp->segment_size) |
817 | mr->retry_count_rtscts++; | |
818 | } while ((tx_time < mp->segment_size) && | |
819 | (++mr->retry_count < mp->max_retry)); | |
820 | } | |
821 | ||
822 | ||
823 | static void | |
824 | minstrel_ht_set_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, | |
a8566662 | 825 | struct ieee80211_sta_rates *ratetbl, int offset, int index) |
ec8aa669 FF |
826 | { |
827 | const struct mcs_group *group = &minstrel_mcs_groups[index / MCS_GROUP_RATES]; | |
828 | struct minstrel_rate_stats *mr; | |
a8566662 | 829 | u8 idx; |
3ec373c4 | 830 | u16 flags = group->flags; |
ec8aa669 FF |
831 | |
832 | mr = minstrel_get_ratestats(mi, index); | |
833 | if (!mr->retry_updated) | |
834 | minstrel_calc_retransmit(mp, mi, index); | |
835 | ||
a8566662 FF |
836 | if (mr->probability < MINSTREL_FRAC(20, 100) || !mr->retry_count) { |
837 | ratetbl->rate[offset].count = 2; | |
838 | ratetbl->rate[offset].count_rts = 2; | |
839 | ratetbl->rate[offset].count_cts = 2; | |
840 | } else { | |
841 | ratetbl->rate[offset].count = mr->retry_count; | |
842 | ratetbl->rate[offset].count_cts = mr->retry_count; | |
843 | ratetbl->rate[offset].count_rts = mr->retry_count_rtscts; | |
844 | } | |
a0497f9f | 845 | |
3ec373c4 | 846 | if (index / MCS_GROUP_RATES == MINSTREL_CCK_GROUP) |
a8566662 | 847 | idx = mp->cck_rates[index % ARRAY_SIZE(mp->cck_rates)]; |
9208247d KB |
848 | else if (flags & IEEE80211_TX_RC_VHT_MCS) |
849 | idx = ((group->streams - 1) << 4) | | |
850 | ((index % MCS_GROUP_RATES) & 0xF); | |
3ec373c4 | 851 | else |
7a5e3fa2 | 852 | idx = index % MCS_GROUP_RATES + (group->streams - 1) * 8; |
a8566662 FF |
853 | |
854 | if (offset > 0) { | |
855 | ratetbl->rate[offset].count = ratetbl->rate[offset].count_rts; | |
856 | flags |= IEEE80211_TX_RC_USE_RTS_CTS; | |
857 | } | |
858 | ||
859 | ratetbl->rate[offset].idx = idx; | |
860 | ratetbl->rate[offset].flags = flags; | |
861 | } | |
862 | ||
863 | static void | |
864 | minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) | |
865 | { | |
866 | struct ieee80211_sta_rates *rates; | |
867 | int i = 0; | |
868 | ||
869 | rates = kzalloc(sizeof(*rates), GFP_ATOMIC); | |
870 | if (!rates) | |
a0497f9f | 871 | return; |
a8566662 | 872 | |
5935839a TH |
873 | /* Start with max_tp_rate[0] */ |
874 | minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_tp_rate[0]); | |
a8566662 FF |
875 | |
876 | if (mp->hw->max_rates >= 3) { | |
5935839a TH |
877 | /* At least 3 tx rates supported, use max_tp_rate[1] next */ |
878 | minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_tp_rate[1]); | |
a8566662 FF |
879 | } |
880 | ||
881 | if (mp->hw->max_rates >= 2) { | |
882 | /* | |
883 | * At least 2 tx rates supported, use max_prob_rate next */ | |
884 | minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_prob_rate); | |
a0497f9f FF |
885 | } |
886 | ||
a8566662 FF |
887 | rates->rate[i].idx = -1; |
888 | rate_control_set_rates(mp->hw, mi->sta, rates); | |
ec8aa669 FF |
889 | } |
890 | ||
891 | static inline int | |
892 | minstrel_get_duration(int index) | |
893 | { | |
894 | const struct mcs_group *group = &minstrel_mcs_groups[index / MCS_GROUP_RATES]; | |
895 | return group->duration[index % MCS_GROUP_RATES]; | |
896 | } | |
897 | ||
898 | static int | |
899 | minstrel_get_sample_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) | |
900 | { | |
901 | struct minstrel_rate_stats *mr; | |
902 | struct minstrel_mcs_group_data *mg; | |
5935839a | 903 | unsigned int sample_dur, sample_group, cur_max_tp_streams; |
ec8aa669 FF |
904 | int sample_idx = 0; |
905 | ||
906 | if (mi->sample_wait > 0) { | |
907 | mi->sample_wait--; | |
908 | return -1; | |
909 | } | |
910 | ||
911 | if (!mi->sample_tries) | |
912 | return -1; | |
913 | ||
f12140c0 KB |
914 | sample_group = mi->sample_group; |
915 | mg = &mi->groups[sample_group]; | |
ec8aa669 | 916 | sample_idx = sample_table[mg->column][mg->index]; |
f12140c0 KB |
917 | minstrel_next_sample_idx(mi); |
918 | ||
919 | if (!(mg->supported & BIT(sample_idx))) | |
920 | return -1; | |
921 | ||
ec8aa669 | 922 | mr = &mg->rates[sample_idx]; |
965237ab | 923 | sample_idx += sample_group * MCS_GROUP_RATES; |
ec8aa669 | 924 | |
ba6fa29c HS |
925 | /* |
926 | * Sampling might add some overhead (RTS, no aggregation) | |
927 | * to the frame. Hence, don't use sampling for the currently | |
a0ca796c | 928 | * used rates. |
ba6fa29c | 929 | */ |
5935839a TH |
930 | if (sample_idx == mi->max_tp_rate[0] || |
931 | sample_idx == mi->max_tp_rate[1] || | |
a0ca796c | 932 | sample_idx == mi->max_prob_rate) |
ba6fa29c | 933 | return -1; |
a0ca796c | 934 | |
ec8aa669 | 935 | /* |
bc96f242 FF |
936 | * Do not sample if the probability is already higher than 95% |
937 | * to avoid wasting airtime. | |
ec8aa669 | 938 | */ |
bc96f242 | 939 | if (mr->probability > MINSTREL_FRAC(95, 100)) |
8d5eab5a | 940 | return -1; |
ec8aa669 FF |
941 | |
942 | /* | |
943 | * Make sure that lower rates get sampled only occasionally, | |
944 | * if the link is working perfectly. | |
945 | */ | |
5935839a TH |
946 | |
947 | cur_max_tp_streams = minstrel_mcs_groups[mi->max_tp_rate[0] / | |
948 | MCS_GROUP_RATES].streams; | |
965237ab | 949 | sample_dur = minstrel_get_duration(sample_idx); |
5935839a TH |
950 | if (sample_dur >= minstrel_get_duration(mi->max_tp_rate[1]) && |
951 | (cur_max_tp_streams - 1 < | |
965237ab FF |
952 | minstrel_mcs_groups[sample_group].streams || |
953 | sample_dur >= minstrel_get_duration(mi->max_prob_rate))) { | |
c7317e41 | 954 | if (mr->sample_skipped < 20) |
8d5eab5a | 955 | return -1; |
ec8aa669 FF |
956 | |
957 | if (mi->sample_slow++ > 2) | |
8d5eab5a | 958 | return -1; |
ec8aa669 | 959 | } |
098b8afb | 960 | mi->sample_tries--; |
ec8aa669 FF |
961 | |
962 | return sample_idx; | |
ec8aa669 FF |
963 | } |
964 | ||
a0497f9f FF |
965 | static void |
966 | minstrel_ht_check_cck_shortpreamble(struct minstrel_priv *mp, | |
967 | struct minstrel_ht_sta *mi, bool val) | |
968 | { | |
969 | u8 supported = mi->groups[MINSTREL_CCK_GROUP].supported; | |
970 | ||
971 | if (!supported || !mi->cck_supported_short) | |
972 | return; | |
973 | ||
974 | if (supported & (mi->cck_supported_short << (val * 4))) | |
975 | return; | |
976 | ||
977 | supported ^= mi->cck_supported_short | (mi->cck_supported_short << 4); | |
978 | mi->groups[MINSTREL_CCK_GROUP].supported = supported; | |
979 | } | |
980 | ||
ec8aa669 FF |
981 | static void |
982 | minstrel_ht_get_rate(void *priv, struct ieee80211_sta *sta, void *priv_sta, | |
983 | struct ieee80211_tx_rate_control *txrc) | |
984 | { | |
a8566662 | 985 | const struct mcs_group *sample_group; |
ec8aa669 | 986 | struct ieee80211_tx_info *info = IEEE80211_SKB_CB(txrc->skb); |
a8566662 | 987 | struct ieee80211_tx_rate *rate = &info->status.rates[0]; |
ec8aa669 FF |
988 | struct minstrel_ht_sta_priv *msp = priv_sta; |
989 | struct minstrel_ht_sta *mi = &msp->ht; | |
990 | struct minstrel_priv *mp = priv; | |
991 | int sample_idx; | |
992 | ||
993 | if (rate_control_send_low(sta, priv_sta, txrc)) | |
994 | return; | |
995 | ||
996 | if (!msp->is_ht) | |
997 | return mac80211_minstrel.get_rate(priv, sta, &msp->legacy, txrc); | |
998 | ||
95943425 FF |
999 | if (!(info->flags & IEEE80211_TX_CTL_AMPDU) && |
1000 | mi->max_prob_rate / MCS_GROUP_RATES != MINSTREL_CCK_GROUP) | |
1001 | minstrel_aggr_check(sta, txrc->skb); | |
1002 | ||
ec8aa669 | 1003 | info->flags |= mi->tx_flags; |
a0497f9f | 1004 | minstrel_ht_check_cck_shortpreamble(mp, mi, txrc->short_preamble); |
6de062ce | 1005 | |
37feb7e2 LB |
1006 | #ifdef CONFIG_MAC80211_DEBUGFS |
1007 | if (mp->fixed_rate_idx != -1) | |
1008 | return; | |
1009 | #endif | |
1010 | ||
6de062ce HS |
1011 | /* Don't use EAPOL frames for sampling on non-mrr hw */ |
1012 | if (mp->hw->max_rates == 1 && | |
af61a165 | 1013 | (info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO)) |
6de062ce HS |
1014 | sample_idx = -1; |
1015 | else | |
1016 | sample_idx = minstrel_get_sample_rate(mp, mi); | |
24f7580e | 1017 | |
ec8aa669 FF |
1018 | mi->total_packets++; |
1019 | ||
1020 | /* wraparound */ | |
1021 | if (mi->total_packets == ~0) { | |
1022 | mi->total_packets = 0; | |
1023 | mi->sample_packets = 0; | |
1024 | } | |
a8566662 FF |
1025 | |
1026 | if (sample_idx < 0) | |
1027 | return; | |
1028 | ||
1029 | sample_group = &minstrel_mcs_groups[sample_idx / MCS_GROUP_RATES]; | |
1030 | info->flags |= IEEE80211_TX_CTL_RATE_CTRL_PROBE; | |
1cd15857 FF |
1031 | rate->count = 1; |
1032 | ||
1033 | if (sample_idx / MCS_GROUP_RATES == MINSTREL_CCK_GROUP) { | |
1034 | int idx = sample_idx % ARRAY_SIZE(mp->cck_rates); | |
1035 | rate->idx = mp->cck_rates[idx]; | |
9208247d KB |
1036 | } else if (sample_group->flags & IEEE80211_TX_RC_VHT_MCS) { |
1037 | ieee80211_rate_set_vht(rate, sample_idx % MCS_GROUP_RATES, | |
1038 | sample_group->streams); | |
3ec373c4 KB |
1039 | } else { |
1040 | rate->idx = sample_idx % MCS_GROUP_RATES + | |
1041 | (sample_group->streams - 1) * 8; | |
1cd15857 FF |
1042 | } |
1043 | ||
3ec373c4 | 1044 | rate->flags = sample_group->flags; |
ec8aa669 FF |
1045 | } |
1046 | ||
a0497f9f FF |
1047 | static void |
1048 | minstrel_ht_update_cck(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, | |
1049 | struct ieee80211_supported_band *sband, | |
1050 | struct ieee80211_sta *sta) | |
1051 | { | |
1052 | int i; | |
1053 | ||
1054 | if (sband->band != IEEE80211_BAND_2GHZ) | |
1055 | return; | |
1056 | ||
2dfca312 FF |
1057 | if (!(mp->hw->flags & IEEE80211_HW_SUPPORTS_HT_CCK_RATES)) |
1058 | return; | |
1059 | ||
a0497f9f FF |
1060 | mi->cck_supported = 0; |
1061 | mi->cck_supported_short = 0; | |
1062 | for (i = 0; i < 4; i++) { | |
1063 | if (!rate_supported(sta, sband->band, mp->cck_rates[i])) | |
1064 | continue; | |
1065 | ||
1066 | mi->cck_supported |= BIT(i); | |
1067 | if (sband->bitrates[i].flags & IEEE80211_RATE_SHORT_PREAMBLE) | |
1068 | mi->cck_supported_short |= BIT(i); | |
1069 | } | |
1070 | ||
1071 | mi->groups[MINSTREL_CCK_GROUP].supported = mi->cck_supported; | |
1072 | } | |
1073 | ||
ec8aa669 FF |
1074 | static void |
1075 | minstrel_ht_update_caps(void *priv, struct ieee80211_supported_band *sband, | |
3de805cf | 1076 | struct cfg80211_chan_def *chandef, |
64f68e5d | 1077 | struct ieee80211_sta *sta, void *priv_sta) |
ec8aa669 FF |
1078 | { |
1079 | struct minstrel_priv *mp = priv; | |
1080 | struct minstrel_ht_sta_priv *msp = priv_sta; | |
1081 | struct minstrel_ht_sta *mi = &msp->ht; | |
1082 | struct ieee80211_mcs_info *mcs = &sta->ht_cap.mcs; | |
ec8aa669 | 1083 | u16 sta_cap = sta->ht_cap.cap; |
9208247d KB |
1084 | struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap; |
1085 | int use_vht; | |
4dc217df | 1086 | int n_supported = 0; |
ec8aa669 FF |
1087 | int ack_dur; |
1088 | int stbc; | |
1089 | int i; | |
1090 | ||
1091 | /* fall back to the old minstrel for legacy stations */ | |
4dc217df FF |
1092 | if (!sta->ht_cap.ht_supported) |
1093 | goto use_legacy; | |
ec8aa669 | 1094 | |
8a0ee4fe | 1095 | BUILD_BUG_ON(ARRAY_SIZE(minstrel_mcs_groups) != MINSTREL_GROUPS_NB); |
ec8aa669 | 1096 | |
9208247d KB |
1097 | #ifdef CONFIG_MAC80211_RC_MINSTREL_VHT |
1098 | if (vht_cap->vht_supported) | |
1099 | use_vht = vht_cap->vht_mcs.tx_mcs_map != cpu_to_le16(~0); | |
1100 | else | |
1101 | #endif | |
1102 | use_vht = 0; | |
1103 | ||
ec8aa669 FF |
1104 | msp->is_ht = true; |
1105 | memset(mi, 0, sizeof(*mi)); | |
a8566662 FF |
1106 | |
1107 | mi->sta = sta; | |
ec8aa669 FF |
1108 | mi->stats_update = jiffies; |
1109 | ||
438b61b7 SW |
1110 | ack_dur = ieee80211_frame_duration(sband->band, 10, 60, 1, 1, 0); |
1111 | mi->overhead = ieee80211_frame_duration(sband->band, 0, 60, 1, 1, 0); | |
1112 | mi->overhead += ack_dur; | |
ec8aa669 FF |
1113 | mi->overhead_rtscts = mi->overhead + 2 * ack_dur; |
1114 | ||
1115 | mi->avg_ampdu_len = MINSTREL_FRAC(1, 1); | |
1116 | ||
1117 | /* When using MRR, sample more on the first attempt, without delay */ | |
1118 | if (mp->has_mrr) { | |
1119 | mi->sample_count = 16; | |
1120 | mi->sample_wait = 0; | |
1121 | } else { | |
1122 | mi->sample_count = 8; | |
1123 | mi->sample_wait = 8; | |
1124 | } | |
1125 | mi->sample_tries = 4; | |
1126 | ||
9208247d KB |
1127 | /* TODO tx_flags for vht - ATM the RC API is not fine-grained enough */ |
1128 | if (!use_vht) { | |
1129 | stbc = (sta_cap & IEEE80211_HT_CAP_RX_STBC) >> | |
1130 | IEEE80211_HT_CAP_RX_STBC_SHIFT; | |
1131 | mi->tx_flags |= stbc << IEEE80211_TX_CTL_STBC_SHIFT; | |
ec8aa669 | 1132 | |
9208247d KB |
1133 | if (sta_cap & IEEE80211_HT_CAP_LDPC_CODING) |
1134 | mi->tx_flags |= IEEE80211_TX_CTL_LDPC; | |
1135 | } | |
ec8aa669 | 1136 | |
ec8aa669 | 1137 | for (i = 0; i < ARRAY_SIZE(mi->groups); i++) { |
3ec373c4 | 1138 | u32 gflags = minstrel_mcs_groups[i].flags; |
9208247d | 1139 | int bw, nss; |
3ec373c4 | 1140 | |
ec8aa669 | 1141 | mi->groups[i].supported = 0; |
a0497f9f FF |
1142 | if (i == MINSTREL_CCK_GROUP) { |
1143 | minstrel_ht_update_cck(mp, mi, sband, sta); | |
1144 | continue; | |
1145 | } | |
1146 | ||
3ec373c4 KB |
1147 | if (gflags & IEEE80211_TX_RC_SHORT_GI) { |
1148 | if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH) { | |
e1a0c6b3 JB |
1149 | if (!(sta_cap & IEEE80211_HT_CAP_SGI_40)) |
1150 | continue; | |
1151 | } else { | |
1152 | if (!(sta_cap & IEEE80211_HT_CAP_SGI_20)) | |
1153 | continue; | |
1154 | } | |
ec8aa669 FF |
1155 | } |
1156 | ||
3ec373c4 | 1157 | if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH && |
e1a0c6b3 | 1158 | sta->bandwidth < IEEE80211_STA_RX_BW_40) |
ec8aa669 FF |
1159 | continue; |
1160 | ||
9208247d KB |
1161 | nss = minstrel_mcs_groups[i].streams; |
1162 | ||
e9219779 | 1163 | /* Mark MCS > 7 as unsupported if STA is in static SMPS mode */ |
9208247d KB |
1164 | if (sta->smps_mode == IEEE80211_SMPS_STATIC && nss > 1) |
1165 | continue; | |
1166 | ||
1167 | /* HT rate */ | |
1168 | if (gflags & IEEE80211_TX_RC_MCS) { | |
1169 | #ifdef CONFIG_MAC80211_RC_MINSTREL_VHT | |
9ffe9044 | 1170 | if (use_vht && minstrel_vht_only) |
9208247d KB |
1171 | continue; |
1172 | #endif | |
1173 | mi->groups[i].supported = mcs->rx_mask[nss - 1]; | |
1174 | if (mi->groups[i].supported) | |
1175 | n_supported++; | |
1176 | continue; | |
1177 | } | |
1178 | ||
1179 | /* VHT rate */ | |
1180 | if (!vht_cap->vht_supported || | |
1181 | WARN_ON(!(gflags & IEEE80211_TX_RC_VHT_MCS)) || | |
1182 | WARN_ON(gflags & IEEE80211_TX_RC_160_MHZ_WIDTH)) | |
e9219779 HS |
1183 | continue; |
1184 | ||
9208247d KB |
1185 | if (gflags & IEEE80211_TX_RC_80_MHZ_WIDTH) { |
1186 | if (sta->bandwidth < IEEE80211_STA_RX_BW_80 || | |
1187 | ((gflags & IEEE80211_TX_RC_SHORT_GI) && | |
1188 | !(vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80))) { | |
1189 | continue; | |
1190 | } | |
1191 | } | |
1192 | ||
1193 | if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH) | |
1194 | bw = BW_40; | |
1195 | else if (gflags & IEEE80211_TX_RC_80_MHZ_WIDTH) | |
1196 | bw = BW_80; | |
1197 | else | |
1198 | bw = BW_20; | |
1199 | ||
1200 | mi->groups[i].supported = minstrel_get_valid_vht_rates(bw, nss, | |
1201 | vht_cap->vht_mcs.tx_mcs_map); | |
4dc217df FF |
1202 | |
1203 | if (mi->groups[i].supported) | |
1204 | n_supported++; | |
ec8aa669 | 1205 | } |
4dc217df FF |
1206 | |
1207 | if (!n_supported) | |
1208 | goto use_legacy; | |
1209 | ||
a8566662 | 1210 | /* create an initial rate table with the lowest supported rates */ |
a3647362 | 1211 | minstrel_ht_update_stats(mp, mi); |
a8566662 | 1212 | minstrel_ht_update_rates(mp, mi); |
a3647362 | 1213 | |
4dc217df FF |
1214 | return; |
1215 | ||
1216 | use_legacy: | |
1217 | msp->is_ht = false; | |
1218 | memset(&msp->legacy, 0, sizeof(msp->legacy)); | |
1219 | msp->legacy.r = msp->ratelist; | |
1220 | msp->legacy.sample_table = msp->sample_table; | |
3de805cf SW |
1221 | return mac80211_minstrel.rate_init(priv, sband, chandef, sta, |
1222 | &msp->legacy); | |
ec8aa669 FF |
1223 | } |
1224 | ||
1225 | static void | |
1226 | minstrel_ht_rate_init(void *priv, struct ieee80211_supported_band *sband, | |
3de805cf | 1227 | struct cfg80211_chan_def *chandef, |
ec8aa669 FF |
1228 | struct ieee80211_sta *sta, void *priv_sta) |
1229 | { | |
3de805cf | 1230 | minstrel_ht_update_caps(priv, sband, chandef, sta, priv_sta); |
ec8aa669 FF |
1231 | } |
1232 | ||
1233 | static void | |
1234 | minstrel_ht_rate_update(void *priv, struct ieee80211_supported_band *sband, | |
3de805cf | 1235 | struct cfg80211_chan_def *chandef, |
ec8aa669 | 1236 | struct ieee80211_sta *sta, void *priv_sta, |
64f68e5d | 1237 | u32 changed) |
ec8aa669 | 1238 | { |
3de805cf | 1239 | minstrel_ht_update_caps(priv, sband, chandef, sta, priv_sta); |
ec8aa669 FF |
1240 | } |
1241 | ||
1242 | static void * | |
1243 | minstrel_ht_alloc_sta(void *priv, struct ieee80211_sta *sta, gfp_t gfp) | |
1244 | { | |
1245 | struct ieee80211_supported_band *sband; | |
1246 | struct minstrel_ht_sta_priv *msp; | |
1247 | struct minstrel_priv *mp = priv; | |
1248 | struct ieee80211_hw *hw = mp->hw; | |
1249 | int max_rates = 0; | |
1250 | int i; | |
1251 | ||
1252 | for (i = 0; i < IEEE80211_NUM_BANDS; i++) { | |
1253 | sband = hw->wiphy->bands[i]; | |
1254 | if (sband && sband->n_bitrates > max_rates) | |
1255 | max_rates = sband->n_bitrates; | |
1256 | } | |
1257 | ||
472dd35c | 1258 | msp = kzalloc(sizeof(*msp), gfp); |
ec8aa669 FF |
1259 | if (!msp) |
1260 | return NULL; | |
1261 | ||
1262 | msp->ratelist = kzalloc(sizeof(struct minstrel_rate) * max_rates, gfp); | |
1263 | if (!msp->ratelist) | |
1264 | goto error; | |
1265 | ||
1266 | msp->sample_table = kmalloc(SAMPLE_COLUMNS * max_rates, gfp); | |
1267 | if (!msp->sample_table) | |
1268 | goto error1; | |
1269 | ||
1270 | return msp; | |
1271 | ||
1272 | error1: | |
7e988014 | 1273 | kfree(msp->ratelist); |
ec8aa669 FF |
1274 | error: |
1275 | kfree(msp); | |
1276 | return NULL; | |
1277 | } | |
1278 | ||
1279 | static void | |
1280 | minstrel_ht_free_sta(void *priv, struct ieee80211_sta *sta, void *priv_sta) | |
1281 | { | |
1282 | struct minstrel_ht_sta_priv *msp = priv_sta; | |
1283 | ||
1284 | kfree(msp->sample_table); | |
1285 | kfree(msp->ratelist); | |
1286 | kfree(msp); | |
1287 | } | |
1288 | ||
1289 | static void * | |
1290 | minstrel_ht_alloc(struct ieee80211_hw *hw, struct dentry *debugfsdir) | |
1291 | { | |
1292 | return mac80211_minstrel.alloc(hw, debugfsdir); | |
1293 | } | |
1294 | ||
1295 | static void | |
1296 | minstrel_ht_free(void *priv) | |
1297 | { | |
1298 | mac80211_minstrel.free(priv); | |
1299 | } | |
1300 | ||
cca674d4 AQ |
1301 | static u32 minstrel_ht_get_expected_throughput(void *priv_sta) |
1302 | { | |
1303 | struct minstrel_ht_sta_priv *msp = priv_sta; | |
1304 | struct minstrel_ht_sta *mi = &msp->ht; | |
1305 | int i, j; | |
1306 | ||
1307 | if (!msp->is_ht) | |
1308 | return mac80211_minstrel.get_expected_throughput(priv_sta); | |
1309 | ||
5935839a TH |
1310 | i = mi->max_tp_rate[0] / MCS_GROUP_RATES; |
1311 | j = mi->max_tp_rate[0] % MCS_GROUP_RATES; | |
cca674d4 AQ |
1312 | |
1313 | /* convert cur_tp from pkt per second in kbps */ | |
1314 | return mi->groups[i].rates[j].cur_tp * AVG_PKT_SIZE * 8 / 1024; | |
1315 | } | |
1316 | ||
631ad703 | 1317 | static const struct rate_control_ops mac80211_minstrel_ht = { |
ec8aa669 | 1318 | .name = "minstrel_ht", |
63558a65 | 1319 | .tx_status_noskb = minstrel_ht_tx_status, |
ec8aa669 FF |
1320 | .get_rate = minstrel_ht_get_rate, |
1321 | .rate_init = minstrel_ht_rate_init, | |
1322 | .rate_update = minstrel_ht_rate_update, | |
1323 | .alloc_sta = minstrel_ht_alloc_sta, | |
1324 | .free_sta = minstrel_ht_free_sta, | |
1325 | .alloc = minstrel_ht_alloc, | |
1326 | .free = minstrel_ht_free, | |
1327 | #ifdef CONFIG_MAC80211_DEBUGFS | |
1328 | .add_sta_debugfs = minstrel_ht_add_sta_debugfs, | |
1329 | .remove_sta_debugfs = minstrel_ht_remove_sta_debugfs, | |
1330 | #endif | |
cca674d4 | 1331 | .get_expected_throughput = minstrel_ht_get_expected_throughput, |
ec8aa669 FF |
1332 | }; |
1333 | ||
1334 | ||
f6e1a73b | 1335 | static void __init init_sample_table(void) |
ec8aa669 FF |
1336 | { |
1337 | int col, i, new_idx; | |
1338 | u8 rnd[MCS_GROUP_RATES]; | |
1339 | ||
1340 | memset(sample_table, 0xff, sizeof(sample_table)); | |
1341 | for (col = 0; col < SAMPLE_COLUMNS; col++) { | |
f7d8ad81 | 1342 | prandom_bytes(rnd, sizeof(rnd)); |
ec8aa669 | 1343 | for (i = 0; i < MCS_GROUP_RATES; i++) { |
ec8aa669 | 1344 | new_idx = (i + rnd[i]) % MCS_GROUP_RATES; |
ec8aa669 FF |
1345 | while (sample_table[col][new_idx] != 0xff) |
1346 | new_idx = (new_idx + 1) % MCS_GROUP_RATES; | |
1347 | ||
1348 | sample_table[col][new_idx] = i; | |
1349 | } | |
1350 | } | |
1351 | } | |
1352 | ||
1353 | int __init | |
1354 | rc80211_minstrel_ht_init(void) | |
1355 | { | |
1356 | init_sample_table(); | |
1357 | return ieee80211_rate_control_register(&mac80211_minstrel_ht); | |
1358 | } | |
1359 | ||
1360 | void | |
1361 | rc80211_minstrel_ht_exit(void) | |
1362 | { | |
1363 | ieee80211_rate_control_unregister(&mac80211_minstrel_ht); | |
1364 | } |