Commit | Line | Data |
---|---|---|
1f5a7e47 JB |
1 | /* |
2 | * Copyright 2002-2005, Instant802 Networks, Inc. | |
3 | * Copyright 2005-2006, Devicescape Software, Inc. | |
4 | * Copyright 2006-2007 Jiri Benc <jbenc@suse.cz> | |
3b96766f | 5 | * Copyright 2007-2008 Johannes Berg <johannes@sipsolutions.net> |
1f5a7e47 JB |
6 | * |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | */ | |
11 | ||
11a843b7 JB |
12 | #include <linux/if_ether.h> |
13 | #include <linux/etherdevice.h> | |
14 | #include <linux/list.h> | |
d4e46a3d | 15 | #include <linux/rcupdate.h> |
db4d1169 | 16 | #include <linux/rtnetlink.h> |
5a0e3ad6 | 17 | #include <linux/slab.h> |
bc3b2d7f | 18 | #include <linux/export.h> |
1f5a7e47 | 19 | #include <net/mac80211.h> |
d26ad377 | 20 | #include <asm/unaligned.h> |
1f5a7e47 | 21 | #include "ieee80211_i.h" |
24487981 | 22 | #include "driver-ops.h" |
1f5a7e47 JB |
23 | #include "debugfs_key.h" |
24 | #include "aes_ccm.h" | |
3cfcf6ac | 25 | #include "aes_cmac.h" |
1f5a7e47 | 26 | |
11a843b7 | 27 | |
dbbea671 JB |
28 | /** |
29 | * DOC: Key handling basics | |
11a843b7 JB |
30 | * |
31 | * Key handling in mac80211 is done based on per-interface (sub_if_data) | |
32 | * keys and per-station keys. Since each station belongs to an interface, | |
33 | * each station key also belongs to that interface. | |
34 | * | |
b5c34f66 JB |
35 | * Hardware acceleration is done on a best-effort basis for algorithms |
36 | * that are implemented in software, for each key the hardware is asked | |
37 | * to enable that key for offloading but if it cannot do that the key is | |
38 | * simply kept for software encryption (unless it is for an algorithm | |
39 | * that isn't implemented in software). | |
40 | * There is currently no way of knowing whether a key is handled in SW | |
41 | * or HW except by looking into debugfs. | |
11a843b7 | 42 | * |
b5c34f66 JB |
43 | * All key management is internally protected by a mutex. Within all |
44 | * other parts of mac80211, key references are, just as STA structure | |
45 | * references, protected by RCU. Note, however, that some things are | |
46 | * unprotected, namely the key->sta dereferences within the hardware | |
47 | * acceleration functions. This means that sta_info_destroy() must | |
48 | * remove the key which waits for an RCU grace period. | |
11a843b7 JB |
49 | */ |
50 | ||
51 | static const u8 bcast_addr[ETH_ALEN] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; | |
11a843b7 | 52 | |
ad0e2b5a | 53 | static void assert_key_lock(struct ieee80211_local *local) |
3b96766f | 54 | { |
46a5ebaf | 55 | lockdep_assert_held(&local->key_mtx); |
3b96766f JB |
56 | } |
57 | ||
3bff1865 YAP |
58 | static void increment_tailroom_need_count(struct ieee80211_sub_if_data *sdata) |
59 | { | |
60 | /* | |
61 | * When this count is zero, SKB resizing for allocating tailroom | |
62 | * for IV or MMIC is skipped. But, this check has created two race | |
63 | * cases in xmit path while transiting from zero count to one: | |
64 | * | |
65 | * 1. SKB resize was skipped because no key was added but just before | |
66 | * the xmit key is added and SW encryption kicks off. | |
67 | * | |
68 | * 2. SKB resize was skipped because all the keys were hw planted but | |
69 | * just before xmit one of the key is deleted and SW encryption kicks | |
70 | * off. | |
71 | * | |
72 | * In both the above case SW encryption will find not enough space for | |
73 | * tailroom and exits with WARN_ON. (See WARN_ONs at wpa.c) | |
74 | * | |
75 | * Solution has been explained at | |
76 | * http://mid.gmane.org/1308590980.4322.19.camel@jlt3.sipsolutions.net | |
77 | */ | |
78 | ||
79 | if (!sdata->crypto_tx_tailroom_needed_cnt++) { | |
80 | /* | |
81 | * Flush all XMIT packets currently using HW encryption or no | |
82 | * encryption at all if the count transition is from 0 -> 1. | |
83 | */ | |
84 | synchronize_net(); | |
85 | } | |
86 | } | |
87 | ||
3ffc2a90 | 88 | static int ieee80211_key_enable_hw_accel(struct ieee80211_key *key) |
11a843b7 | 89 | { |
dc822b5d | 90 | struct ieee80211_sub_if_data *sdata; |
89c91cae | 91 | struct sta_info *sta; |
11a843b7 JB |
92 | int ret; |
93 | ||
3b96766f JB |
94 | might_sleep(); |
95 | ||
27b3eb9c JB |
96 | if (key->flags & KEY_FLAG_TAINTED) |
97 | return -EINVAL; | |
98 | ||
e31b8213 | 99 | if (!key->local->ops->set_key) |
3ffc2a90 | 100 | goto out_unsupported; |
11a843b7 | 101 | |
ad0e2b5a JB |
102 | assert_key_lock(key->local); |
103 | ||
89c91cae | 104 | sta = key->sta; |
dc822b5d | 105 | |
e31b8213 JB |
106 | /* |
107 | * If this is a per-STA GTK, check if it | |
108 | * is supported; if not, return. | |
109 | */ | |
110 | if (sta && !(key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE) && | |
111 | !(key->local->hw.flags & IEEE80211_HW_SUPPORTS_PER_STA_GTK)) | |
112 | goto out_unsupported; | |
113 | ||
89c91cae JB |
114 | if (sta && !sta->uploaded) |
115 | goto out_unsupported; | |
116 | ||
dc822b5d | 117 | sdata = key->sdata; |
18890d4b HS |
118 | if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) { |
119 | /* | |
120 | * The driver doesn't know anything about VLAN interfaces. | |
121 | * Hence, don't send GTKs for VLAN interfaces to the driver. | |
122 | */ | |
123 | if (!(key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE)) | |
124 | goto out_unsupported; | |
18890d4b | 125 | } |
11a843b7 | 126 | |
89c91cae JB |
127 | ret = drv_set_key(key->local, SET_KEY, sdata, |
128 | sta ? &sta->sta : NULL, &key->conf); | |
11a843b7 | 129 | |
e31b8213 | 130 | if (!ret) { |
11a843b7 | 131 | key->flags |= KEY_FLAG_UPLOADED_TO_HARDWARE; |
3bff1865 YAP |
132 | |
133 | if (!((key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_MMIC) || | |
077a9154 AN |
134 | (key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_IV) || |
135 | (key->conf.flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))) | |
3bff1865 YAP |
136 | sdata->crypto_tx_tailroom_needed_cnt--; |
137 | ||
077a9154 AN |
138 | WARN_ON((key->conf.flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE) && |
139 | (key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_IV)); | |
140 | ||
e31b8213 JB |
141 | return 0; |
142 | } | |
11a843b7 | 143 | |
e31b8213 | 144 | if (ret != -ENOSPC && ret != -EOPNOTSUPP) |
bdcbd8e0 | 145 | sdata_err(sdata, |
0fb9a9ec | 146 | "failed to set key (%d, %pM) to hardware (%d)\n", |
89c91cae JB |
147 | key->conf.keyidx, |
148 | sta ? sta->sta.addr : bcast_addr, ret); | |
3ffc2a90 | 149 | |
e31b8213 JB |
150 | out_unsupported: |
151 | switch (key->conf.cipher) { | |
152 | case WLAN_CIPHER_SUITE_WEP40: | |
153 | case WLAN_CIPHER_SUITE_WEP104: | |
154 | case WLAN_CIPHER_SUITE_TKIP: | |
155 | case WLAN_CIPHER_SUITE_CCMP: | |
156 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
157 | /* all of these we can do in software */ | |
158 | return 0; | |
159 | default: | |
160 | return -EINVAL; | |
3ffc2a90 | 161 | } |
11a843b7 JB |
162 | } |
163 | ||
164 | static void ieee80211_key_disable_hw_accel(struct ieee80211_key *key) | |
165 | { | |
dc822b5d | 166 | struct ieee80211_sub_if_data *sdata; |
89c91cae | 167 | struct sta_info *sta; |
11a843b7 JB |
168 | int ret; |
169 | ||
3b96766f JB |
170 | might_sleep(); |
171 | ||
db4d1169 | 172 | if (!key || !key->local->ops->set_key) |
11a843b7 JB |
173 | return; |
174 | ||
ad0e2b5a JB |
175 | assert_key_lock(key->local); |
176 | ||
177 | if (!(key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE)) | |
11a843b7 JB |
178 | return; |
179 | ||
89c91cae | 180 | sta = key->sta; |
dc822b5d JB |
181 | sdata = key->sdata; |
182 | ||
3bff1865 | 183 | if (!((key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_MMIC) || |
077a9154 AN |
184 | (key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_IV) || |
185 | (key->conf.flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))) | |
3bff1865 YAP |
186 | increment_tailroom_need_count(sdata); |
187 | ||
12375ef9 | 188 | ret = drv_set_key(key->local, DISABLE_KEY, sdata, |
89c91cae | 189 | sta ? &sta->sta : NULL, &key->conf); |
11a843b7 JB |
190 | |
191 | if (ret) | |
bdcbd8e0 | 192 | sdata_err(sdata, |
0fb9a9ec | 193 | "failed to remove key (%d, %pM) from hardware (%d)\n", |
89c91cae JB |
194 | key->conf.keyidx, |
195 | sta ? sta->sta.addr : bcast_addr, ret); | |
11a843b7 | 196 | |
3b96766f | 197 | key->flags &= ~KEY_FLAG_UPLOADED_TO_HARDWARE; |
3b96766f JB |
198 | } |
199 | ||
200 | static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, | |
f7e0104c | 201 | int idx, bool uni, bool multi) |
3b96766f JB |
202 | { |
203 | struct ieee80211_key *key = NULL; | |
204 | ||
ad0e2b5a JB |
205 | assert_key_lock(sdata->local); |
206 | ||
3b96766f | 207 | if (idx >= 0 && idx < NUM_DEFAULT_KEYS) |
40b275b6 | 208 | key = key_mtx_dereference(sdata->local, sdata->keys[idx]); |
3b96766f | 209 | |
de5fad81 | 210 | if (uni) { |
f7e0104c | 211 | rcu_assign_pointer(sdata->default_unicast_key, key); |
de5fad81 YD |
212 | drv_set_default_unicast_key(sdata->local, sdata, idx); |
213 | } | |
214 | ||
f7e0104c JB |
215 | if (multi) |
216 | rcu_assign_pointer(sdata->default_multicast_key, key); | |
3b96766f | 217 | |
f7e0104c | 218 | ieee80211_debugfs_key_update_default(sdata); |
3b96766f JB |
219 | } |
220 | ||
f7e0104c JB |
221 | void ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, int idx, |
222 | bool uni, bool multi) | |
3b96766f | 223 | { |
ad0e2b5a | 224 | mutex_lock(&sdata->local->key_mtx); |
f7e0104c | 225 | __ieee80211_set_default_key(sdata, idx, uni, multi); |
ad0e2b5a | 226 | mutex_unlock(&sdata->local->key_mtx); |
3b96766f JB |
227 | } |
228 | ||
3cfcf6ac JM |
229 | static void |
230 | __ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, int idx) | |
231 | { | |
232 | struct ieee80211_key *key = NULL; | |
233 | ||
ad0e2b5a JB |
234 | assert_key_lock(sdata->local); |
235 | ||
3cfcf6ac JM |
236 | if (idx >= NUM_DEFAULT_KEYS && |
237 | idx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS) | |
40b275b6 | 238 | key = key_mtx_dereference(sdata->local, sdata->keys[idx]); |
3cfcf6ac JM |
239 | |
240 | rcu_assign_pointer(sdata->default_mgmt_key, key); | |
241 | ||
f7e0104c | 242 | ieee80211_debugfs_key_update_default(sdata); |
3cfcf6ac JM |
243 | } |
244 | ||
245 | void ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, | |
246 | int idx) | |
247 | { | |
ad0e2b5a | 248 | mutex_lock(&sdata->local->key_mtx); |
3cfcf6ac | 249 | __ieee80211_set_default_mgmt_key(sdata, idx); |
ad0e2b5a | 250 | mutex_unlock(&sdata->local->key_mtx); |
3cfcf6ac JM |
251 | } |
252 | ||
3b96766f | 253 | |
3b8d9c29 JB |
254 | static void ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, |
255 | struct sta_info *sta, | |
256 | bool pairwise, | |
257 | struct ieee80211_key *old, | |
258 | struct ieee80211_key *new) | |
3b96766f | 259 | { |
f7e0104c JB |
260 | int idx; |
261 | bool defunikey, defmultikey, defmgmtkey; | |
3b96766f | 262 | |
5282c3ba JB |
263 | /* caller must provide at least one old/new */ |
264 | if (WARN_ON(!new && !old)) | |
265 | return; | |
266 | ||
3b96766f | 267 | if (new) |
f850e00f | 268 | list_add_tail(&new->list, &sdata->key_list); |
3b96766f | 269 | |
2475b1cc | 270 | WARN_ON(new && old && new->conf.keyidx != old->conf.keyidx); |
3b96766f | 271 | |
2475b1cc MS |
272 | if (old) |
273 | idx = old->conf.keyidx; | |
274 | else | |
275 | idx = new->conf.keyidx; | |
3b96766f | 276 | |
2475b1cc MS |
277 | if (sta) { |
278 | if (pairwise) { | |
279 | rcu_assign_pointer(sta->ptk[idx], new); | |
280 | sta->ptk_idx = idx; | |
281 | } else { | |
282 | rcu_assign_pointer(sta->gtk[idx], new); | |
283 | sta->gtk_idx = idx; | |
284 | } | |
285 | } else { | |
40b275b6 JB |
286 | defunikey = old && |
287 | old == key_mtx_dereference(sdata->local, | |
288 | sdata->default_unicast_key); | |
289 | defmultikey = old && | |
290 | old == key_mtx_dereference(sdata->local, | |
291 | sdata->default_multicast_key); | |
292 | defmgmtkey = old && | |
293 | old == key_mtx_dereference(sdata->local, | |
294 | sdata->default_mgmt_key); | |
3b96766f | 295 | |
f7e0104c JB |
296 | if (defunikey && !new) |
297 | __ieee80211_set_default_key(sdata, -1, true, false); | |
298 | if (defmultikey && !new) | |
299 | __ieee80211_set_default_key(sdata, -1, false, true); | |
3cfcf6ac JM |
300 | if (defmgmtkey && !new) |
301 | __ieee80211_set_default_mgmt_key(sdata, -1); | |
3b96766f JB |
302 | |
303 | rcu_assign_pointer(sdata->keys[idx], new); | |
f7e0104c JB |
304 | if (defunikey && new) |
305 | __ieee80211_set_default_key(sdata, new->conf.keyidx, | |
306 | true, false); | |
307 | if (defmultikey && new) | |
308 | __ieee80211_set_default_key(sdata, new->conf.keyidx, | |
309 | false, true); | |
3cfcf6ac JM |
310 | if (defmgmtkey && new) |
311 | __ieee80211_set_default_mgmt_key(sdata, | |
312 | new->conf.keyidx); | |
3b96766f JB |
313 | } |
314 | ||
b5c34f66 JB |
315 | if (old) |
316 | list_del(&old->list); | |
11a843b7 JB |
317 | } |
318 | ||
2475b1cc MS |
319 | struct ieee80211_key * |
320 | ieee80211_key_alloc(u32 cipher, int idx, size_t key_len, | |
321 | const u8 *key_data, | |
322 | size_t seq_len, const u8 *seq, | |
323 | const struct ieee80211_cipher_scheme *cs) | |
1f5a7e47 JB |
324 | { |
325 | struct ieee80211_key *key; | |
1ac62ba7 | 326 | int i, j, err; |
1f5a7e47 | 327 | |
8c5bb1fa JB |
328 | if (WARN_ON(idx < 0 || idx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS)) |
329 | return ERR_PTR(-EINVAL); | |
11a843b7 JB |
330 | |
331 | key = kzalloc(sizeof(struct ieee80211_key) + key_len, GFP_KERNEL); | |
1f5a7e47 | 332 | if (!key) |
1ac62ba7 | 333 | return ERR_PTR(-ENOMEM); |
11a843b7 JB |
334 | |
335 | /* | |
336 | * Default to software encryption; we'll later upload the | |
337 | * key to the hardware if possible. | |
338 | */ | |
11a843b7 JB |
339 | key->conf.flags = 0; |
340 | key->flags = 0; | |
341 | ||
97359d12 | 342 | key->conf.cipher = cipher; |
11a843b7 JB |
343 | key->conf.keyidx = idx; |
344 | key->conf.keylen = key_len; | |
97359d12 JB |
345 | switch (cipher) { |
346 | case WLAN_CIPHER_SUITE_WEP40: | |
347 | case WLAN_CIPHER_SUITE_WEP104: | |
4325f6ca JB |
348 | key->conf.iv_len = IEEE80211_WEP_IV_LEN; |
349 | key->conf.icv_len = IEEE80211_WEP_ICV_LEN; | |
76708dee | 350 | break; |
97359d12 | 351 | case WLAN_CIPHER_SUITE_TKIP: |
4325f6ca JB |
352 | key->conf.iv_len = IEEE80211_TKIP_IV_LEN; |
353 | key->conf.icv_len = IEEE80211_TKIP_ICV_LEN; | |
9f26a952 | 354 | if (seq) { |
5a306f58 | 355 | for (i = 0; i < IEEE80211_NUM_TIDS; i++) { |
faa8fdc8 JM |
356 | key->u.tkip.rx[i].iv32 = |
357 | get_unaligned_le32(&seq[2]); | |
358 | key->u.tkip.rx[i].iv16 = | |
359 | get_unaligned_le16(seq); | |
360 | } | |
361 | } | |
523b02ea | 362 | spin_lock_init(&key->u.tkip.txlock); |
76708dee | 363 | break; |
97359d12 | 364 | case WLAN_CIPHER_SUITE_CCMP: |
4325f6ca JB |
365 | key->conf.iv_len = IEEE80211_CCMP_HDR_LEN; |
366 | key->conf.icv_len = IEEE80211_CCMP_MIC_LEN; | |
9f26a952 | 367 | if (seq) { |
5a306f58 | 368 | for (i = 0; i < IEEE80211_NUM_TIDS + 1; i++) |
4325f6ca | 369 | for (j = 0; j < IEEE80211_CCMP_PN_LEN; j++) |
faa8fdc8 | 370 | key->u.ccmp.rx_pn[i][j] = |
4325f6ca | 371 | seq[IEEE80211_CCMP_PN_LEN - j - 1]; |
faa8fdc8 | 372 | } |
11a843b7 JB |
373 | /* |
374 | * Initialize AES key state here as an optimization so that | |
375 | * it does not need to be initialized for every packet. | |
376 | */ | |
377 | key->u.ccmp.tfm = ieee80211_aes_key_setup_encrypt(key_data); | |
1ac62ba7 BH |
378 | if (IS_ERR(key->u.ccmp.tfm)) { |
379 | err = PTR_ERR(key->u.ccmp.tfm); | |
3b96766f | 380 | kfree(key); |
1f951a7f | 381 | return ERR_PTR(err); |
11a843b7 | 382 | } |
60ae0f20 JB |
383 | break; |
384 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
385 | key->conf.iv_len = 0; | |
386 | key->conf.icv_len = sizeof(struct ieee80211_mmie); | |
387 | if (seq) | |
4325f6ca | 388 | for (j = 0; j < IEEE80211_CMAC_PN_LEN; j++) |
0f927323 | 389 | key->u.aes_cmac.rx_pn[j] = |
4325f6ca | 390 | seq[IEEE80211_CMAC_PN_LEN - j - 1]; |
3cfcf6ac JM |
391 | /* |
392 | * Initialize AES key state here as an optimization so that | |
393 | * it does not need to be initialized for every packet. | |
394 | */ | |
395 | key->u.aes_cmac.tfm = | |
396 | ieee80211_aes_cmac_key_setup(key_data); | |
1ac62ba7 BH |
397 | if (IS_ERR(key->u.aes_cmac.tfm)) { |
398 | err = PTR_ERR(key->u.aes_cmac.tfm); | |
3cfcf6ac | 399 | kfree(key); |
1f951a7f | 400 | return ERR_PTR(err); |
3cfcf6ac | 401 | } |
60ae0f20 | 402 | break; |
2475b1cc MS |
403 | default: |
404 | if (cs) { | |
405 | size_t len = (seq_len > MAX_PN_LEN) ? | |
406 | MAX_PN_LEN : seq_len; | |
407 | ||
408 | key->conf.iv_len = cs->hdr_len; | |
409 | key->conf.icv_len = cs->mic_len; | |
410 | for (i = 0; i < IEEE80211_NUM_TIDS + 1; i++) | |
411 | for (j = 0; j < len; j++) | |
412 | key->u.gen.rx_pn[i][j] = | |
413 | seq[len - j - 1]; | |
414 | } | |
3cfcf6ac | 415 | } |
60ae0f20 JB |
416 | memcpy(key->conf.key, key_data, key_len); |
417 | INIT_LIST_HEAD(&key->list); | |
3cfcf6ac | 418 | |
db4d1169 JB |
419 | return key; |
420 | } | |
11a843b7 | 421 | |
79cf2dfa JB |
422 | static void ieee80211_key_free_common(struct ieee80211_key *key) |
423 | { | |
424 | if (key->conf.cipher == WLAN_CIPHER_SUITE_CCMP) | |
425 | ieee80211_aes_key_free(key->u.ccmp.tfm); | |
426 | if (key->conf.cipher == WLAN_CIPHER_SUITE_AES_CMAC) | |
427 | ieee80211_aes_cmac_key_free(key->u.aes_cmac.tfm); | |
428 | kfree(key); | |
429 | } | |
430 | ||
6d10e46b JB |
431 | static void __ieee80211_key_destroy(struct ieee80211_key *key, |
432 | bool delay_tailroom) | |
ad0e2b5a | 433 | { |
32162a4d JM |
434 | if (key->local) |
435 | ieee80211_key_disable_hw_accel(key); | |
ad0e2b5a | 436 | |
3bff1865 | 437 | if (key->local) { |
8d1f7ecd JB |
438 | struct ieee80211_sub_if_data *sdata = key->sdata; |
439 | ||
32162a4d | 440 | ieee80211_debugfs_key_remove(key); |
8d1f7ecd JB |
441 | |
442 | if (delay_tailroom) { | |
443 | /* see ieee80211_delayed_tailroom_dec */ | |
444 | sdata->crypto_tx_tailroom_pending_dec++; | |
445 | schedule_delayed_work(&sdata->dec_tailroom_needed_wk, | |
446 | HZ/2); | |
447 | } else { | |
448 | sdata->crypto_tx_tailroom_needed_cnt--; | |
449 | } | |
3bff1865 | 450 | } |
ad0e2b5a | 451 | |
79cf2dfa JB |
452 | ieee80211_key_free_common(key); |
453 | } | |
454 | ||
6d10e46b JB |
455 | static void ieee80211_key_destroy(struct ieee80211_key *key, |
456 | bool delay_tailroom) | |
457 | { | |
458 | if (!key) | |
459 | return; | |
460 | ||
461 | /* | |
462 | * Synchronize so the TX path can no longer be using | |
463 | * this key before we free/remove it. | |
464 | */ | |
465 | synchronize_net(); | |
466 | ||
467 | __ieee80211_key_destroy(key, delay_tailroom); | |
468 | } | |
469 | ||
79cf2dfa JB |
470 | void ieee80211_key_free_unused(struct ieee80211_key *key) |
471 | { | |
472 | WARN_ON(key->sdata || key->local); | |
473 | ieee80211_key_free_common(key); | |
ad0e2b5a JB |
474 | } |
475 | ||
3ffc2a90 JB |
476 | int ieee80211_key_link(struct ieee80211_key *key, |
477 | struct ieee80211_sub_if_data *sdata, | |
478 | struct sta_info *sta) | |
db4d1169 | 479 | { |
27b3eb9c | 480 | struct ieee80211_local *local = sdata->local; |
db4d1169 | 481 | struct ieee80211_key *old_key; |
3ffc2a90 | 482 | int idx, ret; |
67aa030c | 483 | bool pairwise; |
db4d1169 | 484 | |
67aa030c | 485 | pairwise = key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE; |
db4d1169 JB |
486 | idx = key->conf.keyidx; |
487 | key->local = sdata->local; | |
488 | key->sdata = sdata; | |
489 | key->sta = sta; | |
490 | ||
ad0e2b5a | 491 | mutex_lock(&sdata->local->key_mtx); |
3b96766f | 492 | |
e31b8213 | 493 | if (sta && pairwise) |
2475b1cc | 494 | old_key = key_mtx_dereference(sdata->local, sta->ptk[idx]); |
e31b8213 | 495 | else if (sta) |
40b275b6 | 496 | old_key = key_mtx_dereference(sdata->local, sta->gtk[idx]); |
d4e46a3d | 497 | else |
40b275b6 | 498 | old_key = key_mtx_dereference(sdata->local, sdata->keys[idx]); |
db4d1169 | 499 | |
3bff1865 YAP |
500 | increment_tailroom_need_count(sdata); |
501 | ||
3b8d9c29 JB |
502 | ieee80211_key_replace(sdata, sta, pairwise, old_key, key); |
503 | ieee80211_key_destroy(old_key, true); | |
d4e46a3d | 504 | |
ad0e2b5a | 505 | ieee80211_debugfs_key_add(key); |
db4d1169 | 506 | |
27b3eb9c JB |
507 | if (!local->wowlan) { |
508 | ret = ieee80211_key_enable_hw_accel(key); | |
509 | if (ret) | |
510 | ieee80211_key_free(key, true); | |
511 | } else { | |
512 | ret = 0; | |
513 | } | |
79cf2dfa | 514 | |
ad0e2b5a | 515 | mutex_unlock(&sdata->local->key_mtx); |
3ffc2a90 JB |
516 | |
517 | return ret; | |
1f5a7e47 JB |
518 | } |
519 | ||
3b8d9c29 | 520 | void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom) |
1f5a7e47 | 521 | { |
5c0c3641 JB |
522 | if (!key) |
523 | return; | |
524 | ||
3b96766f JB |
525 | /* |
526 | * Replace key with nothingness if it was ever used. | |
527 | */ | |
3a245766 | 528 | if (key->sdata) |
3b8d9c29 | 529 | ieee80211_key_replace(key->sdata, key->sta, |
e31b8213 JB |
530 | key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, |
531 | key, NULL); | |
3b8d9c29 | 532 | ieee80211_key_destroy(key, delay_tailroom); |
3b96766f | 533 | } |
d4e46a3d | 534 | |
ad0e2b5a | 535 | void ieee80211_enable_keys(struct ieee80211_sub_if_data *sdata) |
3a245766 JB |
536 | { |
537 | struct ieee80211_key *key; | |
11a843b7 | 538 | |
3a245766 | 539 | ASSERT_RTNL(); |
11a843b7 | 540 | |
9607e6b6 | 541 | if (WARN_ON(!ieee80211_sdata_running(sdata))) |
3a245766 | 542 | return; |
11a843b7 | 543 | |
ad0e2b5a | 544 | mutex_lock(&sdata->local->key_mtx); |
11a843b7 | 545 | |
3bff1865 YAP |
546 | sdata->crypto_tx_tailroom_needed_cnt = 0; |
547 | ||
548 | list_for_each_entry(key, &sdata->key_list, list) { | |
549 | increment_tailroom_need_count(sdata); | |
ad0e2b5a | 550 | ieee80211_key_enable_hw_accel(key); |
3bff1865 | 551 | } |
3b96766f | 552 | |
ad0e2b5a | 553 | mutex_unlock(&sdata->local->key_mtx); |
11a843b7 JB |
554 | } |
555 | ||
830af02f JB |
556 | void ieee80211_iter_keys(struct ieee80211_hw *hw, |
557 | struct ieee80211_vif *vif, | |
558 | void (*iter)(struct ieee80211_hw *hw, | |
559 | struct ieee80211_vif *vif, | |
560 | struct ieee80211_sta *sta, | |
561 | struct ieee80211_key_conf *key, | |
562 | void *data), | |
563 | void *iter_data) | |
564 | { | |
565 | struct ieee80211_local *local = hw_to_local(hw); | |
27b3eb9c | 566 | struct ieee80211_key *key, *tmp; |
830af02f JB |
567 | struct ieee80211_sub_if_data *sdata; |
568 | ||
569 | ASSERT_RTNL(); | |
570 | ||
571 | mutex_lock(&local->key_mtx); | |
572 | if (vif) { | |
573 | sdata = vif_to_sdata(vif); | |
27b3eb9c | 574 | list_for_each_entry_safe(key, tmp, &sdata->key_list, list) |
830af02f JB |
575 | iter(hw, &sdata->vif, |
576 | key->sta ? &key->sta->sta : NULL, | |
577 | &key->conf, iter_data); | |
578 | } else { | |
579 | list_for_each_entry(sdata, &local->interfaces, list) | |
27b3eb9c JB |
580 | list_for_each_entry_safe(key, tmp, |
581 | &sdata->key_list, list) | |
830af02f JB |
582 | iter(hw, &sdata->vif, |
583 | key->sta ? &key->sta->sta : NULL, | |
584 | &key->conf, iter_data); | |
585 | } | |
586 | mutex_unlock(&local->key_mtx); | |
587 | } | |
588 | EXPORT_SYMBOL(ieee80211_iter_keys); | |
589 | ||
7907c7d3 JB |
590 | static void ieee80211_free_keys_iface(struct ieee80211_sub_if_data *sdata, |
591 | struct list_head *keys) | |
3b96766f JB |
592 | { |
593 | struct ieee80211_key *key, *tmp; | |
3b96766f | 594 | |
8d1f7ecd JB |
595 | sdata->crypto_tx_tailroom_needed_cnt -= |
596 | sdata->crypto_tx_tailroom_pending_dec; | |
597 | sdata->crypto_tx_tailroom_pending_dec = 0; | |
598 | ||
3cfcf6ac | 599 | ieee80211_debugfs_key_remove_mgmt_default(sdata); |
3b96766f | 600 | |
6d10e46b JB |
601 | list_for_each_entry_safe(key, tmp, &sdata->key_list, list) { |
602 | ieee80211_key_replace(key->sdata, key->sta, | |
603 | key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, | |
604 | key, NULL); | |
7907c7d3 | 605 | list_add_tail(&key->list, keys); |
6d10e46b | 606 | } |
3b96766f | 607 | |
f7e0104c | 608 | ieee80211_debugfs_key_update_default(sdata); |
7907c7d3 | 609 | } |
f7e0104c | 610 | |
7907c7d3 JB |
611 | void ieee80211_free_keys(struct ieee80211_sub_if_data *sdata, |
612 | bool force_synchronize) | |
613 | { | |
614 | struct ieee80211_local *local = sdata->local; | |
615 | struct ieee80211_sub_if_data *vlan; | |
616 | struct ieee80211_key *key, *tmp; | |
617 | LIST_HEAD(keys); | |
618 | ||
619 | cancel_delayed_work_sync(&sdata->dec_tailroom_needed_wk); | |
620 | ||
621 | mutex_lock(&local->key_mtx); | |
622 | ||
623 | ieee80211_free_keys_iface(sdata, &keys); | |
624 | ||
625 | if (sdata->vif.type == NL80211_IFTYPE_AP) { | |
626 | list_for_each_entry(vlan, &sdata->u.ap.vlans, u.vlan.list) | |
627 | ieee80211_free_keys_iface(vlan, &keys); | |
6d10e46b JB |
628 | } |
629 | ||
7907c7d3 JB |
630 | if (!list_empty(&keys) || force_synchronize) |
631 | synchronize_net(); | |
632 | list_for_each_entry_safe(key, tmp, &keys, list) | |
633 | __ieee80211_key_destroy(key, false); | |
634 | ||
8d1f7ecd JB |
635 | WARN_ON_ONCE(sdata->crypto_tx_tailroom_needed_cnt || |
636 | sdata->crypto_tx_tailroom_pending_dec); | |
7907c7d3 JB |
637 | if (sdata->vif.type == NL80211_IFTYPE_AP) { |
638 | list_for_each_entry(vlan, &sdata->u.ap.vlans, u.vlan.list) | |
639 | WARN_ON_ONCE(vlan->crypto_tx_tailroom_needed_cnt || | |
640 | vlan->crypto_tx_tailroom_pending_dec); | |
641 | } | |
8d1f7ecd | 642 | |
7907c7d3 | 643 | mutex_unlock(&local->key_mtx); |
11a843b7 | 644 | } |
c68f4b89 | 645 | |
6d10e46b JB |
646 | void ieee80211_free_sta_keys(struct ieee80211_local *local, |
647 | struct sta_info *sta) | |
648 | { | |
c8782078 | 649 | struct ieee80211_key *key; |
6d10e46b JB |
650 | int i; |
651 | ||
652 | mutex_lock(&local->key_mtx); | |
653 | for (i = 0; i < NUM_DEFAULT_KEYS; i++) { | |
654 | key = key_mtx_dereference(local, sta->gtk[i]); | |
655 | if (!key) | |
656 | continue; | |
657 | ieee80211_key_replace(key->sdata, key->sta, | |
658 | key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, | |
659 | key, NULL); | |
c8782078 | 660 | __ieee80211_key_destroy(key, true); |
6d10e46b JB |
661 | } |
662 | ||
2475b1cc MS |
663 | for (i = 0; i < NUM_DEFAULT_KEYS; i++) { |
664 | key = key_mtx_dereference(local, sta->ptk[i]); | |
665 | if (!key) | |
666 | continue; | |
6d10e46b JB |
667 | ieee80211_key_replace(key->sdata, key->sta, |
668 | key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, | |
669 | key, NULL); | |
6d10e46b | 670 | __ieee80211_key_destroy(key, true); |
c8782078 | 671 | } |
6d10e46b JB |
672 | |
673 | mutex_unlock(&local->key_mtx); | |
674 | } | |
675 | ||
8d1f7ecd JB |
676 | void ieee80211_delayed_tailroom_dec(struct work_struct *wk) |
677 | { | |
678 | struct ieee80211_sub_if_data *sdata; | |
679 | ||
680 | sdata = container_of(wk, struct ieee80211_sub_if_data, | |
681 | dec_tailroom_needed_wk.work); | |
682 | ||
683 | /* | |
684 | * The reason for the delayed tailroom needed decrementing is to | |
685 | * make roaming faster: during roaming, all keys are first deleted | |
686 | * and then new keys are installed. The first new key causes the | |
687 | * crypto_tx_tailroom_needed_cnt to go from 0 to 1, which invokes | |
688 | * the cost of synchronize_net() (which can be slow). Avoid this | |
689 | * by deferring the crypto_tx_tailroom_needed_cnt decrementing on | |
690 | * key removal for a while, so if we roam the value is larger than | |
691 | * zero and no 0->1 transition happens. | |
692 | * | |
693 | * The cost is that if the AP switching was from an AP with keys | |
694 | * to one without, we still allocate tailroom while it would no | |
695 | * longer be needed. However, in the typical (fast) roaming case | |
696 | * within an ESS this usually won't happen. | |
697 | */ | |
698 | ||
699 | mutex_lock(&sdata->local->key_mtx); | |
700 | sdata->crypto_tx_tailroom_needed_cnt -= | |
701 | sdata->crypto_tx_tailroom_pending_dec; | |
702 | sdata->crypto_tx_tailroom_pending_dec = 0; | |
703 | mutex_unlock(&sdata->local->key_mtx); | |
704 | } | |
c68f4b89 JB |
705 | |
706 | void ieee80211_gtk_rekey_notify(struct ieee80211_vif *vif, const u8 *bssid, | |
707 | const u8 *replay_ctr, gfp_t gfp) | |
708 | { | |
709 | struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif); | |
710 | ||
711 | trace_api_gtk_rekey_notify(sdata, bssid, replay_ctr); | |
712 | ||
713 | cfg80211_gtk_rekey_notify(sdata->dev, bssid, replay_ctr, gfp); | |
714 | } | |
715 | EXPORT_SYMBOL_GPL(ieee80211_gtk_rekey_notify); | |
3ea542d3 JB |
716 | |
717 | void ieee80211_get_key_tx_seq(struct ieee80211_key_conf *keyconf, | |
718 | struct ieee80211_key_seq *seq) | |
719 | { | |
720 | struct ieee80211_key *key; | |
721 | u64 pn64; | |
722 | ||
723 | if (WARN_ON(!(keyconf->flags & IEEE80211_KEY_FLAG_GENERATE_IV))) | |
724 | return; | |
725 | ||
726 | key = container_of(keyconf, struct ieee80211_key, conf); | |
727 | ||
728 | switch (key->conf.cipher) { | |
729 | case WLAN_CIPHER_SUITE_TKIP: | |
730 | seq->tkip.iv32 = key->u.tkip.tx.iv32; | |
731 | seq->tkip.iv16 = key->u.tkip.tx.iv16; | |
732 | break; | |
733 | case WLAN_CIPHER_SUITE_CCMP: | |
734 | pn64 = atomic64_read(&key->u.ccmp.tx_pn); | |
735 | seq->ccmp.pn[5] = pn64; | |
736 | seq->ccmp.pn[4] = pn64 >> 8; | |
737 | seq->ccmp.pn[3] = pn64 >> 16; | |
738 | seq->ccmp.pn[2] = pn64 >> 24; | |
739 | seq->ccmp.pn[1] = pn64 >> 32; | |
740 | seq->ccmp.pn[0] = pn64 >> 40; | |
741 | break; | |
742 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
743 | pn64 = atomic64_read(&key->u.aes_cmac.tx_pn); | |
744 | seq->ccmp.pn[5] = pn64; | |
745 | seq->ccmp.pn[4] = pn64 >> 8; | |
746 | seq->ccmp.pn[3] = pn64 >> 16; | |
747 | seq->ccmp.pn[2] = pn64 >> 24; | |
748 | seq->ccmp.pn[1] = pn64 >> 32; | |
749 | seq->ccmp.pn[0] = pn64 >> 40; | |
750 | break; | |
751 | default: | |
752 | WARN_ON(1); | |
753 | } | |
754 | } | |
755 | EXPORT_SYMBOL(ieee80211_get_key_tx_seq); | |
756 | ||
757 | void ieee80211_get_key_rx_seq(struct ieee80211_key_conf *keyconf, | |
758 | int tid, struct ieee80211_key_seq *seq) | |
759 | { | |
760 | struct ieee80211_key *key; | |
761 | const u8 *pn; | |
762 | ||
763 | key = container_of(keyconf, struct ieee80211_key, conf); | |
764 | ||
765 | switch (key->conf.cipher) { | |
766 | case WLAN_CIPHER_SUITE_TKIP: | |
5a306f58 | 767 | if (WARN_ON(tid < 0 || tid >= IEEE80211_NUM_TIDS)) |
3ea542d3 JB |
768 | return; |
769 | seq->tkip.iv32 = key->u.tkip.rx[tid].iv32; | |
770 | seq->tkip.iv16 = key->u.tkip.rx[tid].iv16; | |
771 | break; | |
772 | case WLAN_CIPHER_SUITE_CCMP: | |
5a306f58 | 773 | if (WARN_ON(tid < -1 || tid >= IEEE80211_NUM_TIDS)) |
3ea542d3 JB |
774 | return; |
775 | if (tid < 0) | |
5a306f58 | 776 | pn = key->u.ccmp.rx_pn[IEEE80211_NUM_TIDS]; |
3ea542d3 JB |
777 | else |
778 | pn = key->u.ccmp.rx_pn[tid]; | |
4325f6ca | 779 | memcpy(seq->ccmp.pn, pn, IEEE80211_CCMP_PN_LEN); |
3ea542d3 JB |
780 | break; |
781 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
782 | if (WARN_ON(tid != 0)) | |
783 | return; | |
784 | pn = key->u.aes_cmac.rx_pn; | |
4325f6ca | 785 | memcpy(seq->aes_cmac.pn, pn, IEEE80211_CMAC_PN_LEN); |
3ea542d3 JB |
786 | break; |
787 | } | |
788 | } | |
789 | EXPORT_SYMBOL(ieee80211_get_key_rx_seq); | |
27b3eb9c JB |
790 | |
791 | void ieee80211_set_key_tx_seq(struct ieee80211_key_conf *keyconf, | |
792 | struct ieee80211_key_seq *seq) | |
793 | { | |
794 | struct ieee80211_key *key; | |
795 | u64 pn64; | |
796 | ||
797 | key = container_of(keyconf, struct ieee80211_key, conf); | |
798 | ||
799 | switch (key->conf.cipher) { | |
800 | case WLAN_CIPHER_SUITE_TKIP: | |
801 | key->u.tkip.tx.iv32 = seq->tkip.iv32; | |
802 | key->u.tkip.tx.iv16 = seq->tkip.iv16; | |
803 | break; | |
804 | case WLAN_CIPHER_SUITE_CCMP: | |
805 | pn64 = (u64)seq->ccmp.pn[5] | | |
806 | ((u64)seq->ccmp.pn[4] << 8) | | |
807 | ((u64)seq->ccmp.pn[3] << 16) | | |
808 | ((u64)seq->ccmp.pn[2] << 24) | | |
809 | ((u64)seq->ccmp.pn[1] << 32) | | |
810 | ((u64)seq->ccmp.pn[0] << 40); | |
811 | atomic64_set(&key->u.ccmp.tx_pn, pn64); | |
812 | break; | |
813 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
814 | pn64 = (u64)seq->aes_cmac.pn[5] | | |
815 | ((u64)seq->aes_cmac.pn[4] << 8) | | |
816 | ((u64)seq->aes_cmac.pn[3] << 16) | | |
817 | ((u64)seq->aes_cmac.pn[2] << 24) | | |
818 | ((u64)seq->aes_cmac.pn[1] << 32) | | |
819 | ((u64)seq->aes_cmac.pn[0] << 40); | |
820 | atomic64_set(&key->u.aes_cmac.tx_pn, pn64); | |
821 | break; | |
822 | default: | |
823 | WARN_ON(1); | |
824 | break; | |
825 | } | |
826 | } | |
827 | EXPORT_SYMBOL_GPL(ieee80211_set_key_tx_seq); | |
828 | ||
829 | void ieee80211_set_key_rx_seq(struct ieee80211_key_conf *keyconf, | |
830 | int tid, struct ieee80211_key_seq *seq) | |
831 | { | |
832 | struct ieee80211_key *key; | |
833 | u8 *pn; | |
834 | ||
835 | key = container_of(keyconf, struct ieee80211_key, conf); | |
836 | ||
837 | switch (key->conf.cipher) { | |
838 | case WLAN_CIPHER_SUITE_TKIP: | |
839 | if (WARN_ON(tid < 0 || tid >= IEEE80211_NUM_TIDS)) | |
840 | return; | |
841 | key->u.tkip.rx[tid].iv32 = seq->tkip.iv32; | |
842 | key->u.tkip.rx[tid].iv16 = seq->tkip.iv16; | |
843 | break; | |
844 | case WLAN_CIPHER_SUITE_CCMP: | |
845 | if (WARN_ON(tid < -1 || tid >= IEEE80211_NUM_TIDS)) | |
846 | return; | |
847 | if (tid < 0) | |
848 | pn = key->u.ccmp.rx_pn[IEEE80211_NUM_TIDS]; | |
849 | else | |
850 | pn = key->u.ccmp.rx_pn[tid]; | |
851 | memcpy(pn, seq->ccmp.pn, IEEE80211_CCMP_PN_LEN); | |
852 | break; | |
853 | case WLAN_CIPHER_SUITE_AES_CMAC: | |
854 | if (WARN_ON(tid != 0)) | |
855 | return; | |
856 | pn = key->u.aes_cmac.rx_pn; | |
857 | memcpy(pn, seq->aes_cmac.pn, IEEE80211_CMAC_PN_LEN); | |
858 | break; | |
859 | default: | |
860 | WARN_ON(1); | |
861 | break; | |
862 | } | |
863 | } | |
864 | EXPORT_SYMBOL_GPL(ieee80211_set_key_rx_seq); | |
865 | ||
866 | void ieee80211_remove_key(struct ieee80211_key_conf *keyconf) | |
867 | { | |
868 | struct ieee80211_key *key; | |
869 | ||
870 | key = container_of(keyconf, struct ieee80211_key, conf); | |
871 | ||
872 | assert_key_lock(key->local); | |
873 | ||
874 | /* | |
875 | * if key was uploaded, we assume the driver will/has remove(d) | |
876 | * it, so adjust bookkeeping accordingly | |
877 | */ | |
878 | if (key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE) { | |
879 | key->flags &= ~KEY_FLAG_UPLOADED_TO_HARDWARE; | |
880 | ||
881 | if (!((key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_MMIC) || | |
882 | (key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_IV) || | |
883 | (key->conf.flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))) | |
884 | increment_tailroom_need_count(key->sdata); | |
885 | } | |
886 | ||
887 | ieee80211_key_free(key, false); | |
888 | } | |
889 | EXPORT_SYMBOL_GPL(ieee80211_remove_key); | |
890 | ||
891 | struct ieee80211_key_conf * | |
892 | ieee80211_gtk_rekey_add(struct ieee80211_vif *vif, | |
893 | struct ieee80211_key_conf *keyconf) | |
894 | { | |
895 | struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif); | |
896 | struct ieee80211_local *local = sdata->local; | |
897 | struct ieee80211_key *key; | |
898 | int err; | |
899 | ||
900 | if (WARN_ON(!local->wowlan)) | |
901 | return ERR_PTR(-EINVAL); | |
902 | ||
903 | if (WARN_ON(vif->type != NL80211_IFTYPE_STATION)) | |
904 | return ERR_PTR(-EINVAL); | |
905 | ||
906 | key = ieee80211_key_alloc(keyconf->cipher, keyconf->keyidx, | |
907 | keyconf->keylen, keyconf->key, | |
2475b1cc | 908 | 0, NULL, NULL); |
27b3eb9c | 909 | if (IS_ERR(key)) |
c5dc164d | 910 | return ERR_CAST(key); |
27b3eb9c JB |
911 | |
912 | if (sdata->u.mgd.mfp != IEEE80211_MFP_DISABLED) | |
913 | key->conf.flags |= IEEE80211_KEY_FLAG_RX_MGMT; | |
914 | ||
915 | err = ieee80211_key_link(key, sdata, NULL); | |
916 | if (err) | |
917 | return ERR_PTR(err); | |
918 | ||
919 | return &key->conf; | |
920 | } | |
921 | EXPORT_SYMBOL_GPL(ieee80211_gtk_rekey_add); |