projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: qat - change slice->regions to slice->region
[deliverable/linux.git] / drivers / crypto / qat / qat_common / qat_hal.c
CommitLineData
b3416fb8
TS		 1/*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46*/
47#include <linux/slab.h>
48
49#include "adf_accel_devices.h"
50#include "adf_common_drv.h"
51#include "icp_qat_hal.h"
52#include "icp_qat_uclo.h"
53
54#define BAD_REGADDR 0xffff
55#define MAX_RETRY_TIMES 10000
56#define INIT_CTX_ARB_VALUE 0x0
57#define INIT_CTX_ENABLE_VALUE 0x0
58#define INIT_PC_VALUE 0x0
59#define INIT_WAKEUP_EVENTS_VALUE 0x1
60#define INIT_SIG_EVENTS_VALUE 0x1
61#define INIT_CCENABLE_VALUE 0x2000
62#define RST_CSR_QAT_LSB 20
63#define RST_CSR_AE_LSB 0
64#define MC_TIMESTAMP_ENABLE (0x1 << 7)
65
66#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
67 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
68 (~(1 << CE_REG_PAR_ERR_BITPOS)))
69#define INSERT_IMMED_GPRA_CONST(inst, const_val) \
70 (inst = ((inst & 0xFFFF00C03FFull) | \
71 ((((const_val) << 12) & 0x0FF00000ull) | \
72 (((const_val) << 10) & 0x0003FC00ull))))
73#define INSERT_IMMED_GPRB_CONST(inst, const_val) \
74 (inst = ((inst & 0xFFFF00FFF00ull) | \
75 ((((const_val) << 12) & 0x0FF00000ull) | \
76 (((const_val) << 0) & 0x000000FFull))))
77
78#define AE(handle, ae) handle->hal_handle->aes[ae]
79
80static const uint64_t inst_4b[] = {
81 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
82 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
83 0x0A021000000ull
84};
85
86static const uint64_t inst[] = {
87 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
88 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
89 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
90 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
91 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
92 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
93 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
94 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
95 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
96 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
97 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
98 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
99 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
100 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
101 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
102 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
103 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
104 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
105 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
106 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
107 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
108 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
109};
110
111void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
112 unsigned char ae, unsigned int ctx_mask)
113{
114 AE(handle, ae).live_ctx_mask = ctx_mask;
115}
116
117#define CSR_RETRY_TIMES 500
118static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
119 unsigned char ae, unsigned int csr,
120 unsigned int *value)
121{
122 unsigned int iterations = CSR_RETRY_TIMES;
123
124 do {
125 *value = GET_AE_CSR(handle, ae, csr);
126 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
127 return 0;
128 } while (iterations--);
129
130 pr_err("QAT: Read CSR timeout\n");
131 return -EFAULT;
132}
133
134static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
135 unsigned char ae, unsigned int csr,
136 unsigned int value)
137{
138 unsigned int iterations = CSR_RETRY_TIMES;
139
140 do {
141 SET_AE_CSR(handle, ae, csr, value);
142 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
143 return 0;
144 } while (iterations--);
145
146 pr_err("QAT: Write CSR Timeout\n");
147 return -EFAULT;
148}
149
150static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
151 unsigned char ae, unsigned char ctx,
152 unsigned int *events)
153{
154 unsigned int cur_ctx;
155
156 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
157 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
158 qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
159 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
160}
161
162static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
163 unsigned char ae, unsigned int cycles,
164 int chk_inactive)
165{
166 unsigned int base_cnt = 0, cur_cnt = 0;
167 unsigned int csr = (1 << ACS_ABO_BITPOS);
168 int times = MAX_RETRY_TIMES;
169 int elapsed_cycles = 0;
170
171 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
172 base_cnt &= 0xffff;
173 while ((int)cycles > elapsed_cycles && times--) {
174 if (chk_inactive)
175 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
176
177 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
178 cur_cnt &= 0xffff;
179 elapsed_cycles = cur_cnt - base_cnt;
180
181 if (elapsed_cycles < 0)
182 elapsed_cycles += 0x10000;
183
184 /* ensure at least 8 time cycles elapsed in wait_cycles */
185 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
186 return 0;
187 }
188 if (!times) {
189 pr_err("QAT: wait_num_cycles time out\n");
190 return -EFAULT;
191 }
192 return 0;
193}
194
195#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
196#define SET_BIT(wrd, bit) (wrd | 1 << bit)
197
198int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
199 unsigned char ae, unsigned char mode)
200{
201 unsigned int csr, new_csr;
202
203 if ((mode != 4) && (mode != 8)) {
204 pr_err("QAT: bad ctx mode=%d\n", mode);
205 return -EINVAL;
206 }
207
208 /* Sets the accelaration engine context mode to either four or eight */
209 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
210 csr = IGNORE_W1C_MASK & csr;
211 new_csr = (mode == 4) ?
212 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
213 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
214 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
215 return 0;
216}
217
218int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
219 unsigned char ae, unsigned char mode)
220{
221 unsigned int csr, new_csr;
222
223 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
224 csr &= IGNORE_W1C_MASK;
225
226 new_csr = (mode) ?
227 SET_BIT(csr, CE_NN_MODE_BITPOS) :
228 CLR_BIT(csr, CE_NN_MODE_BITPOS);
229
230 if (new_csr != csr)
231 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
232
233 return 0;
234}
235
236int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
237 unsigned char ae, enum icp_qat_uof_regtype lm_type,
238 unsigned char mode)
239{
240 unsigned int csr, new_csr;
241
242 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
243 csr &= IGNORE_W1C_MASK;
244 switch (lm_type) {
245 case ICP_LMEM0:
246 new_csr = (mode) ?
247 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
248 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
249 break;
250 case ICP_LMEM1:
251 new_csr = (mode) ?
252 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
253 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
254 break;
255 default:
256 pr_err("QAT: lmType = 0x%x\n", lm_type);
257 return -EINVAL;
258 }
259
260 if (new_csr != csr)
261 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
262 return 0;
263}
264
265static unsigned short qat_hal_get_reg_addr(unsigned int type,
266 unsigned short reg_num)
267{
268 unsigned short reg_addr;
d65071ec 269
b3416fb8
TS		 270 switch (type) {
271 case ICP_GPA_ABS:
272 case ICP_GPB_ABS:
273 reg_addr = 0x80 | (reg_num & 0x7f);
274 break;
275 case ICP_GPA_REL:
276 case ICP_GPB_REL:
277 reg_addr = reg_num & 0x1f;
278 break;
279 case ICP_SR_RD_REL:
280 case ICP_SR_WR_REL:
281 case ICP_SR_REL:
282 reg_addr = 0x180 | (reg_num & 0x1f);
283 break;
284 case ICP_SR_ABS:
285 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
286 break;
287 case ICP_DR_RD_REL:
288 case ICP_DR_WR_REL:
289 case ICP_DR_REL:
290 reg_addr = 0x1c0 | (reg_num & 0x1f);
291 break;
292 case ICP_DR_ABS:
293 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
294 break;
295 case ICP_NEIGH_REL:
296 reg_addr = 0x280 | (reg_num & 0x1f);
297 break;
298 case ICP_LMEM0:
299 reg_addr = 0x200;
300 break;
301 case ICP_LMEM1:
302 reg_addr = 0x220;
303 break;
304 case ICP_NO_DEST:
305 reg_addr = 0x300 | (reg_num & 0xff);
306 break;
307 default:
308 reg_addr = BAD_REGADDR;
309 break;
310 }
311 return reg_addr;
312}
313
314void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
315{
316 unsigned int ae_reset_csr;
317
318 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
319 ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
320 ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
321 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
322}
323
324static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
325 unsigned char ae, unsigned int ctx_mask,
326 unsigned int ae_csr, unsigned int csr_val)
327{
328 unsigned int ctx, cur_ctx;
329
330 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
331
332 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
333 if (!(ctx_mask & (1 << ctx)))
334 continue;
335 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
336 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
337 }
338
339 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
340}
341
342static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
343 unsigned char ae, unsigned char ctx,
344 unsigned int ae_csr, unsigned int *csr_val)
345{
346 unsigned int cur_ctx;
347
348 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
349 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
350 qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
351 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
352}
353
354static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
355 unsigned char ae, unsigned int ctx_mask,
356 unsigned int events)
357{
358 unsigned int ctx, cur_ctx;
359
360 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
361 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
362 if (!(ctx_mask & (1 << ctx)))
363 continue;
364 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
365 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
366 }
367 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
368}
369
370static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
371 unsigned char ae, unsigned int ctx_mask,
372 unsigned int events)
373{
374 unsigned int ctx, cur_ctx;
375
376 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
377 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
378 if (!(ctx_mask & (1 << ctx)))
379 continue;
380 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
381 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
382 events);
383 }
384 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
385}
386
387static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
388{
389 unsigned int base_cnt, cur_cnt;
390 unsigned char ae;
391 unsigned int times = MAX_RETRY_TIMES;
392
393 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
394 if (!(handle->hal_handle->ae_mask & (1 << ae)))
395 continue;
396
397 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
398 (unsigned int *)&base_cnt);
399 base_cnt &= 0xffff;
400
401 do {
402 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
403 (unsigned int *)&cur_cnt);
404 cur_cnt &= 0xffff;
405 } while (times-- && (cur_cnt == base_cnt));
406
407 if (!times) {
408 pr_err("QAT: AE%d is inactive!!\n", ae);
409 return -EFAULT;
410 }
411 }
412
413 return 0;
414}
415
416static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
417{
418 unsigned int misc_ctl;
419 unsigned char ae;
420
421 /* stop the timestamp timers */
422 misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
423 if (misc_ctl & MC_TIMESTAMP_ENABLE)
424 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
425 (~MC_TIMESTAMP_ENABLE));
426
427 for (ae = 0; ae <= handle->hal_handle->ae_max_num; ae++) {
428 if (!(handle->hal_handle->ae_mask & (1 << ae)))
429 continue;
430 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
431 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
432 }
433 /* start timestamp timers */
434 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
435}
436
437#define ESRAM_AUTO_TINIT (1<<2)
438#define ESRAM_AUTO_TINIT_DONE (1<<3)
439#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
440#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
441static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
442{
443 void __iomem *csr_addr = handle->hal_ep_csr_addr_v +
444 ESRAM_AUTO_INIT_CSR_OFFSET;
445 unsigned int csr_val, times = 30;
446
447 csr_val = ADF_CSR_RD(csr_addr, 0);
448 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
449 return 0;
450
451 csr_val = ADF_CSR_RD(csr_addr, 0);
452 csr_val |= ESRAM_AUTO_TINIT;
453 ADF_CSR_WR(csr_addr, 0, csr_val);
454
455 do {
456 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
457 csr_val = ADF_CSR_RD(csr_addr, 0);
458 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
459 if ((!times)) {
460 pr_err("QAT: Fail to init eSram!\n");
461 return -EFAULT;
462 }
463 return 0;
464}
465
466#define SHRAM_INIT_CYCLES 2060
467int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
468{
469 unsigned int ae_reset_csr;
470 unsigned char ae;
471 unsigned int clk_csr;
472 unsigned int times = 100;
473 unsigned int csr;
474
475 /* write to the reset csr */
476 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
477 ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
478 ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
479 do {
480 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
481 if (!(times--))
482 goto out_err;
483 csr = GET_GLB_CSR(handle, ICP_RESET);
484 } while ((handle->hal_handle->ae_mask |
485 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
486 /* enable clock */
487 clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
488 clk_csr |= handle->hal_handle->ae_mask << 0;
489 clk_csr |= handle->hal_handle->slice_mask << 20;
490 SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
491 if (qat_hal_check_ae_alive(handle))
492 goto out_err;
493
494 /* Set undefined power-up/reset states to reasonable default values */
495 for (ae = 0; ae <= handle->hal_handle->ae_max_num; ae++) {
496 if (!(handle->hal_handle->ae_mask & (1 << ae)))
497 continue;
498 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
499 INIT_CTX_ENABLE_VALUE);
500 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
501 CTX_STS_INDIRECT,
502 handle->hal_handle->upc_mask &
503 INIT_PC_VALUE);
504 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
505 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
506 qat_hal_put_wakeup_event(handle, ae,
507 ICP_QAT_UCLO_AE_ALL_CTX,
508 INIT_WAKEUP_EVENTS_VALUE);
509 qat_hal_put_sig_event(handle, ae,
510 ICP_QAT_UCLO_AE_ALL_CTX,
511 INIT_SIG_EVENTS_VALUE);
512 }
513 if (qat_hal_init_esram(handle))
514 goto out_err;
515 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
516 goto out_err;
517 qat_hal_reset_timestamp(handle);
518
519 return 0;
520out_err:
521 pr_err("QAT: failed to get device out of reset\n");
522 return -EFAULT;
523}
524
525static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
526 unsigned char ae, unsigned int ctx_mask)
527{
528 unsigned int ctx;
529
530 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
531 ctx &= IGNORE_W1C_MASK &
532 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
533 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
534}
535
536static uint64_t qat_hal_parity_64bit(uint64_t word)
537{
538 word ^= word >> 1;
539 word ^= word >> 2;
540 word ^= word >> 4;
541 word ^= word >> 8;
542 word ^= word >> 16;
543 word ^= word >> 32;
544 return word & 1;
545}
546
547static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
548{
549 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
550 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
551 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
552 bit6_mask = 0xdaf69a46910ULL;
553
554 /* clear the ecc bits */
555 uword &= ~(0x7fULL << 0x2C);
556 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
557 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
558 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
559 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
560 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
561 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
562 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
563 return uword;
564}
565
566void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
567 unsigned char ae, unsigned int uaddr,
568 unsigned int words_num, uint64_t *uword)
569{
570 unsigned int ustore_addr;
571 unsigned int i;
572
573 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
574 uaddr |= UA_ECS;
575 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
576 for (i = 0; i < words_num; i++) {
577 unsigned int uwrd_lo, uwrd_hi;
578 uint64_t tmp;
579
580 tmp = qat_hal_set_uword_ecc(uword[i]);
581 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
582 uwrd_hi = (unsigned int)(tmp >> 0x20);
583 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
584 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
585 }
586 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
587}
588
589static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
590 unsigned char ae, unsigned int ctx_mask)
591{
592 unsigned int ctx;
593
594 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
595 ctx &= IGNORE_W1C_MASK;
596 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
597 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
598 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
599}
600
601static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
602{
603 unsigned char ae;
604 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
605 int times = MAX_RETRY_TIMES;
606 unsigned int csr_val = 0;
607 unsigned short reg;
608 unsigned int savctx = 0;
609 int ret = 0;
610
611 for (ae = 0; ae <= handle->hal_handle->ae_max_num; ae++) {
612 if (!(handle->hal_handle->ae_mask & (1 << ae)))
613 continue;
614 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
615 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
616 reg, 0);
617 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
618 reg, 0);
619 }
620 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
621 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
622 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
623 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
624 csr_val &= IGNORE_W1C_MASK;
625 csr_val |= CE_NN_MODE;
626 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
627 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
628 (uint64_t *)inst);
629 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
630 handle->hal_handle->upc_mask &
631 INIT_PC_VALUE);
632 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
633 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
634 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
635 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
636 CTX_SIG_EVENTS_INDIRECT, 0);
637 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
638 qat_hal_enable_ctx(handle, ae, ctx_mask);
639 }
640 for (ae = 0; ae <= handle->hal_handle->ae_max_num; ae++) {
641 if (!(handle->hal_handle->ae_mask & (1 << ae)))
642 continue;
643 /* wait for AE to finish */
644 do {
645 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
646 } while (ret && times--);
647
648 if (!times) {
649 pr_err("QAT: clear GPR of AE %d failed", ae);
650 return -EINVAL;
651 }
652 qat_hal_disable_ctx(handle, ae, ctx_mask);
653 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
654 savctx & ACS_ACNO);
655 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
656 INIT_CTX_ENABLE_VALUE);
657 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
658 handle->hal_handle->upc_mask &
659 INIT_PC_VALUE);
660 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
661 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
662 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
663 INIT_WAKEUP_EVENTS_VALUE);
664 qat_hal_put_sig_event(handle, ae, ctx_mask,
665 INIT_SIG_EVENTS_VALUE);
666 }
667 return 0;
668}
669
670#define ICP_DH895XCC_AE_OFFSET 0x20000
671#define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000)
672#define LOCAL_TO_XFER_REG_OFFSET 0x800
673#define ICP_DH895XCC_EP_OFFSET 0x3a000
674#define ICP_DH895XCC_PMISC_BAR 1
675int qat_hal_init(struct adf_accel_dev *accel_dev)
676{
677 unsigned char ae = 0;
678 unsigned int csr_val = 0;
679 unsigned int max_en_ae_num = 0;
680 struct icp_qat_fw_loader_handle *handle = NULL;
681 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
682 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
683 struct adf_bar *bar = &pci_info->pci_bars[ADF_DH895XCC_PMISC_BAR];
684
685 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
686 if (!handle)
687 goto out_handle;
688
689 handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr +
690 ICP_DH895XCC_CAP_OFFSET;
691 handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr +
692 ICP_DH895XCC_AE_OFFSET;
693 handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET;
694 handle->hal_cap_ae_local_csr_addr_v =
695 handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET;
696
697 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
698 if (!handle->hal_handle)
699 goto out_hal_handle;
700 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
701 handle->hal_handle->ae_mask = hw_data->ae_mask;
702 handle->hal_handle->slice_mask = hw_data->accel_mask;
703 /* create AE objects */
704 handle->hal_handle->upc_mask = 0x1ffff;
705 handle->hal_handle->max_ustore = 0x4000;
706 for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
707 if (!(hw_data->ae_mask & (1 << ae)))
708 continue;
709 handle->hal_handle->aes[ae].free_addr = 0;
710 handle->hal_handle->aes[ae].free_size =
711 handle->hal_handle->max_ustore;
712 handle->hal_handle->aes[ae].ustore_size =
713 handle->hal_handle->max_ustore;
714 handle->hal_handle->aes[ae].live_ctx_mask =
715 ICP_QAT_UCLO_AE_ALL_CTX;
716 max_en_ae_num = ae;
717 }
718 handle->hal_handle->ae_max_num = max_en_ae_num;
719 /* take all AEs out of reset */
720 if (qat_hal_clr_reset(handle)) {
721 pr_err("QAT: qat_hal_clr_reset error\n");
722 goto out_err;
723 }
724 if (qat_hal_clear_gpr(handle))
725 goto out_err;
726 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
727 for (ae = 0; ae <= handle->hal_handle->ae_max_num; ae++) {
728 if (!(hw_data->ae_mask & (1 << ae)))
729 continue;
730 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
731 csr_val |= 0x1;
732 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
733 }
734 accel_dev->fw_loader->fw_loader = handle;
735 return 0;
736
737out_err:
738 kfree(handle->hal_handle);
739out_hal_handle:
740 kfree(handle);
741out_handle:
742 return -EFAULT;
743}
744
745void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
746{
747 if (!handle)
748 return;
749 kfree(handle->hal_handle);
750 kfree(handle);
751}
752
753void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
754 unsigned int ctx_mask)
755{
756 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
757 ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
758 qat_hal_enable_ctx(handle, ae, ctx_mask);
759}
760
761void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
762 unsigned int ctx_mask)
763{
764 qat_hal_disable_ctx(handle, ae, ctx_mask);
765}
766
767void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
768 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
769{
770 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
771 handle->hal_handle->upc_mask & upc);
772}
773
774static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
775 unsigned char ae, unsigned int uaddr,
776 unsigned int words_num, uint64_t *uword)
777{
778 unsigned int i, uwrd_lo, uwrd_hi;
779 unsigned int ustore_addr, misc_control;
780
781 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
782 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
783 misc_control & 0xfffffffb);
784 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
785 uaddr |= UA_ECS;
786 for (i = 0; i < words_num; i++) {
787 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
788 uaddr++;
789 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
790 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
791 uword[i] = uwrd_hi;
792 uword[i] = (uword[i] << 0x20) | uwrd_lo;
793 }
794 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
795 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
796}
797
798static int qat_hal_count_bits(unsigned int word)
799{
800 int n = 0;
801
802 while (word) {
803 n++;
804 word &= word - 1;
805 }
806 return n;
807}
808
809void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
810 unsigned char ae, unsigned int uaddr,
811 unsigned int words_num, unsigned int *data)
812{
813 unsigned int i, ustore_addr;
814
815 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
816 uaddr |= UA_ECS;
817 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
818 for (i = 0; i < words_num; i++) {
819 unsigned int uwrd_lo, uwrd_hi, tmp;
d65071ec 820
b3416fb8
TS		 821 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
822 ((data[i] & 0xff00) << 2) |
823 (0x3 << 8) | (data[i] & 0xff);
824 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
825 uwrd_hi |= (qat_hal_count_bits(data[i] & 0xffff) & 0x1) << 8;
826 tmp = ((data[i] >> 0x10) & 0xffff);
827 uwrd_hi |= (qat_hal_count_bits(tmp) & 0x1) << 9;
828 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
829 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
830 }
831 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
832}
833
834#define MAX_EXEC_INST 100
835static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
836 unsigned char ae, unsigned char ctx,
837 uint64_t *micro_inst, unsigned int inst_num,
838 int code_off, unsigned int max_cycle,
839 unsigned int *endpc)
840{
841 uint64_t savuwords[MAX_EXEC_INST];
842 unsigned int ind_lm_addr0, ind_lm_addr1;
843 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
844 unsigned int ind_cnt_sig;
845 unsigned int ind_sig, act_sig;
846 unsigned int csr_val = 0, newcsr_val;
847 unsigned int savctx;
848 unsigned int savcc, wakeup_events, savpc;
849 unsigned int ctxarb_ctl, ctx_enables;
850
851 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
852 pr_err("QAT: invalid instructs inst_num=%d, micro_inst=0x%p\n ",
ac1a2b49 853 inst_num, micro_inst);
b3416fb8
TS		 854 return -EINVAL;
855 }
856 /* save current context */
857 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
858 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
859 qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
860 &ind_lm_addr_byte0);
861 qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
862 &ind_lm_addr_byte1);
863 if (inst_num <= MAX_EXEC_INST)
864 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
865 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
866 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
867 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
868 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
869 ctx_enables &= IGNORE_W1C_MASK;
870 qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
871 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
872 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
873 qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
874 &ind_cnt_sig);
875 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
876 qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
877 /* execute micro codes */
878 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
879 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
880 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
881 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
882 if (code_off)
883 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
884 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
885 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
886 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
887 qat_hal_enable_ctx(handle, ae, (1 << ctx));
888 /* wait for micro codes to finish */
889 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
890 return -EFAULT;
891 if (endpc) {
892 unsigned int ctx_status;
d65071ec 893
b3416fb8
TS		 894 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT,
895 &ctx_status);
896 *endpc = ctx_status & handle->hal_handle->upc_mask;
897 }
898 /* retore to saved context */
899 qat_hal_disable_ctx(handle, ae, (1 << ctx));
900 if (inst_num <= MAX_EXEC_INST)
901 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
902 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
903 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
904 handle->hal_handle->upc_mask & savpc);
905 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
906 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
907 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
908 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
909 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
910 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
911 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
912 LM_ADDR_0_INDIRECT, ind_lm_addr0);
913 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
914 LM_ADDR_1_INDIRECT, ind_lm_addr1);
915 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
916 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
917 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
918 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
919 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
920 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
921 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
922 CTX_SIG_EVENTS_INDIRECT, ind_sig);
923 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
924 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
925
926 return 0;
927}
928
929static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
930 unsigned char ae, unsigned char ctx,
931 enum icp_qat_uof_regtype reg_type,
932 unsigned short reg_num, unsigned int *data)
933{
934 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
935 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
936 unsigned short reg_addr;
937 int status = 0;
938 uint64_t insts, savuword;
939
940 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
941 if (reg_addr == BAD_REGADDR) {
942 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
943 return -EINVAL;
944 }
945 switch (reg_type) {
946 case ICP_GPA_REL:
947 insts = 0xA070000000ull | (reg_addr & 0x3ff);
948 break;
949 default:
950 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
951 break;
952 }
953 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
954 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
955 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
956 ctx_enables &= IGNORE_W1C_MASK;
957 if (ctx != (savctx & ACS_ACNO))
958 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
959 ctx & ACS_ACNO);
960 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
961 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
962 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
963 uaddr = UA_ECS;
964 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
965 insts = qat_hal_set_uword_ecc(insts);
966 uwrd_lo = (unsigned int)(insts & 0xffffffff);
967 uwrd_hi = (unsigned int)(insts >> 0x20);
968 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
969 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
970 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
971 /* delay for at least 8 cycles */
972 qat_hal_wait_cycles(handle, ae, 0x8, 0);
973 /*
974 * read ALU output
975 * the instruction should have been executed
976 * prior to clearing the ECS in putUwords
977 */
978 qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
979 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
980 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
981 if (ctx != (savctx & ACS_ACNO))
982 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
983 savctx & ACS_ACNO);
984 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
985 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
986
987 return status;
988}
989
990static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
991 unsigned char ae, unsigned char ctx,
992 enum icp_qat_uof_regtype reg_type,
993 unsigned short reg_num, unsigned int data)
994{
995 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
996 uint64_t insts[] = {
997 0x0F440000000ull,
998 0x0F040000000ull,
999 0x0F0000C0300ull,
1000 0x0E000010000ull
1001 };
1002 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1003 const int imm_w1 = 0, imm_w0 = 1;
1004
1005 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1006 if (dest_addr == BAD_REGADDR) {
1007 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1008 return -EINVAL;
1009 }
1010
1011 data16lo = 0xffff & data;
1012 data16hi = 0xffff & (data >> 0x10);
1013 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1014 (0xff & data16hi));
1015 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1016 (0xff & data16lo));
1017 switch (reg_type) {
1018 case ICP_GPA_REL:
1019 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1020 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1021 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1022 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1023 break;
1024 default:
1025 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1026 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1027
1028 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1029 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1030 break;
1031 }
1032
1033 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1034 code_off, num_inst * 0x5, NULL);
1035}
1036
1037int qat_hal_get_ins_num(void)
1038{
1039 return ARRAY_SIZE(inst_4b);
1040}
1041
1042static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1043 unsigned int inst_num, unsigned int size,
1044 unsigned int addr, unsigned int *value)
1045{
1046 int i, val_indx;
1047 unsigned int cur_value;
1048 const uint64_t *inst_arr;
1049 int fixup_offset;
1050 int usize = 0;
1051 int orig_num;
1052
1053 orig_num = inst_num;
1054 val_indx = 0;
1055 cur_value = value[val_indx++];
1056 inst_arr = inst_4b;
1057 usize = ARRAY_SIZE(inst_4b);
1058 fixup_offset = inst_num;
1059 for (i = 0; i < usize; i++)
1060 micro_inst[inst_num++] = inst_arr[i];
1061 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1062 fixup_offset++;
1063 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1064 fixup_offset++;
1065 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1066 fixup_offset++;
1067 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1068
1069 return inst_num - orig_num;
1070}
1071
1072static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1073 unsigned char ae, unsigned char ctx,
1074 int *pfirst_exec, uint64_t *micro_inst,
1075 unsigned int inst_num)
1076{
1077 int stat = 0;
1078 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1079 unsigned int gprb0 = 0, gprb1 = 0;
1080
1081 if (*pfirst_exec) {
1082 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1083 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1084 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1085 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1086 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1087 *pfirst_exec = 0;
1088 }
1089 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1090 inst_num * 0x5, NULL);
1091 if (stat != 0)
1092 return -EFAULT;
1093 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1094 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1095 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1096 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1097 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1098
1099 return 0;
1100}
1101
1102int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1103 unsigned char ae,
1104 struct icp_qat_uof_batch_init *lm_init_header)
1105{
1106 struct icp_qat_uof_batch_init *plm_init;
1107 uint64_t *micro_inst_arry;
1108 int micro_inst_num;
1109 int alloc_inst_size;
1110 int first_exec = 1;
1111 int stat = 0;
1112
1113 plm_init = lm_init_header->next;
1114 alloc_inst_size = lm_init_header->size;
1115 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1116 alloc_inst_size = handle->hal_handle->max_ustore;
d65071ec
TS		 1117 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1118 GFP_KERNEL);
b3416fb8
TS		 1119 if (!micro_inst_arry)
1120 return -ENOMEM;
1121 micro_inst_num = 0;
1122 while (plm_init) {
1123 unsigned int addr, *value, size;
1124
1125 ae = plm_init->ae;
1126 addr = plm_init->addr;
1127 value = plm_init->value;
1128 size = plm_init->size;
1129 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1130 micro_inst_num,
1131 size, addr, value);
1132 plm_init = plm_init->next;
1133 }
1134 /* exec micro codes */
1135 if (micro_inst_arry && (micro_inst_num > 0)) {
1136 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1137 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1138 micro_inst_arry,
1139 micro_inst_num);
1140 }
1141 kfree(micro_inst_arry);
1142 return stat;
1143}
1144
1145static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1146 unsigned char ae, unsigned char ctx,
1147 enum icp_qat_uof_regtype reg_type,
1148 unsigned short reg_num, unsigned int val)
1149{
1150 int status = 0;
1151 unsigned int reg_addr;
1152 unsigned int ctx_enables;
1153 unsigned short mask;
1154 unsigned short dr_offset = 0x10;
1155
1156 status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1157 if (CE_INUSE_CONTEXTS & ctx_enables) {
1158 if (ctx & 0x1) {
1159 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1160 return -EINVAL;
1161 }
1162 mask = 0x1f;
1163 dr_offset = 0x20;
1164 } else {
1165 mask = 0x0f;
1166 }
1167 if (reg_num & ~mask)
1168 return -EINVAL;
1169 reg_addr = reg_num + (ctx << 0x5);
1170 switch (reg_type) {
1171 case ICP_SR_RD_REL:
1172 case ICP_SR_REL:
1173 SET_AE_XFER(handle, ae, reg_addr, val);
1174 break;
1175 case ICP_DR_RD_REL:
1176 case ICP_DR_REL:
1177 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1178 break;
1179 default:
1180 status = -EINVAL;
1181 break;
1182 }
1183 return status;
1184}
1185
1186static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1187 unsigned char ae, unsigned char ctx,
1188 enum icp_qat_uof_regtype reg_type,
1189 unsigned short reg_num, unsigned int data)
1190{
1191 unsigned int gprval, ctx_enables;
1192 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1193 data16low;
1194 unsigned short reg_mask;
1195 int status = 0;
1196 uint64_t micro_inst[] = {
1197 0x0F440000000ull,
1198 0x0F040000000ull,
1199 0x0A000000000ull,
1200 0x0F0000C0300ull,
1201 0x0E000010000ull
1202 };
1203 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1204 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1205
1206 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1207 if (CE_INUSE_CONTEXTS & ctx_enables) {
1208 if (ctx & 0x1) {
1209 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1210 return -EINVAL;
1211 }
1212 reg_mask = (unsigned short)~0x1f;
1213 } else {
1214 reg_mask = (unsigned short)~0xf;
1215 }
1216 if (reg_num & reg_mask)
1217 return -EINVAL;
1218 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1219 if (xfr_addr == BAD_REGADDR) {
1220 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1221 return -EINVAL;
1222 }
1223 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1224 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1225 data16low = 0xffff & data;
1226 data16hi = 0xffff & (data >> 0x10);
1227 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1228 (unsigned short)(0xff & data16hi));
1229 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1230 (unsigned short)(0xff & data16low));
1231 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1232 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1233 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1234 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1235 micro_inst[0x2] = micro_inst[0x2] |
1236 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1237 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1238 code_off, dly, NULL);
1239 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1240 return status;
1241}
1242
1243static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1244 unsigned char ae, unsigned char ctx,
1245 unsigned short nn, unsigned int val)
1246{
1247 unsigned int ctx_enables;
1248 int stat = 0;
1249
1250 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1251 ctx_enables &= IGNORE_W1C_MASK;
1252 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1253
1254 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1255 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1256 return stat;
1257}
1258
1259static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1260 *handle, unsigned char ae,
1261 unsigned short absreg_num,
1262 unsigned short *relreg,
1263 unsigned char *ctx)
1264{
1265 unsigned int ctx_enables;
1266
1267 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1268 if (ctx_enables & CE_INUSE_CONTEXTS) {
1269 /* 4-ctx mode */
1270 *relreg = absreg_num & 0x1F;
1271 *ctx = (absreg_num >> 0x4) & 0x6;
1272 } else {
1273 /* 8-ctx mode */
1274 *relreg = absreg_num & 0x0F;
1275 *ctx = (absreg_num >> 0x4) & 0x7;
1276 }
1277 return 0;
1278}
1279
1280int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1281 unsigned char ae, unsigned char ctx_mask,
1282 enum icp_qat_uof_regtype reg_type,
1283 unsigned short reg_num, unsigned int regdata)
1284{
1285 int stat = 0;
1286 unsigned short reg;
1287 unsigned char ctx = 0;
1288 enum icp_qat_uof_regtype type;
1289
1290 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1291 return -EINVAL;
1292
1293 do {
1294 if (ctx_mask == 0) {
1295 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1296 &ctx);
1297 type = reg_type - 1;
1298 } else {
1299 reg = reg_num;
1300 type = reg_type;
1301 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1302 continue;
1303 }
1304 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1305 if (stat) {
1306 pr_err("QAT: write gpr fail\n");
1307 return -EINVAL;
1308 }
1309 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1310
1311 return 0;
1312}
1313
1314int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1315 unsigned char ae, unsigned char ctx_mask,
1316 enum icp_qat_uof_regtype reg_type,
1317 unsigned short reg_num, unsigned int regdata)
1318{
1319 int stat = 0;
1320 unsigned short reg;
1321 unsigned char ctx = 0;
1322 enum icp_qat_uof_regtype type;
1323
1324 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1325 return -EINVAL;
1326
1327 do {
1328 if (ctx_mask == 0) {
1329 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1330 &ctx);
1331 type = reg_type - 3;
1332 } else {
1333 reg = reg_num;
1334 type = reg_type;
1335 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1336 continue;
1337 }
1338 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1339 regdata);
1340 if (stat) {
1341 pr_err("QAT: write wr xfer fail\n");
1342 return -EINVAL;
1343 }
1344 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1345
1346 return 0;
1347}
1348
1349int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1350 unsigned char ae, unsigned char ctx_mask,
1351 enum icp_qat_uof_regtype reg_type,
1352 unsigned short reg_num, unsigned int regdata)
1353{
1354 int stat = 0;
1355 unsigned short reg;
1356 unsigned char ctx = 0;
1357 enum icp_qat_uof_regtype type;
1358
1359 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1360 return -EINVAL;
1361
1362 do {
1363 if (ctx_mask == 0) {
1364 qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1365 &ctx);
1366 type = reg_type - 3;
1367 } else {
1368 reg = reg_num;
1369 type = reg_type;
1370 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1371 continue;
1372 }
1373 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1374 regdata);
1375 if (stat) {
1376 pr_err("QAT: write rd xfer fail\n");
1377 return -EINVAL;
1378 }
1379 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1380
1381 return 0;
1382}
1383
1384int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1385 unsigned char ae, unsigned char ctx_mask,
1386 unsigned short reg_num, unsigned int regdata)
1387{
1388 int stat = 0;
1389 unsigned char ctx;
1390
1391 if (ctx_mask == 0)
1392 return -EINVAL;
1393
1394 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1395 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1396 continue;
1397 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1398 if (stat) {
1399 pr_err("QAT: write neigh error\n");
1400 return -EINVAL;
1401 }
1402 }
1403
1404 return 0;
1405}
Linux kernel - Deliverables
This page took 0.07722 seconds and 5 git commands to generate.