4 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #include <linux/dma-mapping.h>
20 #include <linux/err.h>
21 #include <linux/kref.h>
22 #include <linux/module.h>
23 #include <linux/scatterlist.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <trace/events/host1x.h>
30 #include "host1x_bo.h"
34 struct host1x_job
*host1x_job_alloc(struct host1x_channel
*ch
,
35 u32 num_cmdbufs
, u32 num_relocs
,
38 struct host1x_job
*job
= NULL
;
39 unsigned int num_unpins
= num_cmdbufs
+ num_relocs
;
43 /* Check that we're not going to overflow */
44 total
= sizeof(struct host1x_job
) +
45 num_relocs
* sizeof(struct host1x_reloc
) +
46 num_unpins
* sizeof(struct host1x_job_unpin_data
) +
47 num_waitchks
* sizeof(struct host1x_waitchk
) +
48 num_cmdbufs
* sizeof(struct host1x_job_gather
) +
49 num_unpins
* sizeof(dma_addr_t
) +
50 num_unpins
* sizeof(u32
*);
51 if (total
> ULONG_MAX
)
54 mem
= job
= kzalloc(total
, GFP_KERNEL
);
61 /* Redistribute memory to the structs */
62 mem
+= sizeof(struct host1x_job
);
63 job
->relocarray
= num_relocs
? mem
: NULL
;
64 mem
+= num_relocs
* sizeof(struct host1x_reloc
);
65 job
->unpins
= num_unpins
? mem
: NULL
;
66 mem
+= num_unpins
* sizeof(struct host1x_job_unpin_data
);
67 job
->waitchk
= num_waitchks
? mem
: NULL
;
68 mem
+= num_waitchks
* sizeof(struct host1x_waitchk
);
69 job
->gathers
= num_cmdbufs
? mem
: NULL
;
70 mem
+= num_cmdbufs
* sizeof(struct host1x_job_gather
);
71 job
->addr_phys
= num_unpins
? mem
: NULL
;
73 job
->reloc_addr_phys
= job
->addr_phys
;
74 job
->gather_addr_phys
= &job
->addr_phys
[num_relocs
];
79 struct host1x_job
*host1x_job_get(struct host1x_job
*job
)
85 static void job_free(struct kref
*ref
)
87 struct host1x_job
*job
= container_of(ref
, struct host1x_job
, ref
);
92 void host1x_job_put(struct host1x_job
*job
)
94 kref_put(&job
->ref
, job_free
);
97 void host1x_job_add_gather(struct host1x_job
*job
, struct host1x_bo
*bo
,
98 u32 words
, u32 offset
)
100 struct host1x_job_gather
*cur_gather
= &job
->gathers
[job
->num_gathers
];
102 cur_gather
->words
= words
;
104 cur_gather
->offset
= offset
;
109 * NULL an already satisfied WAIT_SYNCPT host method, by patching its
110 * args in the command stream. The method data is changed to reference
111 * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
112 * with a matching threshold value of 0, so is guaranteed to be popped
115 static void host1x_syncpt_patch_offset(struct host1x_syncpt
*sp
,
116 struct host1x_bo
*h
, u32 offset
)
118 void *patch_addr
= NULL
;
121 patch_addr
= host1x_bo_kmap(h
, offset
>> PAGE_SHIFT
);
123 host1x_syncpt_patch_wait(sp
,
124 patch_addr
+ (offset
& ~PAGE_MASK
));
125 host1x_bo_kunmap(h
, offset
>> PAGE_SHIFT
, patch_addr
);
127 pr_err("Could not map cmdbuf for wait check\n");
131 * Check driver supplied waitchk structs for syncpt thresholds
132 * that have already been satisfied and NULL the comparison (to
133 * avoid a wrap condition in the HW).
135 static int do_waitchks(struct host1x_job
*job
, struct host1x
*host
,
136 struct host1x_bo
*patch
)
140 /* compare syncpt vs wait threshold */
141 for (i
= 0; i
< job
->num_waitchk
; i
++) {
142 struct host1x_waitchk
*wait
= &job
->waitchk
[i
];
143 struct host1x_syncpt
*sp
=
144 host1x_syncpt_get(host
, wait
->syncpt_id
);
146 /* validate syncpt id */
147 if (wait
->syncpt_id
> host1x_syncpt_nb_pts(host
))
150 /* skip all other gathers */
151 if (patch
!= wait
->bo
)
154 trace_host1x_syncpt_wait_check(wait
->bo
, wait
->offset
,
155 wait
->syncpt_id
, wait
->thresh
,
156 host1x_syncpt_read_min(sp
));
158 if (host1x_syncpt_is_expired(sp
, wait
->thresh
)) {
160 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
161 wait
->syncpt_id
, sp
->name
, wait
->thresh
,
162 host1x_syncpt_read_min(sp
));
164 host1x_syncpt_patch_offset(sp
, patch
, wait
->offset
);
173 static unsigned int pin_job(struct host1x_job
*job
)
179 for (i
= 0; i
< job
->num_relocs
; i
++) {
180 struct host1x_reloc
*reloc
= &job
->relocarray
[i
];
181 struct sg_table
*sgt
;
182 dma_addr_t phys_addr
;
184 reloc
->target
= host1x_bo_get(reloc
->target
);
188 phys_addr
= host1x_bo_pin(reloc
->target
, &sgt
);
192 job
->addr_phys
[job
->num_unpins
] = phys_addr
;
193 job
->unpins
[job
->num_unpins
].bo
= reloc
->target
;
194 job
->unpins
[job
->num_unpins
].sgt
= sgt
;
198 for (i
= 0; i
< job
->num_gathers
; i
++) {
199 struct host1x_job_gather
*g
= &job
->gathers
[i
];
200 struct sg_table
*sgt
;
201 dma_addr_t phys_addr
;
203 g
->bo
= host1x_bo_get(g
->bo
);
207 phys_addr
= host1x_bo_pin(g
->bo
, &sgt
);
211 job
->addr_phys
[job
->num_unpins
] = phys_addr
;
212 job
->unpins
[job
->num_unpins
].bo
= g
->bo
;
213 job
->unpins
[job
->num_unpins
].sgt
= sgt
;
217 return job
->num_unpins
;
220 host1x_job_unpin(job
);
224 static unsigned int do_relocs(struct host1x_job
*job
, struct host1x_bo
*cmdbuf
)
228 void *cmdbuf_page_addr
= NULL
;
230 /* pin & patch the relocs for one gather */
231 while (i
< job
->num_relocs
) {
232 struct host1x_reloc
*reloc
= &job
->relocarray
[i
];
233 u32 reloc_addr
= (job
->reloc_addr_phys
[i
] +
234 reloc
->target_offset
) >> reloc
->shift
;
237 /* skip all other gathers */
238 if (!(reloc
->cmdbuf
&& cmdbuf
== reloc
->cmdbuf
)) {
243 if (last_page
!= reloc
->cmdbuf_offset
>> PAGE_SHIFT
) {
244 if (cmdbuf_page_addr
)
245 host1x_bo_kunmap(cmdbuf
, last_page
,
248 cmdbuf_page_addr
= host1x_bo_kmap(cmdbuf
,
249 reloc
->cmdbuf_offset
>> PAGE_SHIFT
);
250 last_page
= reloc
->cmdbuf_offset
>> PAGE_SHIFT
;
252 if (unlikely(!cmdbuf_page_addr
)) {
253 pr_err("Could not map cmdbuf for relocation\n");
258 target
= cmdbuf_page_addr
+ (reloc
->cmdbuf_offset
& ~PAGE_MASK
);
259 *target
= reloc_addr
;
261 /* mark this gather as handled */
265 if (cmdbuf_page_addr
)
266 host1x_bo_kunmap(cmdbuf
, last_page
, cmdbuf_page_addr
);
271 static int check_reloc(struct host1x_reloc
*reloc
, struct host1x_bo
*cmdbuf
,
274 offset
*= sizeof(u32
);
276 if (reloc
->cmdbuf
!= cmdbuf
|| reloc
->cmdbuf_offset
!= offset
)
282 struct host1x_firewall
{
283 struct host1x_job
*job
;
286 unsigned int num_relocs
;
287 struct host1x_reloc
*reloc
;
289 struct host1x_bo
*cmdbuf_id
;
299 static int check_mask(struct host1x_firewall
*fw
)
309 if (fw
->job
->is_addr_reg(fw
->dev
, fw
->class, reg
)) {
310 bool bad_reloc
= check_reloc(fw
->reloc
,
313 if (!fw
->num_relocs
|| bad_reloc
)
328 static int check_incr(struct host1x_firewall
*fw
)
330 u32 count
= fw
->count
;
337 if (fw
->job
->is_addr_reg(fw
->dev
, fw
->class, reg
)) {
338 bool bad_reloc
= check_reloc(fw
->reloc
, fw
->cmdbuf_id
,
340 if (!fw
->num_relocs
|| bad_reloc
)
354 static int check_nonincr(struct host1x_firewall
*fw
)
356 int is_addr_reg
= fw
->job
->is_addr_reg(fw
->dev
, fw
->class, fw
->reg
);
357 u32 count
= fw
->count
;
364 bool bad_reloc
= check_reloc(fw
->reloc
, fw
->cmdbuf_id
,
366 if (!fw
->num_relocs
|| bad_reloc
)
379 static int validate(struct host1x_job
*job
, struct device
*dev
,
380 struct host1x_job_gather
*g
)
384 struct host1x_firewall fw
;
388 fw
.reloc
= job
->relocarray
;
389 fw
.num_relocs
= job
->num_relocs
;
390 fw
.cmdbuf_id
= g
->bo
;
395 if (!job
->is_addr_reg
)
398 cmdbuf_base
= host1x_bo_mmap(g
->bo
);
403 while (fw
.words
&& !err
) {
404 u32 word
= cmdbuf_base
[fw
.offset
];
405 u32 opcode
= (word
& 0xf0000000) >> 28;
415 fw
.class = word
>> 6 & 0x3ff;
416 fw
.mask
= word
& 0x3f;
417 fw
.reg
= word
>> 16 & 0xfff;
418 err
= check_mask(&fw
);
423 fw
.reg
= word
>> 16 & 0xfff;
424 fw
.count
= word
& 0xffff;
425 err
= check_incr(&fw
);
431 fw
.reg
= word
>> 16 & 0xfff;
432 fw
.count
= word
& 0xffff;
433 err
= check_nonincr(&fw
);
439 fw
.mask
= word
& 0xffff;
440 fw
.reg
= word
>> 16 & 0xfff;
441 err
= check_mask(&fw
);
455 /* No relocs should remain at this point */
460 host1x_bo_munmap(g
->bo
, cmdbuf_base
);
465 static inline int copy_gathers(struct host1x_job
*job
, struct device
*dev
)
471 for (i
= 0; i
< job
->num_gathers
; i
++) {
472 struct host1x_job_gather
*g
= &job
->gathers
[i
];
473 size
+= g
->words
* sizeof(u32
);
476 job
->gather_copy_mapped
= dma_alloc_writecombine(dev
, size
,
479 if (!job
->gather_copy_mapped
) {
480 int err
= PTR_ERR(job
->gather_copy_mapped
);
481 job
->gather_copy_mapped
= NULL
;
485 job
->gather_copy_size
= size
;
487 for (i
= 0; i
< job
->num_gathers
; i
++) {
488 struct host1x_job_gather
*g
= &job
->gathers
[i
];
491 gather
= host1x_bo_mmap(g
->bo
);
492 memcpy(job
->gather_copy_mapped
+ offset
, gather
+ g
->offset
,
493 g
->words
* sizeof(u32
));
494 host1x_bo_munmap(g
->bo
, gather
);
496 g
->base
= job
->gather_copy
;
500 offset
+= g
->words
* sizeof(u32
);
506 int host1x_job_pin(struct host1x_job
*job
, struct device
*dev
)
510 struct host1x
*host
= dev_get_drvdata(dev
->parent
);
511 DECLARE_BITMAP(waitchk_mask
, host1x_syncpt_nb_pts(host
));
513 bitmap_zero(waitchk_mask
, host1x_syncpt_nb_pts(host
));
514 for (i
= 0; i
< job
->num_waitchk
; i
++) {
515 u32 syncpt_id
= job
->waitchk
[i
].syncpt_id
;
516 if (syncpt_id
< host1x_syncpt_nb_pts(host
))
517 set_bit(syncpt_id
, waitchk_mask
);
520 /* get current syncpt values for waitchk */
521 for_each_set_bit(i
, waitchk_mask
, host1x_syncpt_nb_pts(host
))
522 host1x_syncpt_load(host
->syncpt
+ i
);
530 for (i
= 0; i
< job
->num_gathers
; i
++) {
531 struct host1x_job_gather
*g
= &job
->gathers
[i
];
533 /* process each gather mem only once */
537 g
->base
= job
->gather_addr_phys
[i
];
539 for (j
= 0; j
< job
->num_gathers
; j
++)
540 if (job
->gathers
[j
].bo
== g
->bo
)
541 job
->gathers
[j
].handled
= true;
545 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL
))
546 err
= validate(job
, dev
, g
);
549 dev_err(dev
, "Job invalid (err=%d)\n", err
);
552 err
= do_relocs(job
, g
->bo
);
555 err
= do_waitchks(job
, host
, g
->bo
);
561 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL
) && !err
) {
562 err
= copy_gathers(job
, dev
);
564 host1x_job_unpin(job
);
575 void host1x_job_unpin(struct host1x_job
*job
)
579 for (i
= 0; i
< job
->num_unpins
; i
++) {
580 struct host1x_job_unpin_data
*unpin
= &job
->unpins
[i
];
581 host1x_bo_unpin(unpin
->bo
, unpin
->sgt
);
582 host1x_bo_put(unpin
->bo
);
586 if (job
->gather_copy_size
)
587 dma_free_writecombine(job
->channel
->dev
, job
->gather_copy_size
,
588 job
->gather_copy_mapped
,
593 * Debug routine used to dump job entries
595 void host1x_job_dump(struct device
*dev
, struct host1x_job
*job
)
597 dev_dbg(dev
, " SYNCPT_ID %d\n", job
->syncpt_id
);
598 dev_dbg(dev
, " SYNCPT_VAL %d\n", job
->syncpt_end
);
599 dev_dbg(dev
, " FIRST_GET 0x%x\n", job
->first_get
);
600 dev_dbg(dev
, " TIMEOUT %d\n", job
->timeout
);
601 dev_dbg(dev
, " NUM_SLOTS %d\n", job
->num_slots
);
602 dev_dbg(dev
, " NUM_HANDLES %d\n", job
->num_unpins
);