drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include <drm/drmP.h>
  28
  29 #include <linux/ktime.h>
  30 #include <linux/hrtimer.h>
  31 #include <trace/events/fence.h>
  32
  33 #include <nvif/notify.h>
  34 #include <nvif/event.h>
  35
  36 #include "nouveau_drm.h"
  37 #include "nouveau_dma.h"
  38 #include "nouveau_fence.h"
  39
  40 static const struct fence_ops nouveau_fence_ops_uevent;
  41 static const struct fence_ops nouveau_fence_ops_legacy;
  42
  43 static inline struct nouveau_fence *
  44 from_fence(struct fence *fence)
  45 {
  46         return container_of(fence, struct nouveau_fence, base);
  47 }
  48
  49 static inline struct nouveau_fence_chan *
  50 nouveau_fctx(struct nouveau_fence *fence)
  51 {
  52         return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
  53 }
  54
  55 static void
  56 nouveau_fence_signal(struct nouveau_fence *fence)
  57 {
  58         fence_signal_locked(&fence->base);
  59         list_del(&fence->head);
  60
  61         if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
  62                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  63
  64                 if (!--fctx->notify_ref)
  65                         nvif_notify_put(&fctx->notify);
  66         }
  67
  68         fence_put(&fence->base);
  69 }
  70
  71 static struct nouveau_fence *
  72 nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
  73         struct nouveau_fence_priv *priv = (void*)drm->fence;
  74
  75         if (fence->ops != &nouveau_fence_ops_legacy &&
  76             fence->ops != &nouveau_fence_ops_uevent)
  77                 return NULL;
  78
  79         if (fence->context < priv->context_base ||
  80             fence->context >= priv->context_base + priv->contexts)
  81                 return NULL;
  82
  83         return from_fence(fence);
  84 }
  85
  86 void
  87 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
  88 {
  89         struct nouveau_fence *fence;
  90
  91         nvif_notify_fini(&fctx->notify);
  92
  93         spin_lock_irq(&fctx->lock);
  94         while (!list_empty(&fctx->pending)) {
  95                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
  96
  97                 nouveau_fence_signal(fence);
  98                 fence->channel = NULL;
  99         }
 100         spin_unlock_irq(&fctx->lock);
 101 }
 102
 103 static void
 104 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 105 {
 106         struct nouveau_fence *fence;
 107
 108         u32 seq = fctx->read(chan);
 109
 110         while (!list_empty(&fctx->pending)) {
 111                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 112
 113                 if ((int)(seq - fence->base.seqno) < 0)
 114                         return;
 115
 116                 nouveau_fence_signal(fence);
 117         }
 118 }
 119
 120 static int
 121 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
 122 {
 123         struct nouveau_fence_chan *fctx =
 124                 container_of(notify, typeof(*fctx), notify);
 125         unsigned long flags;
 126
 127         spin_lock_irqsave(&fctx->lock, flags);
 128         if (!list_empty(&fctx->pending)) {
 129                 struct nouveau_fence *fence;
 130
 131                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 132                 nouveau_fence_update(fence->channel, fctx);
 133         }
 134         spin_unlock_irqrestore(&fctx->lock, flags);
 135
 136         /* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
 137         return NVIF_NOTIFY_KEEP;
 138 }
 139
 140 void
 141 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 142 {
 143         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 144         int ret;
 145
 146         INIT_LIST_HEAD(&fctx->flip);
 147         INIT_LIST_HEAD(&fctx->pending);
 148         spin_lock_init(&fctx->lock);
 149         fctx->context = priv->context_base + chan->chid;
 150
 151         if (!priv->uevent)
 152                 return;
 153
 154         ret = nvif_notify_init(chan->object, NULL,
 155                          nouveau_fence_wait_uevent_handler, false,
 156                          G82_CHANNEL_DMA_V0_NTFY_UEVENT,
 157                          &(struct nvif_notify_uevent_req) { },
 158                          sizeof(struct nvif_notify_uevent_req),
 159                          sizeof(struct nvif_notify_uevent_rep),
 160                          &fctx->notify);
 161
 162         WARN_ON(ret);
 163 }
 164
 165 struct nouveau_fence_work {
 166         struct work_struct work;
 167         struct fence_cb cb;
 168         void (*func)(void *);
 169         void *data;
 170 };
 171
 172 static void
 173 nouveau_fence_work_handler(struct work_struct *kwork)
 174 {
 175         struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
 176         work->func(work->data);
 177         kfree(work);
 178 }
 179
 180 static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
 181 {
 182         struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);
 183
 184         schedule_work(&work->work);
 185 }
 186
 187 void
 188 nouveau_fence_work(struct fence *fence,
 189                    void (*func)(void *), void *data)
 190 {
 191         struct nouveau_fence_work *work;
 192
 193         if (fence_is_signaled(fence))
 194                 goto err;
 195
 196         work = kmalloc(sizeof(*work), GFP_KERNEL);
 197         if (!work) {
 198                 WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
 199                                            false, false));
 200                 goto err;
 201         }
 202
 203         INIT_WORK(&work->work, nouveau_fence_work_handler);
 204         work->func = func;
 205         work->data = data;
 206
 207         if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
 208                 goto err_free;
 209         return;
 210
 211 err_free:
 212         kfree(work);
 213 err:
 214         func(data);
 215 }
 216
 217 int
 218 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
 219 {
 220         struct nouveau_fence_chan *fctx = chan->fence;
 221         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 222         int ret;
 223
 224         fence->channel  = chan;
 225         fence->timeout  = jiffies + (15 * HZ);
 226
 227         if (priv->uevent)
 228                 fence_init(&fence->base, &nouveau_fence_ops_uevent,
 229                            &fctx->lock,
 230                            priv->context_base + chan->chid, ++fctx->sequence);
 231         else
 232                 fence_init(&fence->base, &nouveau_fence_ops_legacy,
 233                            &fctx->lock,
 234                            priv->context_base + chan->chid, ++fctx->sequence);
 235
 236         trace_fence_emit(&fence->base);
 237         ret = fctx->emit(fence);
 238         if (!ret) {
 239                 fence_get(&fence->base);
 240                 spin_lock_irq(&fctx->lock);
 241                 nouveau_fence_update(chan, fctx);
 242                 list_add_tail(&fence->head, &fctx->pending);
 243                 spin_unlock_irq(&fctx->lock);
 244         }
 245
 246         return ret;
 247 }
 248
 249 bool
 250 nouveau_fence_done(struct nouveau_fence *fence)
 251 {
 252         if (fence->base.ops == &nouveau_fence_ops_legacy ||
 253             fence->base.ops == &nouveau_fence_ops_uevent) {
 254                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 255                 unsigned long flags;
 256
 257                 if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 258                         return true;
 259
 260                 spin_lock_irqsave(&fctx->lock, flags);
 261                 nouveau_fence_update(fence->channel, fctx);
 262                 spin_unlock_irqrestore(&fctx->lock, flags);
 263         }
 264         return fence_is_signaled(&fence->base);
 265 }
 266
 267 static long
 268 nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
 269 {
 270         struct nouveau_fence *fence = from_fence(f);
 271         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 272         unsigned long t = jiffies, timeout = t + wait;
 273
 274         while (!nouveau_fence_done(fence)) {
 275                 ktime_t kt;
 276
 277                 t = jiffies;
 278
 279                 if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
 280                         __set_current_state(TASK_RUNNING);
 281                         return 0;
 282                 }
 283
 284                 __set_current_state(intr ? TASK_INTERRUPTIBLE :
 285                                            TASK_UNINTERRUPTIBLE);
 286
 287                 kt = ktime_set(0, sleep_time);
 288                 schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
 289                 sleep_time *= 2;
 290                 if (sleep_time > NSEC_PER_MSEC)
 291                         sleep_time = NSEC_PER_MSEC;
 292
 293                 if (intr && signal_pending(current))
 294                         return -ERESTARTSYS;
 295         }
 296
 297         __set_current_state(TASK_RUNNING);
 298
 299         return timeout - t;
 300 }
 301
 302 static int
 303 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
 304 {
 305         int ret = 0;
 306
 307         while (!nouveau_fence_done(fence)) {
 308                 if (time_after_eq(jiffies, fence->timeout)) {
 309                         ret = -EBUSY;
 310                         break;
 311                 }
 312
 313                 __set_current_state(intr ?
 314                                     TASK_INTERRUPTIBLE :
 315                                     TASK_UNINTERRUPTIBLE);
 316
 317                 if (intr && signal_pending(current)) {
 318                         ret = -ERESTARTSYS;
 319                         break;
 320                 }
 321         }
 322
 323         __set_current_state(TASK_RUNNING);
 324         return ret;
 325 }
 326
 327 int
 328 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 329 {
 330         long ret;
 331
 332         if (!lazy)
 333                 return nouveau_fence_wait_busy(fence, intr);
 334
 335         ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
 336         if (ret < 0)
 337                 return ret;
 338         else if (!ret)
 339                 return -EBUSY;
 340         else
 341                 return 0;
 342 }
 343
 344 int
 345 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan)
 346 {
 347         struct nouveau_fence_chan *fctx = chan->fence;
 348         struct fence *fence = NULL;
 349         struct reservation_object *resv = nvbo->bo.resv;
 350         struct reservation_object_list *fobj;
 351         int ret = 0, i;
 352
 353         fence = reservation_object_get_excl(resv);
 354
 355         if (fence && !fence_is_signaled(fence)) {
 356                 struct nouveau_fence *f = from_fence(fence);
 357                 struct nouveau_channel *prev = f->channel;
 358
 359                 if (prev != chan) {
 360                         ret = fctx->sync(f, prev, chan);
 361                         if (unlikely(ret))
 362                                 ret = nouveau_fence_wait(f, true, true);
 363                 }
 364         }
 365
 366         if (ret)
 367                 return ret;
 368
 369         fobj = reservation_object_get_list(resv);
 370         if (!fobj)
 371                 return ret;
 372
 373         for (i = 0; i < fobj->shared_count && !ret; ++i) {
 374                 fence = rcu_dereference_protected(fobj->shared[i],
 375                                                 reservation_object_held(resv));
 376
 377                 /* should always be true, for now */
 378                 if (!nouveau_local_fence(fence, chan->drm))
 379                         ret = fence_wait(fence, true);
 380         }
 381
 382         return ret;
 383 }
 384
 385 void
 386 nouveau_fence_unref(struct nouveau_fence **pfence)
 387 {
 388         if (*pfence)
 389                 fence_put(&(*pfence)->base);
 390         *pfence = NULL;
 391 }
 392
 393 struct nouveau_fence *
 394 nouveau_fence_ref(struct nouveau_fence *fence)
 395 {
 396         if (fence)
 397                 fence_get(&fence->base);
 398         return fence;
 399 }
 400
 401 int
 402 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
 403                   struct nouveau_fence **pfence)
 404 {
 405         struct nouveau_fence *fence;
 406         int ret = 0;
 407
 408         if (unlikely(!chan->fence))
 409                 return -ENODEV;
 410
 411         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 412         if (!fence)
 413                 return -ENOMEM;
 414
 415         fence->sysmem = sysmem;
 416
 417         ret = nouveau_fence_emit(fence, chan);
 418         if (ret)
 419                 nouveau_fence_unref(&fence);
 420
 421         *pfence = fence;
 422         return ret;
 423 }
 424
 425 static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
 426 {
 427         return "nouveau";
 428 }
 429
 430 static const char *nouveau_fence_get_timeline_name(struct fence *f)
 431 {
 432         struct nouveau_fence *fence = from_fence(f);
 433         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 434
 435         return fence->channel ? fctx->name : "dead channel";
 436 }
 437
 438 /*
 439  * In an ideal world, read would not assume the channel context is still alive.
 440  * This function may be called from another device, running into free memory as a
 441  * result. The drm node should still be there, so we can derive the index from
 442  * the fence context.
 443  */
 444 static bool nouveau_fence_is_signaled(struct fence *f)
 445 {
 446         struct nouveau_fence *fence = from_fence(f);
 447         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 448         struct nouveau_channel *chan = fence->channel;
 449
 450         return (int)(fctx->read(chan) - fence->base.seqno) >= 0;
 451 }
 452
 453 static bool nouveau_fence_no_signaling(struct fence *f)
 454 {
 455         struct nouveau_fence *fence = from_fence(f);
 456
 457         /*
 458          * caller should have a reference on the fence,
 459          * else fence could get freed here
 460          */
 461         WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
 462
 463         /*
 464          * This needs uevents to work correctly, but fence_add_callback relies on
 465          * being able to enable signaling. It will still get signaled eventually,
 466          * just not right away.
 467          */
 468         if (nouveau_fence_is_signaled(f)) {
 469                 list_del(&fence->head);
 470
 471                 fence_put(&fence->base);
 472                 return false;
 473         }
 474
 475         return true;
 476 }
 477
 478 static const struct fence_ops nouveau_fence_ops_legacy = {
 479         .get_driver_name = nouveau_fence_get_get_driver_name,
 480         .get_timeline_name = nouveau_fence_get_timeline_name,
 481         .enable_signaling = nouveau_fence_no_signaling,
 482         .signaled = nouveau_fence_is_signaled,
 483         .wait = nouveau_fence_wait_legacy,
 484         .release = NULL
 485 };
 486
 487 static bool nouveau_fence_enable_signaling(struct fence *f)
 488 {
 489         struct nouveau_fence *fence = from_fence(f);
 490         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 491         bool ret;
 492
 493         if (!fctx->notify_ref++)
 494                 nvif_notify_get(&fctx->notify);
 495
 496         ret = nouveau_fence_no_signaling(f);
 497         if (ret)
 498                 set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
 499         else if (!--fctx->notify_ref)
 500                 nvif_notify_put(&fctx->notify);
 501
 502         return ret;
 503 }
 504
 505 static const struct fence_ops nouveau_fence_ops_uevent = {
 506         .get_driver_name = nouveau_fence_get_get_driver_name,
 507         .get_timeline_name = nouveau_fence_get_timeline_name,
 508         .enable_signaling = nouveau_fence_enable_signaling,
 509         .signaled = nouveau_fence_is_signaled,
 510         .wait = fence_default_wait,
 511         .release = NULL
 512 };