2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Disclaimer: The codes contained in these modules may be specific to
19 * the Intel Software Development Platform codenamed: Knights Ferry, and
20 * the Intel product codenamed: Knights Corner, and are not backward
21 * compatible with other Intel products. Additionally, Intel will NOT
22 * support the codes or instruction set in future products.
26 * virtio for kvm on s390
28 * Copyright IBM Corp. 2008
30 * This program is free software; you can redistribute it and/or modify
31 * it under the terms of the GNU General Public License (version 2 only)
32 * as published by the Free Software Foundation.
34 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
36 * Intel MIC Card driver.
39 #include <linux/delay.h>
40 #include <linux/slab.h>
41 #include <linux/virtio_config.h>
43 #include "../common/mic_dev.h"
44 #include "mic_virtio.h"
46 #define VIRTIO_SUBCODE_64 0x0D00
48 #define MIC_MAX_VRINGS 4
50 struct virtio_device vdev
;
51 struct mic_device_desc __iomem
*desc
;
52 struct mic_device_ctrl __iomem
*dc
;
53 struct mic_device
*mdev
;
54 void __iomem
*vr
[MIC_MAX_VRINGS
];
55 int used_size
[MIC_MAX_VRINGS
];
56 struct completion reset_done
;
57 struct mic_irq
*virtio_cookie
;
61 static struct mic_irq
*virtio_config_cookie
;
62 #define to_micvdev(vd) container_of(vd, struct mic_vdev, vdev)
64 /* Helper API to obtain the parent of the virtio device */
65 static inline struct device
*mic_dev(struct mic_vdev
*mvdev
)
67 return mvdev
->vdev
.dev
.parent
;
70 /* This gets the device's feature bits. */
71 static u32
mic_get_features(struct virtio_device
*vdev
)
75 struct mic_device_desc __iomem
*desc
= to_micvdev(vdev
)->desc
;
76 u8 __iomem
*in_features
= mic_vq_features(desc
);
77 int feature_len
= ioread8(&desc
->feature_len
);
79 bits
= min_t(unsigned, feature_len
,
80 sizeof(vdev
->features
)) * 8;
81 for (i
= 0; i
< bits
; i
++)
82 if (ioread8(&in_features
[i
/ 8]) & (BIT(i
% 8)))
88 static void mic_finalize_features(struct virtio_device
*vdev
)
91 struct mic_device_desc __iomem
*desc
= to_micvdev(vdev
)->desc
;
92 u8 feature_len
= ioread8(&desc
->feature_len
);
93 /* Second half of bitmap is features we accept. */
94 u8 __iomem
*out_features
=
95 mic_vq_features(desc
) + feature_len
;
97 /* Give virtio_ring a chance to accept features. */
98 vring_transport_features(vdev
);
100 memset_io(out_features
, 0, feature_len
);
101 bits
= min_t(unsigned, feature_len
,
102 sizeof(vdev
->features
)) * 8;
103 for (i
= 0; i
< bits
; i
++) {
104 if (test_bit(i
, vdev
->features
))
105 iowrite8(ioread8(&out_features
[i
/ 8]) | (1 << (i
% 8)),
106 &out_features
[i
/ 8]);
111 * Reading and writing elements in config space
113 static void mic_get(struct virtio_device
*vdev
, unsigned int offset
,
114 void *buf
, unsigned len
)
116 struct mic_device_desc __iomem
*desc
= to_micvdev(vdev
)->desc
;
118 if (offset
+ len
> ioread8(&desc
->config_len
))
120 memcpy_fromio(buf
, mic_vq_configspace(desc
) + offset
, len
);
123 static void mic_set(struct virtio_device
*vdev
, unsigned int offset
,
124 const void *buf
, unsigned len
)
126 struct mic_device_desc __iomem
*desc
= to_micvdev(vdev
)->desc
;
128 if (offset
+ len
> ioread8(&desc
->config_len
))
130 memcpy_toio(mic_vq_configspace(desc
) + offset
, buf
, len
);
134 * The operations to get and set the status word just access the status
135 * field of the device descriptor. set_status also interrupts the host
136 * to tell about status changes.
138 static u8
mic_get_status(struct virtio_device
*vdev
)
140 return ioread8(&to_micvdev(vdev
)->desc
->status
);
143 static void mic_set_status(struct virtio_device
*vdev
, u8 status
)
145 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
148 iowrite8(status
, &mvdev
->desc
->status
);
149 mic_send_intr(mvdev
->mdev
, mvdev
->c2h_vdev_db
);
152 /* Inform host on a virtio device reset and wait for ack from host */
153 static void mic_reset_inform_host(struct virtio_device
*vdev
)
155 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
156 struct mic_device_ctrl __iomem
*dc
= mvdev
->dc
;
159 iowrite8(0, &dc
->host_ack
);
160 iowrite8(1, &dc
->vdev_reset
);
161 mic_send_intr(mvdev
->mdev
, mvdev
->c2h_vdev_db
);
163 /* Wait till host completes all card accesses and acks the reset */
164 for (i
= retry
; i
--;) {
165 if (ioread8(&dc
->host_ack
))
170 dev_dbg(mic_dev(mvdev
), "%s: retry: %d\n", __func__
, retry
);
172 /* Reset status to 0 in case we timed out */
173 iowrite8(0, &mvdev
->desc
->status
);
176 static void mic_reset(struct virtio_device
*vdev
)
178 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
180 dev_dbg(mic_dev(mvdev
), "%s: virtio id %d\n",
181 __func__
, vdev
->id
.device
);
183 mic_reset_inform_host(vdev
);
184 complete_all(&mvdev
->reset_done
);
188 * The virtio_ring code calls this API when it wants to notify the Host.
190 static void mic_notify(struct virtqueue
*vq
)
192 struct mic_vdev
*mvdev
= vq
->priv
;
194 mic_send_intr(mvdev
->mdev
, mvdev
->c2h_vdev_db
);
197 static void mic_del_vq(struct virtqueue
*vq
, int n
)
199 struct mic_vdev
*mvdev
= to_micvdev(vq
->vdev
);
200 struct vring
*vr
= (struct vring
*)(vq
+ 1);
202 free_pages((unsigned long) vr
->used
, get_order(mvdev
->used_size
[n
]));
203 vring_del_virtqueue(vq
);
204 mic_card_unmap(mvdev
->mdev
, mvdev
->vr
[n
]);
208 static void mic_del_vqs(struct virtio_device
*vdev
)
210 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
211 struct virtqueue
*vq
, *n
;
214 dev_dbg(mic_dev(mvdev
), "%s\n", __func__
);
216 list_for_each_entry_safe(vq
, n
, &vdev
->vqs
, list
)
217 mic_del_vq(vq
, idx
++);
221 * This routine will assign vring's allocated in host/io memory. Code in
222 * virtio_ring.c however continues to access this io memory as if it were local
223 * memory without io accessors.
225 static struct virtqueue
*mic_find_vq(struct virtio_device
*vdev
,
227 void (*callback
)(struct virtqueue
*vq
),
230 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
231 struct mic_vqconfig __iomem
*vqconfig
;
232 struct mic_vqconfig config
;
233 struct virtqueue
*vq
;
235 struct _mic_vring_info __iomem
*info
;
237 int vr_size
, _vr_size
, err
, magic
;
239 u8 type
= ioread8(&mvdev
->desc
->type
);
241 if (index
>= ioread8(&mvdev
->desc
->num_vq
))
242 return ERR_PTR(-ENOENT
);
245 return ERR_PTR(-ENOENT
);
247 /* First assign the vring's allocated in host memory */
248 vqconfig
= mic_vq_config(mvdev
->desc
) + index
;
249 memcpy_fromio(&config
, vqconfig
, sizeof(config
));
250 _vr_size
= vring_size(config
.num
, MIC_VIRTIO_RING_ALIGN
);
251 vr_size
= PAGE_ALIGN(_vr_size
+ sizeof(struct _mic_vring_info
));
252 va
= mic_card_map(mvdev
->mdev
, config
.address
, vr_size
);
254 return ERR_PTR(-ENOMEM
);
255 mvdev
->vr
[index
] = va
;
256 memset_io(va
, 0x0, _vr_size
);
257 vq
= vring_new_virtqueue(index
,
258 config
.num
, MIC_VIRTIO_RING_ALIGN
, vdev
,
260 va
, mic_notify
, callback
, name
);
265 info
= va
+ _vr_size
;
266 magic
= ioread32(&info
->magic
);
268 if (WARN(magic
!= MIC_MAGIC
+ type
+ index
, "magic mismatch")) {
273 /* Allocate and reassign used ring now */
274 mvdev
->used_size
[index
] = PAGE_ALIGN(sizeof(__u16
) * 3 +
275 sizeof(struct vring_used_elem
) * config
.num
);
276 used
= (void *)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
,
277 get_order(mvdev
->used_size
[index
]));
280 dev_err(mic_dev(mvdev
), "%s %d err %d\n",
281 __func__
, __LINE__
, err
);
284 iowrite64(virt_to_phys(used
), &vqconfig
->used_address
);
287 * To reassign the used ring here we are directly accessing
288 * struct vring_virtqueue which is a private data structure
289 * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
290 * vring_new_virtqueue() would ensure that
291 * (&vq->vring == (struct vring *) (&vq->vq + 1));
293 vr
= (struct vring
*)(vq
+ 1);
299 vring_del_virtqueue(vq
);
301 mic_card_unmap(mvdev
->mdev
, mvdev
->vr
[index
]);
305 static int mic_find_vqs(struct virtio_device
*vdev
, unsigned nvqs
,
306 struct virtqueue
*vqs
[],
307 vq_callback_t
*callbacks
[],
310 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
311 struct mic_device_ctrl __iomem
*dc
= mvdev
->dc
;
312 int i
, err
, retry
= 100;
314 /* We must have this many virtqueues. */
315 if (nvqs
> ioread8(&mvdev
->desc
->num_vq
))
318 for (i
= 0; i
< nvqs
; ++i
) {
319 dev_dbg(mic_dev(mvdev
), "%s: %d: %s\n",
320 __func__
, i
, names
[i
]);
321 vqs
[i
] = mic_find_vq(vdev
, i
, callbacks
[i
], names
[i
]);
322 if (IS_ERR(vqs
[i
])) {
323 err
= PTR_ERR(vqs
[i
]);
328 iowrite8(1, &dc
->used_address_updated
);
330 * Send an interrupt to the host to inform it that used
331 * rings have been re-assigned.
333 mic_send_intr(mvdev
->mdev
, mvdev
->c2h_vdev_db
);
334 for (i
= retry
; i
--;) {
335 if (!ioread8(&dc
->used_address_updated
))
340 dev_dbg(mic_dev(mvdev
), "%s: retry: %d\n", __func__
, retry
);
353 * The config ops structure as defined by virtio config
355 static struct virtio_config_ops mic_vq_config_ops
= {
356 .get_features
= mic_get_features
,
357 .finalize_features
= mic_finalize_features
,
360 .get_status
= mic_get_status
,
361 .set_status
= mic_set_status
,
363 .find_vqs
= mic_find_vqs
,
364 .del_vqs
= mic_del_vqs
,
368 mic_virtio_intr_handler(int irq
, void *data
)
370 struct mic_vdev
*mvdev
= data
;
371 struct virtqueue
*vq
;
373 mic_ack_interrupt(mvdev
->mdev
);
374 list_for_each_entry(vq
, &mvdev
->vdev
.vqs
, list
)
375 vring_interrupt(0, vq
);
380 static void mic_virtio_release_dev(struct device
*_d
)
383 * No need for a release method similar to virtio PCI.
384 * Provide an empty one to avoid getting a warning from core.
389 * adds a new device and register it with virtio
390 * appropriate drivers are loaded by the device model
392 static int mic_add_device(struct mic_device_desc __iomem
*d
,
393 unsigned int offset
, struct mic_driver
*mdrv
)
395 struct mic_vdev
*mvdev
;
398 u8 type
= ioread8(&d
->type
);
400 mvdev
= kzalloc(sizeof(*mvdev
), GFP_KERNEL
);
402 dev_err(mdrv
->dev
, "Cannot allocate mic dev %u type %u\n",
407 mvdev
->mdev
= &mdrv
->mdev
;
408 mvdev
->vdev
.dev
.parent
= mdrv
->dev
;
409 mvdev
->vdev
.dev
.release
= mic_virtio_release_dev
;
410 mvdev
->vdev
.id
.device
= type
;
411 mvdev
->vdev
.config
= &mic_vq_config_ops
;
413 mvdev
->dc
= (void __iomem
*)d
+ mic_aligned_desc_size(d
);
414 init_completion(&mvdev
->reset_done
);
416 virtio_db
= mic_next_card_db();
417 mvdev
->virtio_cookie
= mic_request_card_irq(mic_virtio_intr_handler
,
418 "virtio intr", mvdev
, virtio_db
);
419 if (IS_ERR(mvdev
->virtio_cookie
)) {
420 ret
= PTR_ERR(mvdev
->virtio_cookie
);
423 iowrite8((u8
)virtio_db
, &mvdev
->dc
->h2c_vdev_db
);
424 mvdev
->c2h_vdev_db
= ioread8(&mvdev
->dc
->c2h_vdev_db
);
426 ret
= register_virtio_device(&mvdev
->vdev
);
428 dev_err(mic_dev(mvdev
),
429 "Failed to register mic device %u type %u\n",
433 iowrite64((u64
)mvdev
, &mvdev
->dc
->vdev
);
434 dev_dbg(mic_dev(mvdev
), "%s: registered mic device %u type %u mvdev %p\n",
435 __func__
, offset
, type
, mvdev
);
440 mic_free_card_irq(mvdev
->virtio_cookie
, mvdev
);
447 * match for a mic device with a specific desc pointer
449 static int mic_match_desc(struct device
*dev
, void *data
)
451 struct virtio_device
*vdev
= dev_to_virtio(dev
);
452 struct mic_vdev
*mvdev
= to_micvdev(vdev
);
454 return mvdev
->desc
== (void __iomem
*)data
;
457 static void mic_handle_config_change(struct mic_device_desc __iomem
*d
,
458 unsigned int offset
, struct mic_driver
*mdrv
)
460 struct mic_device_ctrl __iomem
*dc
461 = (void __iomem
*)d
+ mic_aligned_desc_size(d
);
462 struct mic_vdev
*mvdev
= (struct mic_vdev
*)ioread64(&dc
->vdev
);
463 struct virtio_driver
*drv
;
465 if (ioread8(&dc
->config_change
) != MIC_VIRTIO_PARAM_CONFIG_CHANGED
)
468 dev_dbg(mdrv
->dev
, "%s %d\n", __func__
, __LINE__
);
469 drv
= container_of(mvdev
->vdev
.dev
.driver
,
470 struct virtio_driver
, driver
);
471 if (drv
->config_changed
)
472 drv
->config_changed(&mvdev
->vdev
);
473 iowrite8(1, &dc
->guest_ack
);
477 * removes a virtio device if a hot remove event has been
478 * requested by the host.
480 static int mic_remove_device(struct mic_device_desc __iomem
*d
,
481 unsigned int offset
, struct mic_driver
*mdrv
)
483 struct mic_device_ctrl __iomem
*dc
484 = (void __iomem
*)d
+ mic_aligned_desc_size(d
);
485 struct mic_vdev
*mvdev
= (struct mic_vdev
*)ioread64(&dc
->vdev
);
489 if (ioread8(&dc
->config_change
) == MIC_VIRTIO_PARAM_DEV_REMOVE
) {
491 "%s %d config_change %d type %d mvdev %p\n",
493 ioread8(&dc
->config_change
), ioread8(&d
->type
), mvdev
);
495 status
= ioread8(&d
->status
);
496 INIT_COMPLETION(mvdev
->reset_done
);
497 unregister_virtio_device(&mvdev
->vdev
);
498 mic_free_card_irq(mvdev
->virtio_cookie
, mvdev
);
499 if (status
& VIRTIO_CONFIG_S_DRIVER_OK
)
500 wait_for_completion(&mvdev
->reset_done
);
502 iowrite8(1, &dc
->guest_ack
);
503 dev_dbg(mdrv
->dev
, "%s %d guest_ack %d\n",
504 __func__
, __LINE__
, ioread8(&dc
->guest_ack
));
511 #define REMOVE_DEVICES true
513 static void mic_scan_devices(struct mic_driver
*mdrv
, bool remove
)
517 struct mic_device_desc __iomem
*d
;
518 struct mic_device_ctrl __iomem
*dc
;
522 for (i
= mic_aligned_size(struct mic_bootparam
);
523 i
< MIC_DP_SIZE
; i
+= mic_total_desc_size(d
)) {
525 dc
= (void __iomem
*)d
+ mic_aligned_desc_size(d
);
527 * This read barrier is paired with the corresponding write
528 * barrier on the host which is inserted before adding or
529 * removing a virtio device descriptor, by updating the type.
532 type
= ioread8(&d
->type
);
541 /* device already exists */
542 dev
= device_find_child(mdrv
->dev
, d
, mic_match_desc
);
545 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE
,
548 mic_handle_config_change(d
, i
, mdrv
);
549 ret
= mic_remove_device(d
, i
, mdrv
);
551 iowrite8(-1, &d
->type
);
553 iowrite8(0, &dc
->config_change
);
554 iowrite8(0, &dc
->guest_ack
);
560 dev_dbg(mdrv
->dev
, "%s %d Adding new virtio device %p\n",
561 __func__
, __LINE__
, d
);
563 mic_add_device(d
, i
, mdrv
);
568 * mic_hotplug_device tries to find changes in the device page.
570 static void mic_hotplug_devices(struct work_struct
*work
)
572 struct mic_driver
*mdrv
= container_of(work
,
573 struct mic_driver
, hotplug_work
);
575 mic_scan_devices(mdrv
, !REMOVE_DEVICES
);
579 * Interrupt handler for hot plug/config changes etc.
582 mic_extint_handler(int irq
, void *data
)
584 struct mic_driver
*mdrv
= (struct mic_driver
*)data
;
586 dev_dbg(mdrv
->dev
, "%s %d hotplug work\n",
588 mic_ack_interrupt(&mdrv
->mdev
);
589 schedule_work(&mdrv
->hotplug_work
);
594 * Init function for virtio
596 int mic_devices_init(struct mic_driver
*mdrv
)
599 struct mic_bootparam __iomem
*bootparam
;
602 INIT_WORK(&mdrv
->hotplug_work
, mic_hotplug_devices
);
603 mic_scan_devices(mdrv
, !REMOVE_DEVICES
);
605 config_db
= mic_next_card_db();
606 virtio_config_cookie
= mic_request_card_irq(mic_extint_handler
,
607 "virtio_config_intr", mdrv
, config_db
);
608 if (IS_ERR(virtio_config_cookie
)) {
609 rc
= PTR_ERR(virtio_config_cookie
);
613 bootparam
= mdrv
->dp
;
614 iowrite8(config_db
, &bootparam
->h2c_config_db
);
621 * Uninit function for virtio
623 void mic_devices_uninit(struct mic_driver
*mdrv
)
625 struct mic_bootparam __iomem
*bootparam
= mdrv
->dp
;
626 iowrite8(-1, &bootparam
->h2c_config_db
);
627 mic_free_card_irq(virtio_config_cookie
, mdrv
);
628 flush_work(&mdrv
->hotplug_work
);
629 mic_scan_devices(mdrv
, REMOVE_DEVICES
);