Commit | Line | Data |
---|---|---|
40603526 RZ |
1 | /* |
2 | * Copyright(c) 2015 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of version 2 of the GNU General Public License as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | */ | |
13 | #ifndef __ASM_X86_PMEM_H__ | |
14 | #define __ASM_X86_PMEM_H__ | |
15 | ||
16 | #include <linux/uaccess.h> | |
17 | #include <asm/cacheflush.h> | |
18 | #include <asm/cpufeature.h> | |
19 | #include <asm/special_insns.h> | |
20 | ||
4a370df5 | 21 | #ifdef CONFIG_ARCH_HAS_PMEM_API |
40603526 RZ |
22 | /** |
23 | * arch_memcpy_to_pmem - copy data to persistent memory | |
24 | * @dst: destination buffer for the copy | |
25 | * @src: source buffer for the copy | |
26 | * @n: length of the copy in bytes | |
27 | * | |
28 | * Copy data to persistent memory media via non-temporal stores so that | |
29 | * a subsequent arch_wmb_pmem() can flush cpu and memory controller | |
30 | * write buffers to guarantee durability. | |
31 | */ | |
32 | static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, | |
33 | size_t n) | |
34 | { | |
35 | int unwritten; | |
36 | ||
37 | /* | |
38 | * We are copying between two kernel buffers, if | |
39 | * __copy_from_user_inatomic_nocache() returns an error (page | |
40 | * fault) we would have already reported a general protection fault | |
41 | * before the WARN+BUG. | |
42 | */ | |
43 | unwritten = __copy_from_user_inatomic_nocache((void __force *) dst, | |
44 | (void __user *) src, n); | |
45 | if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n", | |
46 | __func__, dst, src, unwritten)) | |
47 | BUG(); | |
48 | } | |
49 | ||
50 | /** | |
51 | * arch_wmb_pmem - synchronize writes to persistent memory | |
52 | * | |
53 | * After a series of arch_memcpy_to_pmem() operations this drains data | |
54 | * from cpu write buffers and any platform (memory controller) buffers | |
55 | * to ensure that written data is durable on persistent memory media. | |
56 | */ | |
57 | static inline void arch_wmb_pmem(void) | |
58 | { | |
59 | /* | |
60 | * wmb() to 'sfence' all previous writes such that they are | |
61 | * architecturally visible to 'pcommit'. Note, that we've | |
62 | * already arranged for pmem writes to avoid the cache via | |
63 | * arch_memcpy_to_pmem(). | |
64 | */ | |
65 | wmb(); | |
66 | pcommit_sfence(); | |
67 | } | |
68 | ||
5de490da RZ |
69 | /** |
70 | * __arch_wb_cache_pmem - write back a cache range with CLWB | |
71 | * @vaddr: virtual start address | |
72 | * @size: number of bytes to write back | |
73 | * | |
74 | * Write back a cache range using the CLWB (cache line write back) | |
75 | * instruction. This function requires explicit ordering with an | |
76 | * arch_wmb_pmem() call. This API is internal to the x86 PMEM implementation. | |
77 | */ | |
78 | static inline void __arch_wb_cache_pmem(void *vaddr, size_t size) | |
79 | { | |
80 | u16 x86_clflush_size = boot_cpu_data.x86_clflush_size; | |
81 | unsigned long clflush_mask = x86_clflush_size - 1; | |
82 | void *vend = vaddr + size; | |
83 | void *p; | |
84 | ||
85 | for (p = (void *)((unsigned long)vaddr & ~clflush_mask); | |
86 | p < vend; p += x86_clflush_size) | |
87 | clwb(p); | |
88 | } | |
89 | ||
90 | /* | |
91 | * copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec | |
92 | * iterators, so for other types (bvec & kvec) we must do a cache write-back. | |
93 | */ | |
94 | static inline bool __iter_needs_pmem_wb(struct iov_iter *i) | |
95 | { | |
96 | return iter_is_iovec(i) == false; | |
97 | } | |
98 | ||
99 | /** | |
100 | * arch_copy_from_iter_pmem - copy data from an iterator to PMEM | |
101 | * @addr: PMEM destination address | |
102 | * @bytes: number of bytes to copy | |
103 | * @i: iterator with source data | |
104 | * | |
105 | * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. | |
106 | * This function requires explicit ordering with an arch_wmb_pmem() call. | |
107 | */ | |
108 | static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, | |
109 | struct iov_iter *i) | |
110 | { | |
111 | void *vaddr = (void __force *)addr; | |
112 | size_t len; | |
113 | ||
114 | /* TODO: skip the write-back by always using non-temporal stores */ | |
115 | len = copy_from_iter_nocache(vaddr, bytes, i); | |
116 | ||
117 | if (__iter_needs_pmem_wb(i)) | |
118 | __arch_wb_cache_pmem(vaddr, bytes); | |
119 | ||
120 | return len; | |
121 | } | |
122 | ||
123 | /** | |
124 | * arch_clear_pmem - zero a PMEM memory range | |
125 | * @addr: virtual start address | |
126 | * @size: number of bytes to zero | |
127 | * | |
128 | * Write zeros into the memory range starting at 'addr' for 'size' bytes. | |
129 | * This function requires explicit ordering with an arch_wmb_pmem() call. | |
130 | */ | |
131 | static inline void arch_clear_pmem(void __pmem *addr, size_t size) | |
132 | { | |
133 | void *vaddr = (void __force *)addr; | |
134 | ||
52db400f | 135 | memset(vaddr, 0, size); |
5de490da RZ |
136 | __arch_wb_cache_pmem(vaddr, size); |
137 | } | |
138 | ||
96601adb | 139 | static inline bool __arch_has_wmb_pmem(void) |
40603526 | 140 | { |
40603526 RZ |
141 | /* |
142 | * We require that wmb() be an 'sfence', that is only guaranteed on | |
143 | * 64-bit builds | |
144 | */ | |
145 | return static_cpu_has(X86_FEATURE_PCOMMIT); | |
40603526 | 146 | } |
4a370df5 | 147 | #endif /* CONFIG_ARCH_HAS_PMEM_API */ |
40603526 | 148 | #endif /* __ASM_X86_PMEM_H__ */ |