Commit | Line | Data |
---|---|---|
61031952 RZ |
1 | /* |
2 | * Copyright(c) 2015 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of version 2 of the GNU General Public License as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | */ | |
13 | #ifndef __PMEM_H__ | |
14 | #define __PMEM_H__ | |
15 | ||
16 | #include <linux/io.h> | |
5de490da | 17 | #include <linux/uio.h> |
61031952 RZ |
18 | |
19 | #ifdef CONFIG_ARCH_HAS_PMEM_API | |
96601adb | 20 | #define ARCH_MEMREMAP_PMEM MEMREMAP_WB |
40603526 | 21 | #include <asm/pmem.h> |
61031952 | 22 | #else |
96601adb DW |
23 | #define ARCH_MEMREMAP_PMEM MEMREMAP_WT |
24 | /* | |
25 | * These are simply here to enable compilation, all call sites gate | |
26 | * calling these symbols with arch_has_pmem_api() and redirect to the | |
27 | * implementation in asm/pmem.h. | |
28 | */ | |
29 | static inline bool __arch_has_wmb_pmem(void) | |
61031952 | 30 | { |
96601adb | 31 | return false; |
61031952 RZ |
32 | } |
33 | ||
96601adb | 34 | static inline void arch_wmb_pmem(void) |
61031952 | 35 | { |
96601adb | 36 | BUG(); |
61031952 RZ |
37 | } |
38 | ||
61031952 RZ |
39 | static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, |
40 | size_t n) | |
41 | { | |
42 | BUG(); | |
43 | } | |
5de490da RZ |
44 | |
45 | static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, | |
46 | struct iov_iter *i) | |
47 | { | |
48 | BUG(); | |
49 | return 0; | |
50 | } | |
51 | ||
52 | static inline void arch_clear_pmem(void __pmem *addr, size_t size) | |
53 | { | |
54 | BUG(); | |
55 | } | |
3f4a2670 RZ |
56 | |
57 | static inline void arch_wb_cache_pmem(void __pmem *addr, size_t size) | |
58 | { | |
59 | BUG(); | |
60 | } | |
59e64739 DW |
61 | |
62 | static inline void arch_invalidate_pmem(void __pmem *addr, size_t size) | |
63 | { | |
64 | BUG(); | |
65 | } | |
61031952 RZ |
66 | #endif |
67 | ||
68 | /* | |
69 | * Architectures that define ARCH_HAS_PMEM_API must provide | |
5de490da | 70 | * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), |
3f4a2670 RZ |
71 | * arch_copy_from_iter_pmem(), arch_clear_pmem(), arch_wb_cache_pmem() |
72 | * and arch_has_wmb_pmem(). | |
61031952 | 73 | */ |
61031952 RZ |
74 | static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size) |
75 | { | |
76 | memcpy(dst, (void __force const *) src, size); | |
77 | } | |
78 | ||
96601adb DW |
79 | static inline bool arch_has_pmem_api(void) |
80 | { | |
81 | return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API); | |
82 | } | |
83 | ||
61031952 | 84 | /** |
96601adb | 85 | * arch_has_wmb_pmem - true if wmb_pmem() ensures durability |
61031952 RZ |
86 | * |
87 | * For a given cpu implementation within an architecture it is possible | |
88 | * that wmb_pmem() resolves to a nop. In the case this returns | |
89 | * false, pmem api users are unable to ensure durability and may want to | |
90 | * fall back to a different data consistency model, or otherwise notify | |
91 | * the user. | |
92 | */ | |
96601adb | 93 | static inline bool arch_has_wmb_pmem(void) |
61031952 | 94 | { |
96601adb | 95 | return arch_has_pmem_api() && __arch_has_wmb_pmem(); |
61031952 RZ |
96 | } |
97 | ||
98 | /* | |
99 | * These defaults seek to offer decent performance and minimize the | |
100 | * window between i/o completion and writes being durable on media. | |
101 | * However, it is undefined / architecture specific whether | |
a639315d | 102 | * ARCH_MEMREMAP_PMEM + default_memcpy_to_pmem is sufficient for |
61031952 RZ |
103 | * making data durable relative to i/o completion. |
104 | */ | |
e836a256 | 105 | static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, |
61031952 RZ |
106 | size_t size) |
107 | { | |
108 | memcpy((void __force *) dst, src, size); | |
109 | } | |
110 | ||
5de490da RZ |
111 | static inline size_t default_copy_from_iter_pmem(void __pmem *addr, |
112 | size_t bytes, struct iov_iter *i) | |
113 | { | |
114 | return copy_from_iter_nocache((void __force *)addr, bytes, i); | |
115 | } | |
116 | ||
117 | static inline void default_clear_pmem(void __pmem *addr, size_t size) | |
118 | { | |
119 | if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0) | |
120 | clear_page((void __force *)addr); | |
121 | else | |
122 | memset((void __force *)addr, 0, size); | |
123 | } | |
124 | ||
61031952 RZ |
125 | /** |
126 | * memcpy_to_pmem - copy data to persistent memory | |
127 | * @dst: destination buffer for the copy | |
128 | * @src: source buffer for the copy | |
129 | * @n: length of the copy in bytes | |
130 | * | |
131 | * Perform a memory copy that results in the destination of the copy | |
132 | * being effectively evicted from, or never written to, the processor | |
133 | * cache hierarchy after the copy completes. After memcpy_to_pmem() | |
134 | * data may still reside in cpu or platform buffers, so this operation | |
135 | * must be followed by a wmb_pmem(). | |
136 | */ | |
137 | static inline void memcpy_to_pmem(void __pmem *dst, const void *src, size_t n) | |
138 | { | |
139 | if (arch_has_pmem_api()) | |
140 | arch_memcpy_to_pmem(dst, src, n); | |
141 | else | |
142 | default_memcpy_to_pmem(dst, src, n); | |
143 | } | |
144 | ||
145 | /** | |
146 | * wmb_pmem - synchronize writes to persistent memory | |
147 | * | |
148 | * After a series of memcpy_to_pmem() operations this drains data from | |
149 | * cpu write buffers and any platform (memory controller) buffers to | |
150 | * ensure that written data is durable on persistent memory media. | |
151 | */ | |
152 | static inline void wmb_pmem(void) | |
153 | { | |
96601adb | 154 | if (arch_has_wmb_pmem()) |
61031952 | 155 | arch_wmb_pmem(); |
96601adb DW |
156 | else |
157 | wmb(); | |
61031952 | 158 | } |
5de490da RZ |
159 | |
160 | /** | |
161 | * copy_from_iter_pmem - copy data from an iterator to PMEM | |
162 | * @addr: PMEM destination address | |
163 | * @bytes: number of bytes to copy | |
164 | * @i: iterator with source data | |
165 | * | |
166 | * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. | |
167 | * This function requires explicit ordering with a wmb_pmem() call. | |
168 | */ | |
169 | static inline size_t copy_from_iter_pmem(void __pmem *addr, size_t bytes, | |
170 | struct iov_iter *i) | |
171 | { | |
172 | if (arch_has_pmem_api()) | |
173 | return arch_copy_from_iter_pmem(addr, bytes, i); | |
174 | return default_copy_from_iter_pmem(addr, bytes, i); | |
175 | } | |
176 | ||
177 | /** | |
178 | * clear_pmem - zero a PMEM memory range | |
179 | * @addr: virtual start address | |
180 | * @size: number of bytes to zero | |
181 | * | |
182 | * Write zeros into the memory range starting at 'addr' for 'size' bytes. | |
183 | * This function requires explicit ordering with a wmb_pmem() call. | |
184 | */ | |
185 | static inline void clear_pmem(void __pmem *addr, size_t size) | |
186 | { | |
187 | if (arch_has_pmem_api()) | |
188 | arch_clear_pmem(addr, size); | |
189 | else | |
190 | default_clear_pmem(addr, size); | |
191 | } | |
3f4a2670 | 192 | |
59e64739 DW |
193 | /** |
194 | * invalidate_pmem - flush a pmem range from the cache hierarchy | |
195 | * @addr: virtual start address | |
196 | * @size: bytes to invalidate (internally aligned to cache line size) | |
197 | * | |
198 | * For platforms that support clearing poison this flushes any poisoned | |
199 | * ranges out of the cache | |
200 | */ | |
201 | static inline void invalidate_pmem(void __pmem *addr, size_t size) | |
202 | { | |
203 | if (arch_has_pmem_api()) | |
204 | arch_invalidate_pmem(addr, size); | |
205 | } | |
206 | ||
3f4a2670 RZ |
207 | /** |
208 | * wb_cache_pmem - write back processor cache for PMEM memory range | |
209 | * @addr: virtual start address | |
210 | * @size: number of bytes to write back | |
211 | * | |
212 | * Write back the processor cache range starting at 'addr' for 'size' bytes. | |
213 | * This function requires explicit ordering with a wmb_pmem() call. | |
214 | */ | |
215 | static inline void wb_cache_pmem(void __pmem *addr, size_t size) | |
216 | { | |
217 | if (arch_has_pmem_api()) | |
218 | arch_wb_cache_pmem(addr, size); | |
219 | } | |
61031952 | 220 | #endif /* __PMEM_H__ */ |