Commit | Line | Data |
---|---|---|
ba4e7d97 TH |
1 | /************************************************************************** |
2 | * | |
3 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA | |
4 | * All Rights Reserved. | |
5 | * | |
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |
7 | * copy of this software and associated documentation files (the | |
8 | * "Software"), to deal in the Software without restriction, including | |
9 | * without limitation the rights to use, copy, modify, merge, publish, | |
10 | * distribute, sub license, and/or sell copies of the Software, and to | |
11 | * permit persons to whom the Software is furnished to do so, subject to | |
12 | * the following conditions: | |
13 | * | |
14 | * The above copyright notice and this permission notice (including the | |
15 | * next paragraph) shall be included in all copies or substantial portions | |
16 | * of the Software. | |
17 | * | |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
25 | * | |
26 | **************************************************************************/ | |
27 | ||
28 | #include "ttm/ttm_memory.h" | |
29 | #include <linux/spinlock.h> | |
30 | #include <linux/sched.h> | |
31 | #include <linux/wait.h> | |
32 | #include <linux/mm.h> | |
33 | #include <linux/module.h> | |
34 | ||
35 | #define TTM_PFX "[TTM] " | |
36 | #define TTM_MEMORY_ALLOC_RETRIES 4 | |
37 | ||
38 | /** | |
39 | * At this point we only support a single shrink callback. | |
40 | * Extend this if needed, perhaps using a linked list of callbacks. | |
41 | * Note that this function is reentrant: | |
42 | * many threads may try to swap out at any given time. | |
43 | */ | |
44 | ||
45 | static void ttm_shrink(struct ttm_mem_global *glob, bool from_workqueue, | |
46 | uint64_t extra) | |
47 | { | |
48 | int ret; | |
49 | struct ttm_mem_shrink *shrink; | |
50 | uint64_t target; | |
51 | uint64_t total_target; | |
52 | ||
53 | spin_lock(&glob->lock); | |
54 | if (glob->shrink == NULL) | |
55 | goto out; | |
56 | ||
57 | if (from_workqueue) { | |
58 | target = glob->swap_limit; | |
59 | total_target = glob->total_memory_swap_limit; | |
60 | } else if (capable(CAP_SYS_ADMIN)) { | |
61 | total_target = glob->emer_total_memory; | |
62 | target = glob->emer_memory; | |
63 | } else { | |
64 | total_target = glob->max_total_memory; | |
65 | target = glob->max_memory; | |
66 | } | |
67 | ||
68 | total_target = (extra >= total_target) ? 0 : total_target - extra; | |
69 | target = (extra >= target) ? 0 : target - extra; | |
70 | ||
71 | while (glob->used_memory > target || | |
72 | glob->used_total_memory > total_target) { | |
73 | shrink = glob->shrink; | |
74 | spin_unlock(&glob->lock); | |
75 | ret = shrink->do_shrink(shrink); | |
76 | spin_lock(&glob->lock); | |
77 | if (unlikely(ret != 0)) | |
78 | goto out; | |
79 | } | |
80 | out: | |
81 | spin_unlock(&glob->lock); | |
82 | } | |
83 | ||
84 | static void ttm_shrink_work(struct work_struct *work) | |
85 | { | |
86 | struct ttm_mem_global *glob = | |
87 | container_of(work, struct ttm_mem_global, work); | |
88 | ||
89 | ttm_shrink(glob, true, 0ULL); | |
90 | } | |
91 | ||
92 | int ttm_mem_global_init(struct ttm_mem_global *glob) | |
93 | { | |
94 | struct sysinfo si; | |
95 | uint64_t mem; | |
96 | ||
97 | spin_lock_init(&glob->lock); | |
98 | glob->swap_queue = create_singlethread_workqueue("ttm_swap"); | |
99 | INIT_WORK(&glob->work, ttm_shrink_work); | |
100 | init_waitqueue_head(&glob->queue); | |
101 | ||
102 | si_meminfo(&si); | |
103 | ||
104 | mem = si.totalram - si.totalhigh; | |
105 | mem *= si.mem_unit; | |
106 | ||
107 | glob->max_memory = mem >> 1; | |
108 | glob->emer_memory = (mem >> 1) + (mem >> 2); | |
109 | glob->swap_limit = glob->max_memory - (mem >> 3); | |
110 | glob->used_memory = 0; | |
111 | glob->used_total_memory = 0; | |
112 | glob->shrink = NULL; | |
113 | ||
114 | mem = si.totalram; | |
115 | mem *= si.mem_unit; | |
116 | ||
117 | glob->max_total_memory = mem >> 1; | |
118 | glob->emer_total_memory = (mem >> 1) + (mem >> 2); | |
119 | ||
120 | glob->total_memory_swap_limit = glob->max_total_memory - (mem >> 3); | |
121 | ||
122 | printk(KERN_INFO TTM_PFX "TTM available graphics memory: %llu MiB\n", | |
123 | glob->max_total_memory >> 20); | |
124 | printk(KERN_INFO TTM_PFX "TTM available object memory: %llu MiB\n", | |
125 | glob->max_memory >> 20); | |
126 | ||
127 | return 0; | |
128 | } | |
129 | EXPORT_SYMBOL(ttm_mem_global_init); | |
130 | ||
131 | void ttm_mem_global_release(struct ttm_mem_global *glob) | |
132 | { | |
133 | printk(KERN_INFO TTM_PFX "Used total memory is %llu bytes.\n", | |
134 | (unsigned long long)glob->used_total_memory); | |
135 | flush_workqueue(glob->swap_queue); | |
136 | destroy_workqueue(glob->swap_queue); | |
137 | glob->swap_queue = NULL; | |
138 | } | |
139 | EXPORT_SYMBOL(ttm_mem_global_release); | |
140 | ||
141 | static inline void ttm_check_swapping(struct ttm_mem_global *glob) | |
142 | { | |
143 | bool needs_swapping; | |
144 | ||
145 | spin_lock(&glob->lock); | |
146 | needs_swapping = (glob->used_memory > glob->swap_limit || | |
147 | glob->used_total_memory > | |
148 | glob->total_memory_swap_limit); | |
149 | spin_unlock(&glob->lock); | |
150 | ||
151 | if (unlikely(needs_swapping)) | |
152 | (void)queue_work(glob->swap_queue, &glob->work); | |
153 | ||
154 | } | |
155 | ||
156 | void ttm_mem_global_free(struct ttm_mem_global *glob, | |
157 | uint64_t amount, bool himem) | |
158 | { | |
159 | spin_lock(&glob->lock); | |
160 | glob->used_total_memory -= amount; | |
161 | if (!himem) | |
162 | glob->used_memory -= amount; | |
163 | wake_up_all(&glob->queue); | |
164 | spin_unlock(&glob->lock); | |
165 | } | |
166 | ||
167 | static int ttm_mem_global_reserve(struct ttm_mem_global *glob, | |
168 | uint64_t amount, bool himem, bool reserve) | |
169 | { | |
170 | uint64_t limit; | |
171 | uint64_t lomem_limit; | |
172 | int ret = -ENOMEM; | |
173 | ||
174 | spin_lock(&glob->lock); | |
175 | ||
176 | if (capable(CAP_SYS_ADMIN)) { | |
177 | limit = glob->emer_total_memory; | |
178 | lomem_limit = glob->emer_memory; | |
179 | } else { | |
180 | limit = glob->max_total_memory; | |
181 | lomem_limit = glob->max_memory; | |
182 | } | |
183 | ||
184 | if (unlikely(glob->used_total_memory + amount > limit)) | |
185 | goto out_unlock; | |
186 | if (unlikely(!himem && glob->used_memory + amount > lomem_limit)) | |
187 | goto out_unlock; | |
188 | ||
189 | if (reserve) { | |
190 | glob->used_total_memory += amount; | |
191 | if (!himem) | |
192 | glob->used_memory += amount; | |
193 | } | |
194 | ret = 0; | |
195 | out_unlock: | |
196 | spin_unlock(&glob->lock); | |
197 | ttm_check_swapping(glob); | |
198 | ||
199 | return ret; | |
200 | } | |
201 | ||
202 | int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, | |
203 | bool no_wait, bool interruptible, bool himem) | |
204 | { | |
205 | int count = TTM_MEMORY_ALLOC_RETRIES; | |
206 | ||
207 | while (unlikely(ttm_mem_global_reserve(glob, memory, himem, true) | |
208 | != 0)) { | |
209 | if (no_wait) | |
210 | return -ENOMEM; | |
211 | if (unlikely(count-- == 0)) | |
212 | return -ENOMEM; | |
213 | ttm_shrink(glob, false, memory + (memory >> 2) + 16); | |
214 | } | |
215 | ||
216 | return 0; | |
217 | } | |
218 | ||
219 | size_t ttm_round_pot(size_t size) | |
220 | { | |
221 | if ((size & (size - 1)) == 0) | |
222 | return size; | |
223 | else if (size > PAGE_SIZE) | |
224 | return PAGE_ALIGN(size); | |
225 | else { | |
226 | size_t tmp_size = 4; | |
227 | ||
228 | while (tmp_size < size) | |
229 | tmp_size <<= 1; | |
230 | ||
231 | return tmp_size; | |
232 | } | |
233 | return 0; | |
234 | } |