1/*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/errno.h>
30
31#include <mach/mach_types.h>
32#include <mach/kern_return.h>
33#include <mach/memory_object_control.h>
34#include <mach/memory_object_types.h>
35#include <mach/port.h>
36#include <mach/policy.h>
37#include <mach/upl.h>
38#include <kern/kern_types.h>
39#include <kern/ipc_kobject.h>
40#include <kern/host.h>
41#include <kern/thread.h>
42#include <ipc/ipc_port.h>
43#include <ipc/ipc_space.h>
44#include <device/device_port.h>
45#include <vm/memory_object.h>
46#include <vm/vm_pageout.h>
47#include <vm/vm_map.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_pageout.h>
50#include <vm/vm_protos.h>
51#include <mach/sdt.h>
52
53
54/* Device VM COMPONENT INTERFACES */
55
56
57/*
58 * Device PAGER
59 */
60
61
62/* until component support available */
63
64
65
66/* until component support available */
67const struct memory_object_pager_ops device_pager_ops = {
68 device_pager_reference,
69 device_pager_deallocate,
70 device_pager_init,
71 device_pager_terminate,
72 device_pager_data_request,
73 device_pager_data_return,
74 device_pager_data_initialize,
75 device_pager_data_unlock,
76 device_pager_synchronize,
77 device_pager_map,
78 device_pager_last_unmap,
79 NULL, /* data_reclaim */
80 "device pager"
81};
82
83typedef uintptr_t device_port_t;
84
85/*
86 * The start of "struct device_pager" MUST match a "struct memory_object".
87 */
88typedef struct device_pager {
89 /* mandatory generic header */
90 struct memory_object dev_pgr_hdr;
91
92 /* pager-specific data */
93 lck_mtx_t lock;
94 unsigned int ref_count; /* reference count */
95 device_port_t device_handle; /* device_handle */
96 vm_size_t size;
97 int flags;
98 boolean_t is_mapped;
99} *device_pager_t;
100
101lck_grp_t device_pager_lck_grp;
102lck_grp_attr_t device_pager_lck_grp_attr;
103lck_attr_t device_pager_lck_attr;
104
105#define device_pager_lock_init(pager) \
106 lck_mtx_init(&(pager)->lock, \
107 &device_pager_lck_grp, \
108 &device_pager_lck_attr)
109#define device_pager_lock_destroy(pager) \
110 lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp)
111#define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock)
112#define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock)
113
114device_pager_t
115device_pager_lookup( /* forward */
116 memory_object_t);
117
118device_pager_t
119device_object_create(void); /* forward */
120
121zone_t device_pager_zone;
122
123
124#define DEVICE_PAGER_NULL ((device_pager_t) 0)
125
126
127#define MAX_DNODE 10000
128
129
130
131
132
133/*
134 *
135 */
136void
137device_pager_bootstrap(void)
138{
139 vm_size_t size;
140
141 size = (vm_size_t) sizeof(struct device_pager);
142 device_pager_zone = zinit(size, (vm_size_t) MAX_DNODE*size,
143 PAGE_SIZE, "device node pager structures");
144 zone_change(device_pager_zone, Z_CALLERACCT, FALSE);
145
146 lck_grp_attr_setdefault(&device_pager_lck_grp_attr);
147 lck_grp_init(&device_pager_lck_grp, "device_pager", &device_pager_lck_grp_attr);
148 lck_attr_setdefault(&device_pager_lck_attr);
149
150 return;
151}
152
153/*
154 *
155 */
156memory_object_t
157device_pager_setup(
158 __unused memory_object_t device,
159 uintptr_t device_handle,
160 vm_size_t size,
161 int flags)
162{
163 device_pager_t device_object;
164 memory_object_control_t control;
165 vm_object_t object;
166
167 device_object = device_object_create();
168 if (device_object == DEVICE_PAGER_NULL)
169 panic("device_pager_setup: device_object_create() failed");
170
171 device_object->device_handle = device_handle;
172 device_object->size = size;
173 device_object->flags = flags;
174
175 memory_object_create_named((memory_object_t) device_object,
176 size,
177 &control);
178 object = memory_object_control_to_vm_object(control);
179
180 assert(object != VM_OBJECT_NULL);
181 vm_object_lock(object);
182 object->true_share = TRUE;
183 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
184 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
185 }
186 vm_object_unlock(object);
187
188 return (memory_object_t)device_object;
189}
190
191/*
192 *
193 */
194kern_return_t
195device_pager_populate_object(
196 memory_object_t device,
197 memory_object_offset_t offset,
198 ppnum_t page_num,
199 vm_size_t size)
200{
201 device_pager_t device_object;
202 vm_object_t vm_object;
203 kern_return_t kr;
204 upl_t upl;
205
206 device_object = device_pager_lookup(device);
207 if(device_object == DEVICE_PAGER_NULL)
208 return KERN_FAILURE;
209
210 vm_object = (vm_object_t)memory_object_control_to_vm_object(
211 device_object->dev_pgr_hdr.mo_control);
212 if(vm_object == NULL)
213 return KERN_FAILURE;
214
215 kr = vm_object_populate_with_private(
216 vm_object, offset, page_num, size);
217 if(kr != KERN_SUCCESS)
218 return kr;
219
220 if(!vm_object->phys_contiguous) {
221 unsigned int null_size = 0;
222 assert((upl_size_t) size == size);
223 kr = vm_object_upl_request(vm_object,
224 (vm_object_offset_t)offset,
225 (upl_size_t) size, &upl, NULL,
226 &null_size,
227 (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE),
228 VM_KERN_MEMORY_NONE);
229 if(kr != KERN_SUCCESS)
230 panic("device_pager_populate_object: list_req failed");
231
232 upl_commit(upl, NULL, 0);
233 upl_deallocate(upl);
234 }
235
236
237 return kr;
238}
239
240/*
241 *
242 */
243device_pager_t
244device_pager_lookup(
245 memory_object_t mem_obj)
246{
247 device_pager_t device_object;
248
249 assert(mem_obj->mo_pager_ops == &device_pager_ops);
250 device_object = (device_pager_t)mem_obj;
251 assert(device_object->ref_count > 0);
252 return device_object;
253}
254
255/*
256 *
257 */
258kern_return_t
259device_pager_init(
260 memory_object_t mem_obj,
261 memory_object_control_t control,
262 __unused memory_object_cluster_size_t pg_size)
263{
264 device_pager_t device_object;
265 kern_return_t kr;
266 memory_object_attr_info_data_t attributes;
267
268 vm_object_t vm_object;
269
270
271 if (control == MEMORY_OBJECT_CONTROL_NULL)
272 return KERN_INVALID_ARGUMENT;
273
274 device_object = device_pager_lookup(mem_obj);
275
276 memory_object_control_reference(control);
277 device_object->dev_pgr_hdr.mo_control = control;
278
279
280/* The following settings should be done through an expanded change */
281/* attributes call */
282
283 vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
284 vm_object_lock(vm_object);
285 vm_object->private = TRUE;
286 if(device_object->flags & DEVICE_PAGER_CONTIGUOUS)
287 vm_object->phys_contiguous = TRUE;
288 if(device_object->flags & DEVICE_PAGER_NOPHYSCACHE)
289 vm_object->nophyscache = TRUE;
290
291 vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;
292 vm_object_unlock(vm_object);
293
294
295 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
296 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
297 attributes.cluster_size = (1 << (PAGE_SHIFT));
298 attributes.may_cache_object = FALSE;
299 attributes.temporary = TRUE;
300
301 kr = memory_object_change_attributes(
302 control,
303 MEMORY_OBJECT_ATTRIBUTE_INFO,
304 (memory_object_info_t) &attributes,
305 MEMORY_OBJECT_ATTR_INFO_COUNT);
306 if (kr != KERN_SUCCESS)
307 panic("device_pager_init: memory_object_change_attributes() failed");
308
309 return(KERN_SUCCESS);
310}
311
312/*
313 *
314 */
315/*ARGSUSED6*/
316kern_return_t
317device_pager_data_return(
318 memory_object_t mem_obj,
319 memory_object_offset_t offset,
320 memory_object_cluster_size_t data_cnt,
321 __unused memory_object_offset_t *resid_offset,
322 __unused int *io_error,
323 __unused boolean_t dirty,
324 __unused boolean_t kernel_copy,
325 __unused int upl_flags)
326{
327 device_pager_t device_object;
328
329 device_object = device_pager_lookup(mem_obj);
330 if (device_object == DEVICE_PAGER_NULL)
331 panic("device_pager_data_return: lookup failed");
332
333 __IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
334 (ipc_port_t) device_object,
335 VM_PROT_READ | VM_PROT_WRITE,
336 offset, data_cnt));
337}
338
339/*
340 *
341 */
342kern_return_t
343device_pager_data_request(
344 memory_object_t mem_obj,
345 memory_object_offset_t offset,
346 memory_object_cluster_size_t length,
347 __unused vm_prot_t protection_required,
348 __unused memory_object_fault_info_t fault_info)
349{
350 device_pager_t device_object;
351
352 device_object = device_pager_lookup(mem_obj);
353
354 if (device_object == DEVICE_PAGER_NULL)
355 panic("device_pager_data_request: lookup failed");
356
357 __IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
358 (ipc_port_t) device_object,
359 VM_PROT_READ, offset, length));
360 return KERN_SUCCESS;
361}
362
363/*
364 *
365 */
366void
367device_pager_reference(
368 memory_object_t mem_obj)
369{
370 device_pager_t device_object;
371 unsigned int new_ref_count;
372
373 device_object = device_pager_lookup(mem_obj);
374 new_ref_count = hw_atomic_add(&device_object->ref_count, 1);
375 assert(new_ref_count > 1);
376 DTRACE_VM2(device_pager_reference,
377 device_pager_t, device_object,
378 unsigned int, device_object->ref_count);
379
380}
381
382/*
383 *
384 */
385void
386device_pager_deallocate(
387 memory_object_t mem_obj)
388{
389 device_pager_t device_object;
390 memory_object_control_t device_control;
391 unsigned int ref_count;
392
393 device_object = device_pager_lookup(mem_obj);
394 assert(device_object->ref_count > 0);
395
396 DTRACE_VM2(device_pager_deallocate,
397 device_pager_t, device_object,
398 unsigned int, device_object->ref_count);
399
400 ref_count = hw_atomic_sub(&device_object->ref_count, 1);
401
402 if (ref_count == 1) {
403 /*
404 * The last reference is our "named" reference.
405 * Close the device and "destroy" the VM object.
406 */
407
408 DTRACE_VM2(device_pager_destroy,
409 device_pager_t, device_object,
410 unsigned int, device_object->ref_count);
411
412 assert(device_object->is_mapped == FALSE);
413 if (device_object->device_handle != (device_port_t) NULL) {
414 device_close(device_object->device_handle);
415 device_object->device_handle = (device_port_t) NULL;
416 }
417 device_control = device_object->dev_pgr_hdr.mo_control;
418 memory_object_destroy(device_control, 0);
419 } else if (ref_count == 0) {
420 /*
421 * No more references: free the pager.
422 */
423 DTRACE_VM2(device_pager_free,
424 device_pager_t, device_object,
425 unsigned int, device_object->ref_count);
426
427 device_pager_lock_destroy(device_object);
428
429 zfree(device_pager_zone, device_object);
430 }
431 return;
432}
433
434kern_return_t
435device_pager_data_initialize(
436 __unused memory_object_t mem_obj,
437 __unused memory_object_offset_t offset,
438 __unused memory_object_cluster_size_t data_cnt)
439{
440 panic("device_pager_data_initialize");
441 return KERN_FAILURE;
442}
443
444kern_return_t
445device_pager_data_unlock(
446 __unused memory_object_t mem_obj,
447 __unused memory_object_offset_t offset,
448 __unused memory_object_size_t size,
449 __unused vm_prot_t desired_access)
450{
451 return KERN_FAILURE;
452}
453
454kern_return_t
455device_pager_terminate(
456 __unused memory_object_t mem_obj)
457{
458 return KERN_SUCCESS;
459}
460
461
462
463/*
464 *
465 */
466kern_return_t
467device_pager_synchronize(
468 __unused memory_object_t mem_obj,
469 __unused memory_object_offset_t offset,
470 __unused memory_object_size_t length,
471 __unused vm_sync_t sync_flags)
472{
473 panic("device_pager_synchronize: memory_object_synchronize no longer supported\n");
474 return KERN_FAILURE;
475}
476
477/*
478 *
479 */
480kern_return_t
481device_pager_map(
482 memory_object_t mem_obj,
483 __unused vm_prot_t prot)
484{
485 device_pager_t device_object;
486
487 device_object = device_pager_lookup(mem_obj);
488
489 device_pager_lock(device_object);
490 assert(device_object->ref_count > 0);
491 if (device_object->is_mapped == FALSE) {
492 /*
493 * First mapping of this pager: take an extra reference
494 * that will remain until all the mappings of this pager
495 * are removed.
496 */
497 device_object->is_mapped = TRUE;
498 device_pager_reference(mem_obj);
499 }
500 device_pager_unlock(device_object);
501
502 return KERN_SUCCESS;
503}
504
505kern_return_t
506device_pager_last_unmap(
507 memory_object_t mem_obj)
508{
509 device_pager_t device_object;
510 boolean_t drop_ref;
511
512 device_object = device_pager_lookup(mem_obj);
513
514 device_pager_lock(device_object);
515 assert(device_object->ref_count > 0);
516 if (device_object->is_mapped) {
517 device_object->is_mapped = FALSE;
518 drop_ref = TRUE;
519 } else {
520 drop_ref = FALSE;
521 }
522 device_pager_unlock(device_object);
523
524 if (drop_ref) {
525 device_pager_deallocate(mem_obj);
526 }
527
528 return KERN_SUCCESS;
529}
530
531
532
533/*
534 *
535 */
536device_pager_t
537device_object_create(void)
538{
539 device_pager_t device_object;
540
541 device_object = (struct device_pager *) zalloc(device_pager_zone);
542 if (device_object == DEVICE_PAGER_NULL)
543 return(DEVICE_PAGER_NULL);
544
545 bzero(device_object, sizeof (*device_object));
546
547 device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
548 device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops;
549 device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
550
551 device_pager_lock_init(device_object);
552 device_object->ref_count = 1;
553 device_object->is_mapped = FALSE;
554
555 DTRACE_VM2(device_pager_create,
556 device_pager_t, device_object,
557 unsigned int, device_object->ref_count);
558
559 return(device_object);
560}
561
562boolean_t
563is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
564{
565 if (pager_ops == &device_pager_ops) {
566 return TRUE;
567 }
568 return FALSE;
569}
570