Index: kern/init_main.c =================================================================== --- kern/init_main.c (revision 1946) +++ kern/init_main.c (working copy) @@ -94,7 +94,6 @@ __KERNEL_RCSID(0, "$NetBSD: init_main.c, #include #include #include -#include #include #include #include @@ -245,8 +244,6 @@ main(void) uvm_init(); - kmem_init(); - /* Do machine-dependent initialization. */ cpu_startup(); Index: kern/kern_malloc.c =================================================================== --- kern/kern_malloc.c (revision 1885) +++ kern/kern_malloc.c (working copy) @@ -75,10 +75,13 @@ __KERNEL_RCSID(0, "$NetBSD: kern_malloc. #include #include #include +#include #include +#if 0 static struct vm_map_kernel kmem_map_store; +#endif struct vm_map *kmem_map = NULL; #include "opt_kmempages.h" @@ -299,6 +302,10 @@ MALLOC_DEFINE(M_1394DATA, "1394data", "I struct simplelock malloc_slock = SIMPLELOCK_INITIALIZER; +struct malloc_header { + size_t mh_size; +}; + /* * Allocate a block of memory */ @@ -311,6 +318,24 @@ void * malloc(unsigned long size, struct malloc_type *ksp, int flags) #endif /* MALLOCLOG */ { + struct malloc_header *mh; + int kmflags = (flags & M_NOWAIT) != 0 ? KM_NOSLEEP : KM_SLEEP; + size_t allocsize = sizeof(struct malloc_header) + size; + void *p; + + if ((flags & M_ZERO) != 0) { + p = kmem_zalloc(allocsize, kmflags); + } else { + p = kmem_alloc(allocsize, kmflags); + } + if (p == NULL) { + return NULL; + } + mh = (void *)p; + mh->mh_size = allocsize; + + return mh + 1; +#if 0 struct kmembuckets *kbp; struct kmemusage *kup; struct freelist *freep; @@ -508,6 +533,7 @@ out: if ((flags & M_ZERO) != 0) memset(va, 0, size); return ((void *) va); +#endif } /* @@ -521,6 +547,12 @@ void free(void *addr, struct malloc_type *ksp) #endif /* MALLOCLOG */ { + struct malloc_header *mh; + + mh = addr; + mh--; + kmem_free(mh, mh->mh_size); +#if 0 struct kmembuckets *kbp; struct kmemusage *kup; struct freelist *freep; @@ -654,6 +686,7 @@ free(void *addr, struct malloc_type *ksp kbp->kb_last = addr; simple_unlock(&malloc_slock); splx(s); +#endif } /* @@ -870,6 +903,7 @@ kmeminit_nkmempages(void) void kmeminit(void) { +#if 0 __link_set_decl(malloc_types, struct malloc_type); struct malloc_type * const *ksp; vaddr_t kmb, kml; @@ -924,6 +958,7 @@ kmeminit(void) #ifdef MALLOC_DEBUG debug_malloc_init(); #endif +#endif } #ifdef DDB Index: kern/subr_pool.c =================================================================== --- kern/subr_pool.c (revision 1885) +++ kern/subr_pool.c (working copy) @@ -91,7 +91,7 @@ static void pool_page_free_meta(struct p /* allocator for pool metadata */ static struct pool_allocator pool_allocator_meta = { pool_page_alloc_meta, pool_page_free_meta, - .pa_backingmapptr = &kmem_map, + .pa_backingmapptr = &kernel_map, }; /* # of seconds to retain page after last use */ @@ -755,7 +755,7 @@ pool_init(struct pool *pp, size_t size, #ifdef POOL_DIAGNOSTIC if (flags & PR_LOGGING) { - if (kmem_map == NULL || + if (kernel_map == NULL || (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log), M_TEMP, M_NOWAIT)) == NULL) pp->pr_roflags &= ~PR_LOGGING; @@ -2277,12 +2277,12 @@ void pool_page_free(struct pool *, void #ifdef POOL_SUBPAGE struct pool_allocator pool_allocator_kmem_fullpage = { pool_page_alloc, pool_page_free, 0, - .pa_backingmapptr = &kmem_map, + .pa_backingmapptr = &kernel_map, }; #else struct pool_allocator pool_allocator_kmem = { pool_page_alloc, pool_page_free, 0, - .pa_backingmapptr = &kmem_map, + .pa_backingmapptr = &kernel_map, }; #endif @@ -2307,7 +2307,7 @@ void pool_subpage_free(struct pool *, vo struct pool_allocator pool_allocator_kmem = { pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, - .pa_backingmapptr = &kmem_map, + .pa_backingmapptr = &kernel_map, }; void *pool_subpage_alloc_nointr(struct pool *, int); @@ -2315,7 +2315,7 @@ void pool_subpage_free_nointr(struct poo struct pool_allocator pool_allocator_nointr = { pool_subpage_alloc, pool_subpage_free, POOL_SUBPAGE, - .pa_backingmapptr = &kmem_map, + .pa_backingmapptr = &kernel_map, }; #endif /* POOL_SUBPAGE */ @@ -2357,14 +2357,14 @@ pool_page_alloc(struct pool *pp, int fla { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok)); + return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok)); } void pool_page_free(struct pool *pp, void *v) { - uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v); + uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v); } static void * @@ -2372,14 +2372,14 @@ pool_page_alloc_meta(struct pool *pp, in { boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE; - return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok)); + return ((void *) uvm_km_alloc_poolpage(kernel_map, waitok)); } static void pool_page_free_meta(struct pool *pp, void *v) { - uvm_km_free_poolpage(kmem_map, (vaddr_t) v); + uvm_km_free_poolpage(kernel_map, (vaddr_t) v); } #ifdef POOL_SUBPAGE Index: kern/subr_vmem.c =================================================================== --- kern/subr_vmem.c (revision 1922) +++ kern/subr_vmem.c (working copy) @@ -1,7 +1,7 @@ /* $NetBSD: subr_vmem.c,v 1.24 2006/11/18 07:51:54 yamt Exp $ */ /*- - * Copyright (c)2006 YAMAMOTO Takashi, + * Copyright (c)2006, 2007 YAMAMOTO Takashi, * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -52,11 +52,15 @@ __KERNEL_RCSID(0, "$NetBSD: subr_vmem.c, #if defined(_KERNEL) #include #include -#include #include #include #include #include +#include + +#include +#include +#include #else /* defined(_KERNEL) */ #include "../sys/vmem.h" #endif /* defined(_KERNEL) */ @@ -89,9 +93,13 @@ void vmem_dump(const vmem_t *); #define VM_FITMASK (VM_BESTFIT | VM_INSTANTFIT) +/* vm_flag_t (internal uses) */ +#define VM_BTPAGE 0x00008000 + CIRCLEQ_HEAD(vmem_seglist, vmem_btag); LIST_HEAD(vmem_freelist, vmem_btag); LIST_HEAD(vmem_hashlist, vmem_btag); +typedef struct vmem_hashlist vmem_hashlist_t; #if defined(QCACHE) #define VMEM_QCACHE_IDX_MAX 32 @@ -111,15 +119,18 @@ typedef struct qcache qcache_t; /* vmem arena */ struct vmem { SIMPLELOCK_DECL(vm_lock); + vm_flag_t vm_flags; + int vm_freetags; vmem_addr_t (*vm_allocfn)(vmem_t *, vmem_size_t, vmem_size_t *, vm_flag_t); void (*vm_freefn)(vmem_t *, vmem_addr_t, vmem_size_t); vmem_t *vm_source; struct vmem_seglist vm_seglist; struct vmem_freelist vm_freelist[VMEM_MAXORDER]; + LIST_HEAD(, btpage_header) vm_btpagelist; size_t vm_hashsize; size_t vm_nbusytag; - struct vmem_hashlist *vm_hashlist; + vmem_hashlist_t *vm_hashlist; size_t vm_quantum_mask; int vm_quantum_shift; const char *vm_name; @@ -141,15 +152,21 @@ struct vmem { #define VMEM_ASSERT_UNLOCKED(vm) \ LOCK_ASSERT(!simple_lock_held(&vm->vm_lock)) +#define vmem_bootstrap_p(vm) (((vm)->vm_flags & VMC_KVA) != 0) + /* boundary tag */ struct vmem_btag { CIRCLEQ_ENTRY(vmem_btag) bt_seglist; union { LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ + SLIST_ENTRY(vmem_btag) u_sfreelist; /* in btpage_header */ + SLIST_ENTRY(vmem_btag) u_tmplist; /* temp use in vmem_xfree */ } bt_u; #define bt_hashlist bt_u.u_hashlist #define bt_freelist bt_u.u_freelist +#define bt_sfreelist bt_u.u_sfreelist +#define bt_tmplist bt_u.u_tmplist vmem_addr_t bt_start; vmem_size_t bt_size; int bt_type; @@ -194,53 +211,178 @@ calc_order(vmem_size_t size) return i; } -#if defined(_KERNEL) -static MALLOC_DEFINE(M_VMEM, "vmem", "vmem"); -#endif /* defined(_KERNEL) */ - static void * xmalloc(size_t sz, vm_flag_t flags) { #if defined(_KERNEL) - return malloc(sz, M_VMEM, - M_CANFAIL | ((flags & VM_SLEEP) ? M_WAITOK : M_NOWAIT)); + return kmem_alloc(sz, (flags & VM_SLEEP) ? KM_SLEEP : KM_NOSLEEP); #else /* defined(_KERNEL) */ return malloc(sz); #endif /* defined(_KERNEL) */ } static void -xfree(void *p) +xfree(void *p, size_t sz) { #if defined(_KERNEL) - return free(p, M_VMEM); + kmem_free(p, sz); #else /* defined(_KERNEL) */ - return free(p); + free(p); #endif /* defined(_KERNEL) */ } +/* ---- static storage for bootstrap */ + +#define STATIC_POOL_NAME(type) static_ ## type +#define STATIC_POOL_IDX(type) static_ ## type ## _idx +#define STATIC_POOL_DEFINE(type, n) \ + type STATIC_POOL_NAME(type)[n] __unused ; \ + int STATIC_POOL_IDX(type) __unused +#define STATIC_POOL_ALLOC(var, type) \ + (var) = &STATIC_POOL_NAME(type)[STATIC_POOL_IDX(type)++]; \ + KASSERT(STATIC_POOL_ELEM_P(type, var)) +#define STATIC_POOL_ELEM_P(type, var) \ + (&STATIC_POOL_NAME(type)[0] <= (var) && \ + (var) < &STATIC_POOL_NAME(type)[__arraycount(STATIC_POOL_NAME(type))]) + +#if 0 +static STATIC_POOL_DEFINE(bt_t, 3); +static STATIC_POOL_DEFINE(vmem_t, 1); +static STATIC_POOL_DEFINE(vmem_hashlist_t, 1); +#else +static STATIC_POOL_DEFINE(bt_t, 3); +static STATIC_POOL_DEFINE(vmem_t, 2); +static STATIC_POOL_DEFINE(vmem_hashlist_t, 2); +#endif + /* ---- boundary tag */ #if defined(_KERNEL) static struct pool_cache bt_poolcache; -static POOL_INIT(bt_pool, sizeof(bt_t), 0, 0, 0, "vmembtpl", NULL); +static struct pool bt_pool; #endif /* defined(_KERNEL) */ +struct btpage_header { + LIST_ENTRY(btpage_header) bh_q; + int bh_nfree; + SLIST_HEAD(, vmem_btag) bh_freelist; + bt_t bh_bt[]; +}; +typedef struct btpage_header btpage_header_t; + +#define BT_PER_PAGE \ + ((PAGE_SIZE - sizeof(btpage_header_t)) / sizeof(bt_t)) + +static int +btpage_alloc(vmem_t *vm, vm_flag_t flags) +{ + vmem_addr_t va; + + VMEM_ASSERT_UNLOCKED(vm); + va = vmem_xalloc(vm, PAGE_SIZE, PAGE_SIZE, 0, 0, 0, 0, + (flags & ~VM_FITMASK) | VM_INSTANTFIT | VM_BTPAGE); + if (va == 0) { + return ENOMEM; + } + return 0; +} + +static void +btpage_init(vmem_t *vm, struct vm_page *pg, vaddr_t va) +{ + btpage_header_t *bh; + int i; + + VMEM_ASSERT_LOCKED(vm); + KASSERT((va & PAGE_MASK) == 0); + pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ|VM_PROT_WRITE); + pmap_update(pmap_kernel()); + bh = (void *)va; + SLIST_INIT(&bh->bh_freelist); + for (i = 0; i < BT_PER_PAGE; i++) { + SLIST_INSERT_HEAD(&bh->bh_freelist, &bh->bh_bt[i], + bt_sfreelist); + } + LIST_INSERT_HEAD(&vm->vm_btpagelist, bh, bh_q); + bh->bh_nfree = BT_PER_PAGE; + vm->vm_freetags += bh->bh_nfree; +} + +static void +btpage_free(vmem_t *vm, btpage_header_t *bh) +{ + + KASSERT(vmem_bootstrap_p(vm)); + pmap_kremove((vaddr_t)bh, PAGE_SIZE); + pmap_update(pmap_kernel()); + vmem_xfree(vm, (vmem_addr_t)bh, PAGE_SIZE); +} + +static btpage_header_t * +btpage_lookup(bt_t *bt) +{ + + return (void *)trunc_page((vaddr_t)bt); +} + +static bt_t * +bt_alloc_bootstrap(vmem_t *vm) +{ + btpage_header_t *bh; + bt_t *bt; + + KASSERT(vmem_bootstrap_p(vm)); + VMEM_ASSERT_LOCKED(vm); + bh = LIST_FIRST(&vm->vm_btpagelist); + if (__predict_false(bh == NULL)) { + STATIC_POOL_ALLOC(bt, bt_t); + return bt; + } + KASSERT(bh->bh_nfree > 0); + bt = SLIST_FIRST(&bh->bh_freelist); + KASSERT(bt != NULL); + SLIST_REMOVE_HEAD(&bh->bh_freelist, bt_sfreelist); + bh->bh_nfree--; + vm->vm_freetags--; + if (SLIST_EMPTY(&bh->bh_freelist)) { + KASSERT(bh->bh_nfree == 0); + LIST_REMOVE(bh, bh_q); + } + return bt; +} + +#define BT_MINRESERVE 1 + static bt_t * bt_alloc(vmem_t *vm, vm_flag_t flags) { bt_t *bt; + VMEM_ASSERT_UNLOCKED(vm); #if defined(_KERNEL) - int s; + if (vmem_bootstrap_p(vm)) { +again: + VMEM_LOCK(vm); + if (vm->vm_freetags <= BT_MINRESERVE && + (flags & VM_BTPAGE) == 0) { + VMEM_UNLOCK(vm); + if (btpage_alloc(vm, flags)) { + return NULL; + } + goto again; + } + bt = bt_alloc_bootstrap(vm); + VMEM_UNLOCK(vm); + } else { + int s; - /* XXX bootstrap */ - s = splvm(); - bt = pool_cache_get(&bt_poolcache, - (flags & VM_SLEEP) != 0 ? PR_WAITOK : PR_NOWAIT); - splx(s); + s = splvm(); + bt = pool_cache_get(&bt_poolcache, + (flags & VM_SLEEP) != 0 ? PR_WAITOK : PR_NOWAIT); + splx(s); + } #else /* defined(_KERNEL) */ bt = malloc(sizeof *bt); #endif /* defined(_KERNEL) */ @@ -252,13 +394,38 @@ static void bt_free(vmem_t *vm, bt_t *bt) { + KASSERT(bt != NULL); + KASSERT(!STATIC_POOL_ELEM_P(bt_t, bt)); + VMEM_ASSERT_UNLOCKED(vm); #if defined(_KERNEL) - int s; + if (vmem_bootstrap_p(vm)) { + btpage_header_t *bh; - /* XXX bootstrap */ - s = splvm(); - pool_cache_put(&bt_poolcache, bt); - splx(s); + bh = btpage_lookup(bt); + VMEM_LOCK(vm); + if (SLIST_EMPTY(&bh->bh_freelist)) { + KASSERT(bh->bh_nfree == 0); + LIST_INSERT_HEAD(&vm->vm_btpagelist, bh, bh_q); + } + SLIST_INSERT_HEAD(&bh->bh_freelist, bt, bt_sfreelist); + bh->bh_nfree++; + vm->vm_freetags++; + if (vm->vm_freetags >= BT_PER_PAGE + BT_MINRESERVE && + bh->bh_nfree == BT_PER_PAGE) { + LIST_REMOVE(bh, bh_q); + vm->vm_freetags -= BT_PER_PAGE; + VMEM_UNLOCK(vm); + btpage_free(vm, bh); + } else { + VMEM_UNLOCK(vm); + } + } else { + int s; + + s = splvm(); + pool_cache_put(&bt_poolcache, bt); + splx(s); + } #else /* defined(_KERNEL) */ free(bt); #endif /* defined(_KERNEL) */ @@ -312,10 +479,10 @@ bt_freehead_toalloc(vmem_t *vm, vmem_siz /* ---- boundary tag hash */ -static struct vmem_hashlist * +static vmem_hashlist_t * bt_hashhead(vmem_t *vm, vmem_addr_t addr) { - struct vmem_hashlist *list; + vmem_hashlist_t *list; unsigned int hash; hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT); @@ -327,7 +494,7 @@ bt_hashhead(vmem_t *vm, vmem_addr_t addr static bt_t * bt_lookupbusy(vmem_t *vm, vmem_addr_t addr) { - struct vmem_hashlist *list; + vmem_hashlist_t *list; bt_t *bt; list = bt_hashhead(vm, addr); @@ -352,7 +519,7 @@ bt_rembusy(vmem_t *vm, bt_t *bt) static void bt_insbusy(vmem_t *vm, bt_t *bt) { - struct vmem_hashlist *list; + vmem_hashlist_t *list; KASSERT(bt->bt_type == BT_TYPE_BUSY); @@ -556,6 +723,7 @@ static int vmem_init(void) { + pool_init(&bt_pool, sizeof(bt_t), 0, 0, 0, "vmembtpl", NULL); pool_cache_init(&bt_poolcache, &bt_pool, NULL, NULL, NULL); return 0; } @@ -572,14 +740,21 @@ vmem_add1(vmem_t *vm, vmem_addr_t addr, KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0); VMEM_ASSERT_UNLOCKED(vm); - btspan = bt_alloc(vm, flags); - if (btspan == NULL) { - return VMEM_ADDR_NULL; - } - btfree = bt_alloc(vm, flags); - if (btfree == NULL) { - bt_free(vm, btspan); - return VMEM_ADDR_NULL; + if ((flags & VMC_KVA) != 0) { + KASSERT(vmem_bootstrap_p(vm)); + KASSERT(CIRCLEQ_EMPTY(&vm->vm_seglist)); + STATIC_POOL_ALLOC(btspan, bt_t); + STATIC_POOL_ALLOC(btfree, bt_t); + } else { + btspan = bt_alloc(vm, flags); + if (btspan == NULL) { + return VMEM_ADDR_NULL; + } + btfree = bt_alloc(vm, flags); + if (btfree == NULL) { + bt_free(vm, btspan); + return VMEM_ADDR_NULL; + } } btspan->bt_type = spanbttype; @@ -596,6 +771,35 @@ vmem_add1(vmem_t *vm, vmem_addr_t addr, bt_insfree(vm, btfree); VMEM_UNLOCK(vm); + if ((flags & VMC_KVA) != 0) { + bt_t *bt; + + /* + * leak a bt. + * this ensure that + */ + + bt = bt_alloc(vm, VM_NOSLEEP); + KASSERT(bt != NULL); + + /* + * don't leave "btfree" on the segment list because + * bt_free() doesn't expect static tags. + */ + + bt = bt_alloc(vm, flags); + VMEM_LOCK(vm); + KASSERT(vm->vm_nbusytag == 1); + bt->bt_start = btfree->bt_start; + bt->bt_size = btfree->bt_size; + bt->bt_type = btfree->bt_type; + bt_insfree(vm, bt); + bt_insseg(vm, bt, btfree); + bt_remseg(vm, btfree); + bt_remfree(vm, btfree); + VMEM_UNLOCK(vm); + } + return addr; } @@ -628,15 +832,14 @@ vmem_rehash(vmem_t *vm, size_t newhashsi { bt_t *bt; int i; - struct vmem_hashlist *newhashlist; - struct vmem_hashlist *oldhashlist; + vmem_hashlist_t *newhashlist; + vmem_hashlist_t *oldhashlist; size_t oldhashsize; KASSERT(newhashsize > 0); VMEM_ASSERT_UNLOCKED(vm); - newhashlist = - xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags); + newhashlist = xmalloc(sizeof(vmem_hashlist_t *) * newhashsize, flags); if (newhashlist == NULL) { return ENOMEM; } @@ -661,7 +864,9 @@ vmem_rehash(vmem_t *vm, size_t newhashsi } VMEM_UNLOCK(vm); - xfree(oldhashlist); + if (!STATIC_POOL_ELEM_P(vmem_hashlist_t, oldhashlist)) { + xfree(oldhashlist, sizeof(vmem_hashlist_t *) * oldhashsize); + } return 0; } @@ -745,13 +950,19 @@ vmem_create(const char *name, vmem_addr_ return NULL; } #endif /* defined(_KERNEL) */ - vm = xmalloc(sizeof(*vm), flags); - if (vm == NULL) { - return NULL; + if ((flags & (VMC_KVA|VMC_KMEM)) != 0) { + STATIC_POOL_ALLOC(vm, vmem_t); + } else { + vm = xmalloc(sizeof(*vm), flags); + if (vm == NULL) { + return NULL; + } } VMEM_LOCK_INIT(vm); vm->vm_name = name; + vm->vm_flags = flags; + vm->vm_freetags = 0; vm->vm_quantum_mask = quantum - 1; vm->vm_quantum_shift = calc_order(quantum); KASSERT(ORDER2SIZE(vm->vm_quantum_shift) == quantum); @@ -768,7 +979,11 @@ vmem_create(const char *name, vmem_addr_ LIST_INIT(&vm->vm_freelist[i]); } vm->vm_hashlist = NULL; - if (vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags)) { + if ((flags & (VMC_KVA|VMC_KMEM)) != 0) { + STATIC_POOL_ALLOC(vm->vm_hashlist, vmem_hashlist_t); + LIST_INIT(&vm->vm_hashlist[0]); + vm->vm_hashsize = 1; + } else if (vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags)) { vmem_destroy(vm); return NULL; } @@ -780,6 +995,12 @@ vmem_create(const char *name, vmem_addr_ } } +#if 0 + if (vmem_bootstrap_p(vm)) { + vmem_rehash(vm, VMEM_HASHSIZE_INIT, flags); + } +#endif + return vm; } @@ -787,6 +1008,7 @@ void vmem_destroy(vmem_t *vm) { + KASSERT(!vmem_bootstrap_p(vm)); VMEM_ASSERT_UNLOCKED(vm); #if defined(QCACHE) @@ -803,9 +1025,10 @@ vmem_destroy(vmem_t *vm) bt_free(vm, bt); } } - xfree(vm->vm_hashlist); + xfree(vm->vm_hashlist, + sizeof(vmem_hashlist_t *) * vm->vm_hashsize); } - xfree(vm); + xfree(vm, sizeof(*vm)); } vmem_size_t @@ -866,6 +1089,7 @@ vmem_xalloc(vmem_t *vm, vmem_size_t size const vmem_size_t size = vmem_roundup_size(vm, size0); vm_flag_t strat = flags & VM_FITMASK; vmem_addr_t start; + struct vm_page *pg; KASSERT(size0 > 0); KASSERT(size > 0); @@ -886,14 +1110,32 @@ vmem_xalloc(vmem_t *vm, vmem_size_t size if (align == 0) { align = vm->vm_quantum_mask + 1; } - btnew = bt_alloc(vm, flags); - if (btnew == NULL) { - return VMEM_ADDR_NULL; - } - btnew2 = bt_alloc(vm, flags); /* XXX not necessary if no restrictions */ - if (btnew2 == NULL) { - bt_free(vm, btnew); - return VMEM_ADDR_NULL; + pg = NULL; + if ((flags & VM_BTPAGE) != 0) { + KASSERT(size == PAGE_SIZE); + KASSERT(align == PAGE_SIZE); + while (pg == NULL) { + pg = uvm_pagealloc(NULL, 0, NULL, 0); + if (pg == NULL) { + if ((flags & VM_NOSLEEP) != 0) { + return ENOMEM; + } + uvm_wait("btpage"); + } + } + btnew = NULL; /* XXX: gcc */ + btnew2 = NULL; + } else { + btnew = bt_alloc(vm, flags); + if (btnew == NULL) { + return VMEM_ADDR_NULL; + } + /* XXX not necessary if no restrictions */ + btnew2 = bt_alloc(vm, flags); + if (btnew2 == NULL) { + bt_free(vm, btnew); + return VMEM_ADDR_NULL; + } } retry_strat: @@ -948,11 +1190,27 @@ retry: } /* XXX */ fail: - bt_free(vm, btnew); - bt_free(vm, btnew2); + if ((flags & VM_BTPAGE) != 0) { + uvm_pagefree(pg); + } else { + bt_free(vm, btnew); + bt_free(vm, btnew2); + } return VMEM_ADDR_NULL; gotit: +#if defined(PMAP_GROWKERNEL) + if (vm == kernel_va_arena) { + uvm_growkernel(start + size); + } +#endif /* defined(PMAP_GROWKERNEL) */ + if ((flags & VM_BTPAGE) != 0) { + vaddr_t va = (vaddr_t)start; + + KASSERT(bt->bt_start == start); + btnew = bt_alloc_bootstrap(vm); + btpage_init(vm, pg, va); + } KASSERT(bt->bt_type == BT_TYPE_FREE); KASSERT(bt->bt_size >= size); bt_remfree(vm, bt); @@ -1026,11 +1284,14 @@ vmem_xfree(vmem_t *vm, vmem_addr_t addr, { bt_t *bt; bt_t *t; + SLIST_HEAD(, vmem_btag) tofree; VMEM_ASSERT_UNLOCKED(vm); KASSERT(addr != VMEM_ADDR_NULL); KASSERT(size > 0); + SLIST_INIT(&tofree); + VMEM_LOCK(vm); bt = bt_lookupbusy(vm, addr); @@ -1049,7 +1310,7 @@ vmem_xfree(vmem_t *vm, vmem_addr_t addr, bt_remfree(vm, t); bt_remseg(vm, t); bt->bt_size += t->bt_size; - bt_free(vm, t); + SLIST_INSERT_HEAD(&tofree, t, bt_tmplist); } t = CIRCLEQ_PREV(bt, bt_seglist); if (t != NULL && t->bt_type == BT_TYPE_FREE) { @@ -1058,7 +1319,7 @@ vmem_xfree(vmem_t *vm, vmem_addr_t addr, bt_remseg(vm, t); bt->bt_size += t->bt_size; bt->bt_start = t->bt_start; - bt_free(vm, t); + SLIST_INSERT_HEAD(&tofree, t, bt_tmplist); } t = CIRCLEQ_PREV(bt, bt_seglist); @@ -1073,15 +1334,19 @@ vmem_xfree(vmem_t *vm, vmem_addr_t addr, spanaddr = bt->bt_start; spansize = bt->bt_size; bt_remseg(vm, bt); - bt_free(vm, bt); + SLIST_INSERT_HEAD(&tofree, bt, bt_tmplist); bt_remseg(vm, t); - bt_free(vm, t); + SLIST_INSERT_HEAD(&tofree, t, bt_tmplist); VMEM_UNLOCK(vm); (*vm->vm_freefn)(vm->vm_source, spanaddr, spansize); } else { bt_insfree(vm, bt); VMEM_UNLOCK(vm); } + while ((t = SLIST_FIRST(&tofree)) != NULL) { + SLIST_REMOVE_HEAD(&tofree, bt_tmplist); + bt_free(vm, t); + } } /* @@ -1131,8 +1396,11 @@ void bt_dump(const bt_t *bt) { - printf("\t%p: %" PRIu64 ", %" PRIu64 ", %d\n", - bt, (uint64_t)bt->bt_start, (uint64_t)bt->bt_size, + printf("\t%p: %" PRIu64 "(0x%" PRIx64 "), %" PRIu64 "(0x%" PRIx64 + "), %d\n", + bt, + (uint64_t)bt->bt_start, (uint64_t)bt->bt_start, + (uint64_t)bt->bt_size, (uint64_t)bt->bt_size, bt->bt_type); } Index: kern/subr_kmem.c =================================================================== --- kern/subr_kmem.c (revision 1885) +++ kern/subr_kmem.c (working copy) @@ -95,9 +95,12 @@ void * kmem_alloc(size_t size, km_flag_t kmflags) { void *p; + int s; + s = splvm(); p = (void *)vmem_alloc(kmem_arena, size, kmf_to_vmf(kmflags) | VM_INSTANTFIT); + splx(s); kmem_poison_check(p, size); return p; } @@ -129,9 +132,12 @@ kmem_zalloc(size_t size, km_flag_t kmfla void kmem_free(void *p, size_t size) { + int s; kmem_poison_fill(p, size); + s = splvm(); vmem_free(kmem_arena, (vmem_addr_t)p, size); + splx(s); } void @@ -140,7 +146,7 @@ kmem_init(void) kmem_arena = vmem_create("kmem", 0, 0, KMEM_QUANTUM_SIZE, kmem_backend_alloc, kmem_backend_free, NULL, - KMEM_QUANTUM_SIZE * 32, VM_SLEEP); + KMEM_QUANTUM_SIZE * 32, VM_SLEEP|VMC_KMEM); callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback, &kmem_kva_reclaim_entry, kmem_arena, kmem_kva_reclaim_callback); } Index: uvm/uvm_km.c =================================================================== --- uvm/uvm_km.c (revision 1885) +++ uvm/uvm_km.c (working copy) @@ -139,6 +139,8 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1 #include #include #include +#include +#include #include @@ -146,6 +148,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1 * global data structures */ +vmem_t *kernel_va_arena; struct vm_map *kernel_map = NULL; /* @@ -282,6 +285,33 @@ void uvm_km_init(vaddr_t start, vaddr_t end) { vaddr_t base = VM_MIN_KERNEL_ADDRESS; + kernel_va_arena = vmem_create("kernelva", + start, end - start, PAGE_SIZE, + NULL, NULL, NULL, 0, VM_NOSLEEP|VMC_KVA); + if (kernel_va_arena == NULL) { + panic("failed to create kernel_va_arena"); + } + +#if 0 +#define N 512 + { + int i; + void *p[N]; + for (i = 0; i < N; i++) { + printf("alloc[%d]\n", i); + p[i] = (void *)vmem_xalloc(kernel_va_arena, PAGE_SIZE, 0, 0, 0, 0, 0, VM_SLEEP|VM_INSTANTFIT); + } +// for (i = 0; i < N; i++) { + for (i = N-1; i >= 0; i--) { + printf("free[%d]: %p\n", i, p[i]); + vmem_xfree(kernel_va_arena, (vmem_addr_t)p[i], PAGE_SIZE); + } + { +extern void vmem_dump(const vmem_t *); + vmem_dump(kernel_va_arena); + } + } +#endif /* * next, init kernel memory objects. @@ -326,6 +356,10 @@ uvm_km_init(vaddr_t start, vaddr_t end) kernel_map = &kernel_map_store.vmk_map; uvm_km_vacache_init(kernel_map, "kvakernel", 0); + + printf("%s aa\n", __func__); + kmem_init(); + printf("%s bb\n", __func__); } /* @@ -373,7 +407,7 @@ uvm_km_suballoc(struct vm_map *map, vadd pmap_reference(vm_map_pmap(map)); if (submap == NULL) { - submap = malloc(sizeof(*submap), M_VMMAP, M_WAITOK); + submap = kmem_alloc(sizeof(*submap), KM_SLEEP); if (submap == NULL) panic("uvm_km_suballoc: unable to create submap"); } @@ -491,6 +525,10 @@ uvm_km_check_empty(vaddr_t start, vaddr_ vaddr_t va; paddr_t pa; + if (!(VM_MIN_KERNEL_ADDRESS <= start)) { +extern void vmem_dump(const vmem_t *); + vmem_dump(kernel_va_arena); + } KDASSERT(VM_MIN_KERNEL_ADDRESS <= start); KDASSERT(start < end); KDASSERT(end <= VM_MAX_KERNEL_ADDRESS); @@ -537,6 +575,9 @@ uvm_km_alloc(struct vm_map *map, vsize_t vm_prot_t prot; UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); + if (map == NULL) { /* XXX kmem_map */ + map = kernel_map; + } KASSERT(vm_map_pmap(map) == pmap_kernel()); KASSERT((flags & UVM_KMF_TYPEMASK) == UVM_KMF_WIRED || (flags & UVM_KMF_TYPEMASK) == UVM_KMF_PAGEABLE || @@ -546,7 +587,6 @@ uvm_km_alloc(struct vm_map *map, vsize_t * setup for call */ - kva = vm_map_min(map); /* hint */ size = round_page(size); obj = (flags & UVM_KMF_PAGEABLE) ? uvm.kernel_object : NULL; UVMHIST_LOG(maphist," (map=0x%x, obj=0x%x, size=0x%x, flags=%d)", @@ -556,13 +596,28 @@ uvm_km_alloc(struct vm_map *map, vsize_t * allocate some virtual space */ - if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET, - align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, - UVM_ADV_RANDOM, - (flags & (UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT | UVM_KMF_WAITVA)) - | UVM_FLAG_QUANTUM)) != 0)) { - UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0); - return(0); + if ((flags & UVM_KMF_PAGEABLE) != 0) { + kva = vm_map_min(map); /* hint */ + if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET, + align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, + UVM_ADV_RANDOM, + (flags & (UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT | UVM_KMF_WAITVA)) + | UVM_FLAG_QUANTUM)) != 0)) { + UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0); + return(0); + } + } else { + int s; + + s = splvm(); + kva = (vaddr_t)vmem_xalloc(kernel_va_arena, size, + align, 0, 0, 0, 0, + ((flags & UVM_KMF_NOWAIT) ? VM_NOSLEEP : VM_SLEEP) + | VM_INSTANTFIT); + splx(s); + if (kva == 0) { + return 0; + } } /* @@ -644,6 +699,9 @@ void uvm_km_free(struct vm_map *map, vaddr_t addr, vsize_t size, uvm_flag_t flags) { + if (map == NULL) { /* XXX kmem_map */ + map = kernel_map; + } KASSERT((flags & UVM_KMF_TYPEMASK) == UVM_KMF_WIRED || (flags & UVM_KMF_TYPEMASK) == UVM_KMF_PAGEABLE || (flags & UVM_KMF_TYPEMASK) == UVM_KMF_VAONLY); @@ -655,12 +713,17 @@ uvm_km_free(struct vm_map *map, vaddr_t if (flags & UVM_KMF_PAGEABLE) { uvm_km_pgremove(addr, addr + size); pmap_remove(pmap_kernel(), addr, addr + size); + uvm_unmap1(map, addr, addr + size, + UVM_FLAG_QUANTUM|UVM_FLAG_VAONLY); } else if (flags & UVM_KMF_WIRED) { + int s; + uvm_km_pgremove_intrsafe(addr, addr + size); pmap_kremove(addr, size); + s = splvm(); + vmem_xfree(kernel_va_arena, addr, size); + splx(s); } - - uvm_unmap1(map, addr, addr + size, UVM_FLAG_QUANTUM|UVM_FLAG_VAONLY); } /* Sanity; must specify both or none. */ @@ -679,7 +742,7 @@ uvm_km_free(struct vm_map *map, vaddr_t vaddr_t uvm_km_alloc_poolpage_cache(struct vm_map *map, boolean_t waitok) { -#if defined(PMAP_MAP_POOLPAGE) +#if defined(PMAP_MAP_POOLPAGE) || 1 return uvm_km_alloc_poolpage(map, waitok); #else struct vm_page *pg; @@ -743,16 +806,13 @@ uvm_km_alloc_poolpage(struct vm_map *map return (va); #else vaddr_t va; - int s = 0xdeadbeaf; /* XXX: gcc */ - const boolean_t intrsafe = (map->flags & VM_MAP_INTRSAFE) != 0; + int s; - if (intrsafe) - s = splvm(); - va = uvm_km_alloc(map, PAGE_SIZE, 0, - (waitok ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK) | UVM_KMF_WIRED); - if (intrsafe) - splx(s); - return (va); + s = splvm(); + va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, + (waitok ? 0 : UVM_KMF_NOWAIT) | UVM_KMF_WIRED); + splx(s); + return va; #endif /* PMAP_MAP_POOLPAGE */ } @@ -766,7 +826,7 @@ uvm_km_alloc_poolpage(struct vm_map *map void uvm_km_free_poolpage_cache(struct vm_map *map, vaddr_t addr) { -#if defined(PMAP_UNMAP_POOLPAGE) +#if defined(PMAP_UNMAP_POOLPAGE) || 1 uvm_km_free_poolpage(map, addr); #else struct pool *pp; Index: uvm/uvm_extern.h =================================================================== --- uvm/uvm_extern.h (revision 1946) +++ uvm/uvm_extern.h (working copy) @@ -463,6 +463,7 @@ extern struct uvmexp uvmexp; * Finally, bring in standard UVM headers. */ #include +#include #include #include #include @@ -472,6 +473,8 @@ extern struct uvmexp uvmexp; #include #include +extern vmem_t *kernel_va_arena; + /* * Shareable process virtual address space. * May eventually be merged with vm_map. Index: uvm/uvm_map.c =================================================================== --- uvm/uvm_map.c (revision 1885) +++ uvm/uvm_map.c (working copy) @@ -83,6 +83,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v #include #include #include +#include #include #include #include @@ -177,7 +178,7 @@ extern struct vm_map *pager_map; /* XXX #define VM_MAP_USE_KMAPENT_FLAGS(flags) \ (((flags) & VM_MAP_INTRSAFE) != 0) #define VM_MAP_USE_KMAPENT(map) \ - (VM_MAP_USE_KMAPENT_FLAGS((map)->flags) || (map) == kernel_map) + (VM_MAP_USE_KMAPENT_FLAGS((map)->flags) /*|| (map) == kernel_map*/) /* * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging @@ -517,9 +518,12 @@ static struct vm_map_entry * uvm_mapent_alloc(struct vm_map *map, int flags) { struct vm_map_entry *me; +#if 0 int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK; +#endif UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist); +#if 0 if (VM_MAP_USE_KMAPENT(map)) { me = uvm_kmapent_alloc(map, flags); } else { @@ -528,6 +532,14 @@ uvm_mapent_alloc(struct vm_map *map, int return NULL; me->flags = 0; } +#else + me = kmem_alloc(sizeof(*me), + (flags & UVM_FLAG_NOWAIT) ? KM_NOSLEEP : KM_SLEEP); + if (__predict_false(me == NULL)) { + return NULL; + } + me->flags = 0; +#endif UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]", me, ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map), 0, 0); @@ -578,11 +590,14 @@ uvm_mapent_free(struct vm_map_entry *me) UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]", me, me->flags, 0, 0); +#if 0 if (me->flags & UVM_MAP_KERNEL) { uvm_kmapent_free(me); } else { pool_put(&uvm_map_entry_pool, me); } +#endif + kmem_free(me, sizeof(*me)); } /* @@ -681,6 +696,14 @@ _uvm_mapent_check(const struct vm_map_en return; bad: + { +extern void vmem_dump(const vmem_t *); + vmem_dump(kernel_va_arena); + } + printf("%p-%p obj=%p off=%" PRIu64 ", type=0x%x\n", + (void *)entry->start, (void *)entry->end, + entry->object.uvm_obj, entry->offset, entry->etype); + panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line); } #endif /* defined(DEBUG) */ @@ -916,7 +939,7 @@ uvm_map(struct vm_map *map, vaddr_t *sta new_entry = NULL; if (VM_MAP_USE_KMAPENT(map) || (flags & UVM_FLAG_QUANTUM) || - map == pager_map) { + map == pager_map || map == kernel_map) { new_entry = uvm_mapent_alloc(map, (flags & UVM_FLAG_NOWAIT)); if (__predict_false(new_entry == NULL)) return ENOMEM; @@ -998,52 +1021,71 @@ retry: } vm_map_lock(map); /* could sleep here */ } - prev_entry = uvm_map_findspace(map, start, size, &start, - uobj, uoffset, align, flags); - if (prev_entry == NULL) { - unsigned int timestamp; + if (map == kernel_map) { + int s; - timestamp = map->timestamp; - UVMHIST_LOG(maphist,"waiting va timestamp=0x%x", - timestamp,0,0,0); - simple_lock(&map->flags_lock); - map->flags |= VM_MAP_WANTVA; - simple_unlock(&map->flags_lock); - vm_map_unlock(map); + KASSERT((flags & UVM_FLAG_FIXED) == 0); + if (align == 1) { + align = 0; /* XXX */ + } + s = splvm(); +#if defined(PMAP_PREFER) + XXX +#else /* defined(PMAP_PREFER) */ + start = (vaddr_t)vmem_xalloc(kernel_va_arena, size, align, + 0, 0, 0, 0, + ((flags & UVM_FLAG_NOWAIT) ? VM_NOSLEEP : VM_SLEEP) | + VM_INSTANTFIT); +#endif /* defined(PMAP_PREFER) */ + splx(s); + if (start == 0) { + return ENOMEM; + } + if (uvm_map_lookup_entry(map, start, &prev_entry)) { + panic("%s: va %p in use", __func__, (void *)start); + } + goto done; + } else { + prev_entry = uvm_map_findspace(map, start, size, &start, + uobj, uoffset, align, flags); + if (prev_entry == NULL) { + unsigned int timestamp; - /* - * try to reclaim kva and wait until someone does unmap. - * XXX fragile locking - */ + timestamp = map->timestamp; + UVMHIST_LOG(maphist,"waiting va timestamp=0x%x", + timestamp,0,0,0); + simple_lock(&map->flags_lock); + map->flags |= VM_MAP_WANTVA; + simple_unlock(&map->flags_lock); + vm_map_unlock(map); - vm_map_drain(map, flags); + /* + * try to reclaim kva and wait until someone does unmap. + * XXX fragile locking + */ - simple_lock(&map->flags_lock); - while ((map->flags & VM_MAP_WANTVA) != 0 && - map->timestamp == timestamp) { - if ((flags & UVM_FLAG_WAITVA) == 0) { - simple_unlock(&map->flags_lock); - UVMHIST_LOG(maphist, - "<- uvm_map_findspace failed!", 0,0,0,0); - return ENOMEM; - } else { - ltsleep(&map->header, PVM, "vmmapva", 0, - &map->flags_lock); + vm_map_drain(map, flags); + + simple_lock(&map->flags_lock); + while ((map->flags & VM_MAP_WANTVA) != 0 && + map->timestamp == timestamp) { + if ((flags & UVM_FLAG_WAITVA) == 0) { + simple_unlock(&map->flags_lock); + UVMHIST_LOG(maphist, + "<- uvm_map_findspace failed!", + 0,0,0,0); + return ENOMEM; + } else { + ltsleep(&map->header, PVM, "vmmapva", 0, + &map->flags_lock); + } } + simple_unlock(&map->flags_lock); + goto retry; } - simple_unlock(&map->flags_lock); - goto retry; } -#ifdef PMAP_GROWKERNEL - /* - * If the kernel pmap can't map the requested space, - * then allocate more resources for it. - */ - if (map == kernel_map && uvm_maxkaddr < (start + size)) - uvm_maxkaddr = pmap_growkernel(start + size); -#endif - +done: UVMMAP_EVCNT_INCR(map_call); /* @@ -2115,6 +2157,14 @@ uvm_unmap_remove(struct vm_map *map, vad } } #endif /* defined(DEBUG) */ + if (map == kernel_map) { + int s; + + s = splvm(); + vmem_xfree(kernel_va_arena, (vmem_addr_t)entry->start, + entry->end - entry->start); + splx(s); + } /* * remove entry from map and put it on our list of entries @@ -3701,6 +3751,25 @@ uvm_map_checkprot(struct vm_map *map, va struct vm_map_entry *tmp_entry; if (!uvm_map_lookup_entry(map, start, &tmp_entry)) { + if (VM_MAP_IS_KERNEL(map)) { + /* + * XXX a kludge for mem(4). + * non-pageable mappings don't have + * corresponding entries in vm_map. + * + * maybe we can iterate segments in kernel_va_arena, + * but don't try too hard to be correct here, + * because uiomove() will pick an error anyway... + */ + vaddr_t va; + + for (va = start; va < end; va += PAGE_SIZE) { + if (!pmap_extract(pmap_kernel(), va, NULL)) { + return FALSE; + } + } + return TRUE; + } return (FALSE); } entry = tmp_entry; Index: uvm/uvm_map.h =================================================================== --- uvm/uvm_map.h (revision 1773) +++ uvm/uvm_map.h (working copy) @@ -305,6 +305,15 @@ do { \ #ifdef PMAP_GROWKERNEL extern vaddr_t uvm_maxkaddr; + +static inline void +uvm_growkernel(vaddr_t endva) +{ + + if (__predict_false(uvm_maxkaddr < endva)) { + uvm_maxkaddr = pmap_growkernel(endva); + } +} #endif /* Index: ddb/db_output.c =================================================================== --- ddb/db_output.c (revision 1464) +++ ddb/db_output.c (working copy) @@ -62,7 +62,8 @@ __KERNEL_RCSID(0, "$NetBSD: db_output.c, */ #ifndef DB_MAX_LINE -#define DB_MAX_LINE 24 /* maximum line */ +//#define DB_MAX_LINE 24 /* maximum line */ +#define DB_MAX_LINE 1000 /* maximum line */ #endif /* DB_MAX_LINE */ #ifndef DB_MAX_WIDTH #define DB_MAX_WIDTH 80 /* maximum width */ Index: sys/vmem.h =================================================================== --- sys/vmem.h (revision 1848) +++ sys/vmem.h (working copy) @@ -65,4 +65,7 @@ boolean_t vmem_reap(vmem_t *); #define VM_INSTANTFIT 0x00001000 #define VM_BESTFIT 0x00002000 +#define VMC_KVA 0x00010000 /* kernel_va_arena */ +#define VMC_KMEM 0x00020000 /* kmem_arena */ + #endif /* !_SYS_VMEM_H_ */