Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/miscfs/genfs/genfs_vnops.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/miscfs/genfs/genfs_vnops.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.150 retrieving revision 1.150.2.13 diff -u -p -r1.150 -r1.150.2.13 --- src/sys/miscfs/genfs/genfs_vnops.c 2007/03/04 06:03:14 1.150 +++ src/sys/miscfs/genfs/genfs_vnops.c 2007/09/01 15:34:15 1.150.2.13 @@ -1,4 +1,4 @@ -/* $NetBSD: genfs_vnops.c,v 1.150 2007/03/04 06:03:14 christos Exp $ */ +/* $NetBSD: genfs_vnops.c,v 1.150.2.13 2007/09/01 15:34:15 yamt Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1993 @@ -31,7 +31,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: genfs_vnops.c,v 1.150 2007/03/04 06:03:14 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: genfs_vnops.c,v 1.150.2.13 2007/09/01 15:34:15 yamt Exp $"); #include #include @@ -239,45 +239,44 @@ genfs_revoke(void *v) #endif vp = ap->a_vp; - simple_lock(&vp->v_interlock); + mutex_enter(&vp->v_interlock); - if (vp->v_flag & VALIASED) { + if (vp->v_iflag & VI_ALIASED) { /* * If a vgone (or vclean) is already in progress, * wait until it is done and return. */ - if (vp->v_flag & VXLOCK) { - vp->v_flag |= VXWANT; - ltsleep(vp, PINOD|PNORELOCK, "vop_revokeall", 0, - &vp->v_interlock); + if (vp->v_iflag & VI_XLOCK) { + vwait(vp, VI_XLOCK); + mutex_exit(&vp->v_interlock); return (0); } /* * Ensure that vp will not be vgone'd while we * are eliminating its aliases. */ - vp->v_flag |= VXLOCK; - simple_unlock(&vp->v_interlock); - while (vp->v_flag & VALIASED) { - simple_lock(&spechash_slock); + vp->v_iflag |= VI_XLOCK; + mutex_exit(&vp->v_interlock); + while (vp->v_iflag & VI_ALIASED) { + mutex_enter(&spechash_lock); for (vq = *vp->v_hashchain; vq; vq = vq->v_specnext) { if (vq->v_rdev != vp->v_rdev || vq->v_type != vp->v_type || vp == vq) continue; - simple_unlock(&spechash_slock); + mutex_exit(&spechash_lock); vgone(vq); break; } if (vq == NULLVP) - simple_unlock(&spechash_slock); + mutex_exit(&spechash_lock); } /* * Remove the lock so that vgone below will * really eliminate the vnode after which time * vgone will awaken any sleepers. */ - simple_lock(&vp->v_interlock); - vp->v_flag &= ~VXLOCK; + mutex_enter(&vp->v_interlock); + vp->v_iflag &= ~VI_XLOCK; } vgonel(vp, l); return (0); @@ -345,7 +344,7 @@ genfs_nolock(void *v) * the interlock here. */ if (ap->a_flags & LK_INTERLOCK) - simple_unlock(&ap->a_vp->v_interlock); + mutex_exit(&ap->a_vp->v_interlock); return (0); } @@ -394,9 +393,9 @@ genfs_rel_pages(struct vm_page **pgs, in pg->flags |= PG_RELEASED; } } - uvm_lock_pageq(); + mutex_enter(&uvm_pageqlock); uvm_page_unbusy(pgs, npages); - uvm_unlock_pageq(); + mutex_exit(&uvm_pageqlock); } /* @@ -425,7 +424,7 @@ genfs_getpages(void *v) int i, error, npages, orignpages, npgs, run, ridx, pidx, pcount; int fs_bshift, fs_bsize, dev_bshift; int flags = ap->a_flags; - size_t bytes, iobytes, tailbytes, totalbytes, skipbytes; + size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; vaddr_t kva; struct buf *bp, *mbp; struct vnode *vp = ap->a_vp; @@ -463,13 +462,23 @@ startover: origvsize = vp->v_size; origoffset = ap->a_offset; orignpages = *ap->a_count; - GOP_SIZE(vp, vp->v_size, &diskeof, 0); + GOP_SIZE(vp, origvsize, &diskeof, 0); if (flags & PGO_PASTEOF) { - newsize = MAX(vp->v_size, +#if defined(DIAGNOSTIC) + off_t writeeof; +#endif /* defined(DIAGNOSTIC) */ + + newsize = MAX(origvsize, origoffset + (orignpages << PAGE_SHIFT)); GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); +#if defined(DIAGNOSTIC) + GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); + if (newsize > round_page(writeeof)) { + panic("%s: past eof", __func__); + } +#endif /* defined(DIAGNOSTIC) */ } else { - GOP_SIZE(vp, vp->v_size, &memeof, GOP_SIZE_MEM); + GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); } KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); @@ -481,7 +490,7 @@ startover: if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { if ((flags & PGO_LOCKED) == 0) { - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); } UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", origoffset, *ap->a_count, memeof,0); @@ -509,11 +518,11 @@ startover: if (write) { gp->g_dirtygen++; - if ((vp->v_flag & VONWORKLST) == 0) { + if ((vp->v_iflag & VI_ONWORKLST) == 0) { vn_syncer_add_to_worklist(vp, filedelay); } - if ((vp->v_flag & (VWRITEMAP|VWRITEMAPDIRTY)) == VWRITEMAP) { - vp->v_flag |= VWRITEMAPDIRTY; + if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { + vp->v_iflag |= VI_WRMAPDIRTY; } } @@ -558,7 +567,7 @@ startover: error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); goto out_err; } - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); /* * find the requested pages and make some simple checks. @@ -599,11 +608,10 @@ startover: UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", ridx, npages, startoffset, endoffset); - if (!has_trans && - (error = fstrans_start(vp->v_mount, FSTRANS_SHARED)) != 0) { - goto out_err; + if (!has_trans) { + fstrans_start(vp->v_mount, FSTRANS_SHARED); + has_trans = true; } - has_trans = true; /* * hold g_glock to prevent a race with truncate. @@ -616,7 +624,7 @@ startover: } else { rw_enter(&gp->g_glock, RW_READER); } - simple_lock(&uobj->vmobjlock); + mutex_enter(&uobj->vmobjlock); if (vp->v_size < origvsize) { rw_exit(&gp->g_glock); if (pgs != pgs_onstack) @@ -629,7 +637,7 @@ startover: rw_exit(&gp->g_glock); KASSERT(async != 0); genfs_rel_pages(&pgs[ridx], orignpages); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); error = EBUSY; goto out_err; } @@ -696,12 +704,12 @@ startover: rw_exit(&gp->g_glock); KASSERT(async != 0); genfs_rel_pages(pgs, npages); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); error = EBUSY; goto out_err; } } - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); /* * read the desired page(s). @@ -715,13 +723,18 @@ startover: kva = uvm_pagermapin(pgs, npages, UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); - mbp = getiobuf(); + mbp = getiobuf(vp, true); mbp->b_bufsize = totalbytes; mbp->b_data = (void *)kva; mbp->b_resid = mbp->b_bcount = bytes; - mbp->b_flags = B_BUSY|B_READ| (async ? B_CALL|B_ASYNC : 0); - mbp->b_iodone = (async ? uvm_aio_biodone : 0); - mbp->b_vp = vp; + mbp->b_cflags = BC_BUSY; + if (async) { + mbp->b_flags = B_READ | B_ASYNC; + mbp->b_iodone = uvm_aio_biodone; + } else { + mbp->b_flags = B_READ; + mbp->b_iodone = NULL; + } if (async) BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); else @@ -729,20 +742,22 @@ startover: /* * if EOF is in the middle of the range, zero the part past EOF. - * if the page including EOF is not PG_FAKE, skip over it since - * in that case it has valid data that we need to preserve. + * skip over pages which are not PG_FAKE since in that case they have + * valid data that we need to preserve. */ - if (tailbytes > 0) { - size_t tailstart = bytes; - - if ((pgs[bytes >> PAGE_SHIFT]->flags & PG_FAKE) == 0) { - tailstart = round_page(tailstart); - tailbytes -= tailstart - bytes; + tailstart = bytes; + while (tailbytes > 0) { + const int len = PAGE_SIZE - (tailstart & PAGE_MASK); + + KASSERT(len <= tailbytes); + if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { + memset((void *)(kva + tailstart), 0, len); + UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", + kva, tailstart, len, 0); } - UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", - kva, tailstart, tailbytes,0); - memset((void *)(kva + tailstart), 0, tailbytes); + tailstart += len; + tailbytes -= len; } /* @@ -847,7 +862,7 @@ startover: if (offset == startoffset && iobytes == bytes) { bp = mbp; } else { - bp = getiobuf(); + bp = getiobuf(vp, true); nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); } bp->b_lblkno = 0; @@ -902,7 +917,7 @@ loopdone: } } rw_exit(&gp->g_glock); - simple_lock(&uobj->vmobjlock); + mutex_enter(&uobj->vmobjlock); /* * we're almost done! release the pages... @@ -922,10 +937,10 @@ loopdone: pgs[i]->flags |= PG_RELEASED; } } - uvm_lock_pageq(); + mutex_enter(&uvm_pageqlock); uvm_page_unbusy(pgs, npages); - uvm_unlock_pageq(); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uvm_pageqlock); + mutex_exit(&uobj->vmobjlock); UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); goto out_err; } @@ -933,7 +948,7 @@ loopdone: out: UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); error = 0; - uvm_lock_pageq(); + mutex_enter(&uvm_pageqlock); for (i = 0; i < npages; i++) { pg = pgs[i]; if (pg == NULL) { @@ -965,8 +980,8 @@ out: UVM_PAGE_OWN(pg, NULL); } } - uvm_unlock_pageq(); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uvm_pageqlock); + mutex_exit(&uobj->vmobjlock); if (ap->a_m != NULL) { memcpy(ap->a_m, &pgs[ridx], orignpages * sizeof(struct vm_page *)); @@ -1037,21 +1052,26 @@ genfs_putpages(void *v) voff_t a_offhi; int a_flags; } */ *ap = v; - struct vnode *vp = ap->a_vp; + + return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, + ap->a_flags, NULL); +} + +int +genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, int flags, + struct vm_page **busypg) +{ struct uvm_object *uobj = &vp->v_uobj; - struct simplelock *slock = &uobj->vmobjlock; - off_t startoff = ap->a_offlo; - off_t endoff = ap->a_offhi; + kmutex_t *slock = &uobj->vmobjlock; off_t off; - int flags = ap->a_flags; /* Even for strange MAXPHYS, the shift rounds down to a page */ #define maxpages (MAXPHYS >> PAGE_SHIFT) - int i, s, error, npages, nback; + int i, error, npages, nback; int freeflag; struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; bool wasclean, by_list, needs_clean, yld; bool async = (flags & PGO_SYNCIO) == 0; - bool pagedaemon = curproc == uvm.pagedaemon_proc; + bool pagedaemon = curlwp == uvm.pagedaemon_lwp; struct lwp *l = curlwp ? curlwp : &lwp0; struct genfs_node *gp = VTOG(vp); int dirtygen; @@ -1068,17 +1088,15 @@ genfs_putpages(void *v) UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", vp, uobj->uo_npages, startoff, endoff - startoff); - KASSERT((vp->v_flag & VONWORKLST) != 0 || - (vp->v_flag & VWRITEMAPDIRTY) == 0); + KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || + (vp->v_iflag & VI_WRMAPDIRTY) == 0); if (uobj->uo_npages == 0) { - s = splbio(); - if (vp->v_flag & VONWORKLST) { - vp->v_flag &= ~VWRITEMAPDIRTY; + if (vp->v_iflag & VI_ONWORKLST) { + vp->v_iflag &= ~VI_WRMAPDIRTY; if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) vn_syncer_remove_from_worklist(vp); } - splx(s); - simple_unlock(slock); + mutex_exit(slock); return (0); } @@ -1087,23 +1105,19 @@ genfs_putpages(void *v) */ if ((flags & PGO_CLEANIT) != 0) { - simple_unlock(slock); - if (pagedaemon) + mutex_exit(slock); + if (pagedaemon) { error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); - else - error = fstrans_start(vp->v_mount, FSTRANS_LAZY); - if (error) - return error; + if (error) + return error; + } else + fstrans_start(vp->v_mount, FSTRANS_LAZY); has_trans = true; - simple_lock(slock); + mutex_enter(slock); } error = 0; - s = splbio(); - simple_lock(&global_v_numoutput_slock); wasclean = (vp->v_numoutput == 0); - simple_unlock(&global_v_numoutput_slock); - splx(s); off = startoff; if (endoff == 0 || flags & PGO_ALLPAGES) { endoff = trunc_page(LLONG_MAX); @@ -1117,7 +1131,7 @@ genfs_putpages(void *v) * don't bother to clean it out. */ - if ((vp->v_flag & VONWORKLST) == 0) { + if ((vp->v_iflag & VI_ONWORKLST) == 0) { if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { goto skip_scan; } @@ -1134,7 +1148,7 @@ genfs_putpages(void *v) cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && startoff == 0 && endoff == trunc_page(LLONG_MAX) && - (vp->v_flag & VONWORKLST) != 0; + (vp->v_iflag & VI_ONWORKLST) != 0; dirtygen = gp->g_dirtygen; freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; if (by_list) { @@ -1146,7 +1160,7 @@ genfs_putpages(void *v) endmp.flags = PG_BUSY; pg = TAILQ_FIRST(&uobj->memq); TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq); - PHOLD(l); + uvm_lwp_hold(l); } else { pg = uvm_pagelookup(uobj, off); } @@ -1197,22 +1211,30 @@ genfs_putpages(void *v) if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); error = EDEADLK; + if (busypg != NULL) + *busypg = pg; + break; + } + if (pagedaemon) { + /* + * someone has taken the page while we + * dropped the lock for fstrans_start. + */ break; } - KASSERT(!pagedaemon); if (by_list) { TAILQ_INSERT_BEFORE(pg, &curmp, listq); UVMHIST_LOG(ubchist, "curmp next %p", TAILQ_NEXT(&curmp, listq), 0,0,0); } if (yld) { - simple_unlock(slock); + mutex_exit(slock); preempt(); - simple_lock(slock); + mutex_enter(slock); } else { pg->flags |= PG_WANTED; UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); - simple_lock(slock); + mutex_enter(slock); } if (by_list) { UVMHIST_LOG(ubchist, "after next %p", @@ -1272,7 +1294,7 @@ genfs_putpages(void *v) */ if (needs_clean) { - KDASSERT((vp->v_flag & VONWORKLST)); + KDASSERT((vp->v_iflag & VI_ONWORKLST)); wasclean = false; memset(pgs, 0, sizeof(pgs)); pg->flags |= PG_BUSY; @@ -1324,7 +1346,7 @@ genfs_putpages(void *v) */ if (flags & (PGO_DEACTIVATE|PGO_FREE)) { - uvm_lock_pageq(); + mutex_enter(&uvm_pageqlock); } for (i = 0; i < npages; i++) { tpg = pgs[i]; @@ -1341,7 +1363,7 @@ genfs_putpages(void *v) if (tpg->flags & PG_BUSY) { tpg->flags |= freeflag; if (pagedaemon) { - uvmexp.paging++; + uvm_pageout_start(1); uvm_pagedequeue(tpg); } } else { @@ -1360,7 +1382,7 @@ genfs_putpages(void *v) } } if (flags & (PGO_DEACTIVATE|PGO_FREE)) { - uvm_unlock_pageq(); + mutex_exit(&uvm_pageqlock); } if (needs_clean) { modified = true; @@ -1374,9 +1396,9 @@ genfs_putpages(void *v) TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, listq); } - simple_unlock(slock); + mutex_exit(slock); error = GOP_WRITE(vp, pgs, npages, flags); - simple_lock(slock); + mutex_enter(slock); if (by_list) { pg = TAILQ_NEXT(&curmp, listq); TAILQ_REMOVE(&uobj->memq, &curmp, listq); @@ -1409,10 +1431,10 @@ genfs_putpages(void *v) } if (by_list) { TAILQ_REMOVE(&uobj->memq, &endmp, listq); - PRELE(l); + uvm_lwp_rele(l); } - if (modified && (vp->v_flag & VWRITEMAPDIRTY) != 0 && + if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && (vp->v_type != VBLK || (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); @@ -1424,35 +1446,23 @@ genfs_putpages(void *v) * and we're doing sync i/o, wait for all writes to finish. */ - s = splbio(); if (cleanall && wasclean && gp->g_dirtygen == dirtygen && - (vp->v_flag & VONWORKLST) != 0) { - vp->v_flag &= ~VWRITEMAPDIRTY; + (vp->v_iflag & VI_ONWORKLST) != 0) { + vp->v_iflag &= ~VI_WRMAPDIRTY; if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) vn_syncer_remove_from_worklist(vp); } - splx(s); #if !defined(DEBUG) skip_scan: #endif /* !defined(DEBUG) */ - if (!wasclean && !async) { - s = splbio(); - /* - * XXX - we want simple_unlock(&global_v_numoutput_slock); - * but the slot in ltsleep() is taken! - * XXX - try to recover from missed wakeups with a timeout.. - * must think of something better. - */ - while (vp->v_numoutput != 0) { - vp->v_flag |= VBWAIT; - UVM_UNLOCK_AND_WAIT(&vp->v_numoutput, slock, false, - "genput2", hz); - simple_lock(slock); - } - splx(s); + + /* Wait for output to complete. */ + if (!wasclean && !async && vp->v_numoutput != 0) { + while (vp->v_numoutput != 0) + cv_wait(&vp->v_cv, slock); } - simple_unlock(slock); + mutex_exit(slock); if (has_trans) fstrans_done(vp->v_mount); @@ -1508,7 +1518,8 @@ genfs_do_io(struct vnode *vp, off_t off, UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", vp, kva, len, flags); - GOP_SIZE(vp, vp->v_size, &eof, 0); + KASSERT(vp->v_size <= vp->v_writesize); + GOP_SIZE(vp, vp->v_writesize, &eof, 0); if (vp->v_type != VBLK) { fs_bshift = vp->v_mount->mnt_fs_bshift; dev_bshift = vp->v_mount->mnt_dev_bshift; @@ -1523,22 +1534,25 @@ genfs_do_io(struct vnode *vp, off_t off, KASSERT(bytes != 0); if (write) { - s = splbio(); - simple_lock(&global_v_numoutput_slock); + mutex_enter(&vp->v_interlock); vp->v_numoutput += 2; - simple_unlock(&global_v_numoutput_slock); - splx(s); + mutex_exit(&vp->v_interlock); } - mbp = getiobuf(); + mbp = getiobuf(vp, true); UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", vp, mbp, vp->v_numoutput, bytes); mbp->b_bufsize = len; mbp->b_data = (void *)kva; mbp->b_resid = mbp->b_bcount = bytes; - mbp->b_flags = B_BUSY | brw | B_AGE | (async ? (B_CALL | B_ASYNC) : 0); - mbp->b_iodone = iodone; - mbp->b_vp = vp; - if (curproc == uvm.pagedaemon_proc) + mbp->b_cflags = BC_BUSY | BC_AGE; + if (async) { + mbp->b_flags = brw | B_ASYNC; + mbp->b_iodone = iodone; + } else { + mbp->b_flags = brw; + mbp->b_iodone = NULL; + } + if (curlwp == uvm.pagedaemon_lwp) BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); else if (async) BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); @@ -1575,7 +1589,7 @@ genfs_do_io(struct vnode *vp, off_t off, } else { UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", vp, bp, vp->v_numoutput, 0); - bp = getiobuf(); + bp = getiobuf(vp, true); nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); } bp->b_lblkno = 0; @@ -1622,7 +1636,7 @@ genfs_null_putpages(void *v) struct vnode *vp = ap->a_vp; KASSERT(vp->v_uobj.uo_npages == 0); - simple_unlock(&vp->v_interlock); + mutex_exit(&vp->v_interlock); return (0); } @@ -1682,7 +1696,7 @@ genfs_compat_getpages(void *v) orignpages = *ap->a_count; pgs = ap->a_m; - if (write && (vp->v_flag & VONWORKLST) == 0) { + if (write && (vp->v_iflag & VI_ONWORKLST) == 0) { vn_syncer_add_to_worklist(vp, filedelay); } if (ap->a_flags & PGO_LOCKED) { @@ -1692,16 +1706,16 @@ genfs_compat_getpages(void *v) return (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); } if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); return (EINVAL); } if ((ap->a_flags & PGO_SYNCIO) == 0) { - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); return 0; } npages = orignpages; uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uobj->vmobjlock); kva = uvm_pagermapin(pgs, npages, UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); for (i = 0; i < npages; i++) { @@ -1727,8 +1741,8 @@ genfs_compat_getpages(void *v) } } uvm_pagermapout(kva, npages); - simple_lock(&uobj->vmobjlock); - uvm_lock_pageq(); + mutex_enter(&uobj->vmobjlock); + mutex_enter(&uvm_pageqlock); for (i = 0; i < npages; i++) { pg = pgs[i]; if (error && (pg->flags & PG_FAKE) != 0) { @@ -1741,8 +1755,8 @@ genfs_compat_getpages(void *v) if (error) { uvm_page_unbusy(pgs, npages); } - uvm_unlock_pageq(); - simple_unlock(&uobj->vmobjlock); + mutex_exit(&uvm_pageqlock); + mutex_exit(&uobj->vmobjlock); return (error); } @@ -1756,7 +1770,7 @@ genfs_compat_gop_write(struct vnode *vp, kauth_cred_t cred = curlwp->l_cred; struct buf *bp; vaddr_t kva; - int s, error; + int error; offset = pgs[0]->offset; kva = uvm_pagermapin(pgs, npages, @@ -1773,22 +1787,18 @@ genfs_compat_gop_write(struct vnode *vp, /* XXX vn_lock */ error = VOP_WRITE(vp, &uio, 0, cred); - s = splbio(); - V_INCR_NUMOUTPUT(vp); - splx(s); + mutex_enter(&vp->v_interlock); + vp->v_numoutput++; + mutex_exit(&vp->v_interlock); - bp = getiobuf(); - bp->b_flags = B_BUSY | B_WRITE | B_AGE; - bp->b_vp = vp; + bp = getiobuf(vp, true); + bp->b_cflags = BC_BUSY | BC_AGE; bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; bp->b_data = (char *)kva; bp->b_bcount = npages << PAGE_SHIFT; bp->b_bufsize = npages << PAGE_SHIFT; bp->b_resid = 0; - if (error) { - bp->b_flags |= B_ERROR; - bp->b_error = error; - } + bp->b_error = error; uvm_aio_aiodone(bp); return (error); } @@ -1826,7 +1836,7 @@ genfs_directio(struct vnode *vp, struct * buffered I/O if the vnode is mapped to avoid this mess. */ - if (vp->v_flag & VMAPPED) { + if (vp->v_vflag & VV_MAPPED) { return; } @@ -1885,15 +1895,14 @@ genfs_directio(struct vnode *vp, struct static void genfs_dio_iodone(struct buf *bp) { - int s; KASSERT((bp->b_flags & B_ASYNC) == 0); - s = splbio(); - if ((bp->b_flags & (B_READ | B_AGE)) == B_AGE) { + if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { + mutex_enter(bp->b_objlock); vwakeup(bp); + mutex_exit(bp->b_objlock); } putiobuf(bp); - splx(s); } /* @@ -1948,7 +1957,7 @@ genfs_do_directio(struct vmspace *vs, va spoff = trunc_page(off); epoff = round_page(off + len); - simple_lock(&vp->v_interlock); + mutex_enter(&vp->v_interlock); error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); if (error) { return error;