Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/ufs/ufs/ufs_lookup.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/ufs/ufs/ufs_lookup.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.47.2.6 retrieving revision 1.99.2.2 diff -u -p -r1.47.2.6 -r1.99.2.2 --- src/sys/ufs/ufs/ufs_lookup.c 2004/09/21 13:39:22 1.47.2.6 +++ src/sys/ufs/ufs/ufs_lookup.c 2009/03/03 18:34:40 1.99.2.2 @@ -1,4 +1,4 @@ -/* $NetBSD: ufs_lookup.c,v 1.47.2.6 2004/09/21 13:39:22 skrll Exp $ */ +/* $NetBSD: ufs_lookup.c,v 1.99.2.2 2009/03/03 18:34:40 skrll Exp $ */ /* * Copyright (c) 1989, 1993 @@ -37,7 +37,12 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ufs_lookup.c,v 1.47.2.6 2004/09/21 13:39:22 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ufs_lookup.c,v 1.99.2.2 2009/03/03 18:34:40 skrll Exp $"); + +#ifdef _KERNEL_OPT +#include "opt_ffs.h" +#include "fs_ffs.h" +#endif #include #include @@ -48,12 +53,21 @@ __KERNEL_RCSID(0, "$NetBSD: ufs_lookup.c #include #include #include +#include +#include +#include +#include +#include #include #include +#ifdef UFS_DIRHASH +#include +#endif #include #include #include +#include #ifdef DIAGNOSTIC int dirchk = 1; @@ -61,7 +75,7 @@ int dirchk = 1; int dirchk = 0; #endif -#define FSFMT(vp) (((vp)->v_mount->mnt_iflag & IMNT_DTYPE) == 0) +#define FSFMT(vp) (((vp)->v_mount->mnt_iflag & IMNT_DTYPE) == 0) /* * Convert a component of a pathname into a pointer to a locked inode. @@ -97,8 +111,7 @@ int dirchk = 0; * nor deleting, add name to cache */ int -ufs_lookup(v) - void *v; +ufs_lookup(void *v) { struct vop_lookup_args /* { struct vnode *a_dvp; @@ -122,12 +135,10 @@ ufs_lookup(v) struct vnode *tdp; /* returned by VFS_VGET */ doff_t enduseful; /* pointer past last used dir slot */ u_long bmask; /* block offset mask */ - int lockparent; /* 1 => lockparent flag is set */ - int wantparent; /* 1 => wantparent or lockparent flag */ int namlen, error; struct vnode **vpp = ap->a_vpp; struct componentname *cnp = ap->a_cnp; - struct ucred *cred = cnp->cn_cred; + kauth_cred_t cred = cnp->cn_cred; int flags; int nameiop = cnp->cn_nameiop; struct ufsmount *ump = dp->i_ump; @@ -135,22 +146,20 @@ ufs_lookup(v) int dirblksiz = ump->um_dirblksiz; ino_t foundino; - cnp->cn_flags &= ~PDIRUNLOCK; flags = cnp->cn_flags; bp = NULL; slotoffset = -1; *vpp = NULL; - lockparent = flags & LOCKPARENT; - wantparent = flags & (LOCKPARENT|WANTPARENT); + endsearch = 0; /* silence compiler warning */ /* * Check accessiblity of directory. */ - if ((error = VOP_ACCESS(vdp, VEXEC, cred, cnp->cn_lwp)) != 0) + if ((error = VOP_ACCESS(vdp, VEXEC, cred)) != 0) return (error); if ((flags & ISLASTCN) && (vdp->v_mount->mnt_flag & MNT_RDONLY) && - (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) + (nameiop == DELETE || nameiop == RENAME)) return (EROFS); /* @@ -160,8 +169,11 @@ ufs_lookup(v) * check the name cache to see if the directory/name pair * we are looking for is known already. */ - if ((error = cache_lookup(vdp, vpp, cnp)) >= 0) + if ((error = cache_lookup(vdp, vpp, cnp)) >= 0) { return (error); + } + + fstrans_start(vdp->v_mount, FSTRANS_SHARED); /* * Suppress search for slots unless creating @@ -189,6 +201,46 @@ ufs_lookup(v) * of simplicity. */ bmask = vdp->v_mount->mnt_stat.f_iosize - 1; + +#ifdef UFS_DIRHASH + /* + * Use dirhash for fast operations on large directories. The logic + * to determine whether to hash the directory is contained within + * ufsdirhash_build(); a zero return means that it decided to hash + * this directory and it successfully built up the hash table. + */ + if (ufsdirhash_build(dp) == 0) { + /* Look for a free slot if needed. */ + enduseful = dp->i_size; + if (slotstatus != FOUND) { + slotoffset = ufsdirhash_findfree(dp, slotneeded, + &slotsize); + if (slotoffset >= 0) { + slotstatus = COMPACT; + enduseful = ufsdirhash_enduseful(dp); + if (enduseful < 0) + enduseful = dp->i_size; + } + } + /* Look up the component. */ + numdirpasses = 1; + entryoffsetinblock = 0; /* silence compiler warning */ + switch (ufsdirhash_lookup(dp, cnp->cn_nameptr, cnp->cn_namelen, + &dp->i_offset, &bp, nameiop == DELETE ? &prevoff : NULL)) { + case 0: + ep = (struct direct *)((char *)bp->b_data + + (dp->i_offset & bmask)); + goto foundentry; + case ENOENT: + dp->i_offset = roundup(dp->i_size, dirblksiz); + goto notfound; + default: + /* Something failed; just do a linear search. */ + break; + } + } +#endif /* UFS_DIRHASH */ + if (nameiop != LOOKUP || dp->i_diroff == 0 || dp->i_diroff >= dp->i_size) { entryoffsetinblock = 0; @@ -197,8 +249,9 @@ ufs_lookup(v) } else { dp->i_offset = dp->i_diroff; if ((entryoffsetinblock = dp->i_offset & bmask) && - (error = VOP_BLKATOFF(vdp, (off_t)dp->i_offset, NULL, &bp))) - return (error); + (error = ufs_blkatoff(vdp, (off_t)dp->i_offset, + NULL, &bp, false))) + goto out; numdirpasses = 2; nchstats.ncs_2passes++; } @@ -209,21 +262,21 @@ ufs_lookup(v) searchloop: while (dp->i_offset < endsearch) { if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) - preempt(1); + preempt(); /* * If necessary, get the next directory block. */ if ((dp->i_offset & bmask) == 0) { if (bp != NULL) - brelse(bp); - error = VOP_BLKATOFF(vdp, (off_t)dp->i_offset, NULL, - &bp); + brelse(bp, 0); + error = ufs_blkatoff(vdp, (off_t)dp->i_offset, NULL, + &bp, false); if (error) - return (error); + goto out; entryoffsetinblock = 0; } /* - * If still looking for a slot, and at a DIRBLKSIZE + * If still looking for a slot, and at a DIRBLKSIZ * boundary, have to start looking for free space again. */ if (slotstatus == NONE && @@ -238,6 +291,7 @@ searchloop: * directory. Complete checks can be run by patching * "dirchk" to be true. */ + KASSERT(bp != NULL); ep = (struct direct *)((char *)bp->b_data + entryoffsetinblock); if (ep->d_reclen == 0 || (dirchk && ufs_dirbadentry(vdp, ep, entryoffsetinblock))) { @@ -292,7 +346,7 @@ searchloop: else namlen = ep->d_namlen; #else - if (FSFMT(vdp) && needswap != 0) + if (FSFMT(vdp) && needswap != 0) namlen = ep->d_type; else namlen = ep->d_namlen; @@ -300,6 +354,9 @@ searchloop: if (namlen == cnp->cn_namelen && !memcmp(cnp->cn_nameptr, ep->d_name, (unsigned)namlen)) { +#ifdef UFS_DIRHASH +foundentry: +#endif /* * Save directory entry's inode number and * reclen in ndp->ni_ufs area, and release @@ -355,7 +412,7 @@ notfound: goto searchloop; } if (bp != NULL) - brelse(bp); + brelse(bp, 0); /* * If creating, and at end of pathname and current * directory has not been removed, then can consider @@ -365,14 +422,14 @@ notfound: (nameiop == DELETE && (ap->a_cnp->cn_flags & DOWHITEOUT) && (ap->a_cnp->cn_flags & ISWHITEOUT))) && - (flags & ISLASTCN) && dp->i_ffs_effnlink != 0) { + (flags & ISLASTCN) && dp->i_nlink != 0) { /* * Access for write is interpreted as allowing * creation of files in the directory. */ - error = VOP_ACCESS(vdp, VWRITE, cred, cnp->cn_lwp); + error = VOP_ACCESS(vdp, VWRITE, cred); if (error) - return (error); + goto out; /* * Return an indication of where the new directory * entry should be put. If we didn't find a slot, @@ -416,18 +473,16 @@ notfound: * information cannot be used. */ cnp->cn_flags |= SAVENAME; - if (!lockparent) { - VOP_UNLOCK(vdp, 0); - cnp->cn_flags |= PDIRUNLOCK; - } - return (EJUSTRETURN); + error = EJUSTRETURN; + goto out; } /* * Insert name into cache (as non-existent) if appropriate. */ if ((cnp->cn_flags & MAKEENTRY) && nameiop != CREATE) cache_enter(vdp, *vpp, cnp); - return (ENOENT); + error = ENOENT; + goto out; found: if (numdirpasses == 2) @@ -441,8 +496,9 @@ found: dp->i_size = dp->i_offset + DIRSIZ(FSFMT(vdp), ep, needswap); DIP_ASSIGN(dp, size, dp->i_size); dp->i_flag |= IN_CHANGE | IN_UPDATE; + UFS_WAPBL_UPDATE(vdp, NULL, NULL, UPDATE_DIROP); } - brelse(bp); + brelse(bp, 0); /* * Found component in pathname. @@ -455,17 +511,15 @@ found: /* * If deleting, and at end of pathname, return * parameters which can be used to remove file. - * If the wantparent flag isn't set, we return only - * the directory (in ndp->ni_dvp), otherwise we go - * on and lock the inode, being careful with ".". + * Lock the inode, being careful with ".". */ if (nameiop == DELETE && (flags & ISLASTCN)) { /* * Write access to directory required to delete files. */ - error = VOP_ACCESS(vdp, VWRITE, cred, cnp->cn_lwp); + error = VOP_ACCESS(vdp, VWRITE, cred); if (error) - return (error); + goto out; /* * Return pointer to current entry in dp->i_offset, * and distance past previous entry (if there @@ -479,7 +533,8 @@ found: if (dp->i_number == foundino) { VREF(vdp); *vpp = vdp; - return (0); + error = 0; + goto out; } if (flags & ISDOTDOT) VOP_UNLOCK(vdp, 0); /* race to get the inode */ @@ -487,7 +542,7 @@ found: if (flags & ISDOTDOT) vn_lock(vdp, LK_EXCLUSIVE | LK_RETRY); if (error) - return (error); + goto out; /* * If directory is "sticky", then user must own * the directory, or the file in it, else she @@ -495,18 +550,17 @@ found: * implements append-only directories. */ if ((dp->i_mode & ISVTX) && - cred->cr_uid != 0 && - cred->cr_uid != dp->i_uid && - VTOI(tdp)->i_uid != cred->cr_uid) { + kauth_authorize_generic(cred, KAUTH_GENERIC_ISSUSER, + NULL) != 0 && + kauth_cred_geteuid(cred) != dp->i_uid && + VTOI(tdp)->i_uid != kauth_cred_geteuid(cred)) { vput(tdp); - return (EPERM); + error = EPERM; + goto out; } *vpp = tdp; - if (!lockparent) { - VOP_UNLOCK(vdp, 0); - cnp->cn_flags |= PDIRUNLOCK; - } - return (0); + error = 0; + goto out; } /* @@ -515,30 +569,29 @@ found: * Must get inode of directory entry to verify it's a * regular file, or empty directory. */ - if (nameiop == RENAME && wantparent && (flags & ISLASTCN)) { - error = VOP_ACCESS(vdp, VWRITE, cred, cnp->cn_lwp); + if (nameiop == RENAME && (flags & ISLASTCN)) { + error = VOP_ACCESS(vdp, VWRITE, cred); if (error) - return (error); + goto out; /* * Careful about locking second inode. * This can only occur if the target is ".". */ - if (dp->i_number == foundino) - return (EISDIR); + if (dp->i_number == foundino) { + error = EISDIR; + goto out; + } if (flags & ISDOTDOT) VOP_UNLOCK(vdp, 0); /* race to get the inode */ error = VFS_VGET(vdp->v_mount, foundino, &tdp); if (flags & ISDOTDOT) vn_lock(vdp, LK_EXCLUSIVE | LK_RETRY); if (error) - return (error); + goto out; *vpp = tdp; cnp->cn_flags |= SAVENAME; - if (!lockparent) { - VOP_UNLOCK(vdp, 0); - cnp->cn_flags |= PDIRUNLOCK; - } - return (0); + error = 0; + goto out; } /* @@ -563,19 +616,10 @@ found: pdp = vdp; if (flags & ISDOTDOT) { VOP_UNLOCK(pdp, 0); /* race to get the inode */ - cnp->cn_flags |= PDIRUNLOCK; error = VFS_VGET(vdp->v_mount, foundino, &tdp); + vn_lock(pdp, LK_EXCLUSIVE | LK_RETRY); if (error) { - if (vn_lock(pdp, LK_EXCLUSIVE | LK_RETRY) == 0) - cnp->cn_flags &= ~PDIRUNLOCK; - return (error); - } - if (lockparent && (flags & ISLASTCN)) { - if ((error = vn_lock(pdp, LK_EXCLUSIVE))) { - vput(tdp); - return (error); - } - cnp->cn_flags &= ~PDIRUNLOCK; + goto out; } *vpp = tdp; } else if (dp->i_number == foundino) { @@ -584,11 +628,7 @@ found: } else { error = VFS_VGET(vdp->v_mount, foundino, &tdp); if (error) - return (error); - if (!lockparent || !(flags & ISLASTCN)) { - VOP_UNLOCK(pdp, 0); - cnp->cn_flags |= PDIRUNLOCK; - } + goto out; *vpp = tdp; } @@ -597,20 +637,22 @@ found: */ if (cnp->cn_flags & MAKEENTRY) cache_enter(vdp, *vpp, cnp); - return (0); + error = 0; + +out: + fstrans_done(vdp->v_mount); + return error; } void -ufs_dirbad(ip, offset, how) - struct inode *ip; - doff_t offset; - char *how; +ufs_dirbad(struct inode *ip, doff_t offset, const char *how) { struct mount *mp; mp = ITOV(ip)->v_mount; - printf("%s: bad dir ino %d at offset %d: %s\n", - mp->mnt_stat.f_mntonname, ip->i_number, offset, how); + printf("%s: bad dir ino %llu at offset %d: %s\n", + mp->mnt_stat.f_mntonname, (unsigned long long)ip->i_number, + offset, how); if ((mp->mnt_stat.f_flag & MNT_RDONLY) == 0) panic("bad dir"); } @@ -620,14 +662,11 @@ ufs_dirbad(ip, offset, how) * record length must be multiple of 4 * entry must fit in rest of its DIRBLKSIZ block * record must be large enough to contain entry - * name is not longer than MAXNAMLEN + * name is not longer than FFS_MAXNAMLEN * name must be as long as advertised, and null terminated */ int -ufs_dirbadentry(dp, ep, entryoffsetinblock) - struct vnode *dp; - struct direct *ep; - int entryoffsetinblock; +ufs_dirbadentry(struct vnode *dp, struct direct *ep, int entryoffsetinblock) { int i; int namlen; @@ -651,13 +690,14 @@ ufs_dirbadentry(dp, ep, entryoffsetinblo dirblksiz - (entryoffsetinblock & (dirblksiz - 1)) || ufs_rw16(ep->d_reclen, needswap) < DIRSIZ(FSFMT(dp), ep, needswap) || - namlen > MAXNAMLEN) { + namlen > FFS_MAXNAMLEN) { /*return (1); */ - printf("First bad, reclen=%x, DIRSIZ=%lu, namlen=%d, flags=%x " - "entryoffsetinblock=%d, dirblksiz = %d\n", + printf("First bad, reclen=%#x, DIRSIZ=%lu, namlen=%d, " + "flags=%#x, entryoffsetinblock=%d, dirblksiz = %d\n", ufs_rw16(ep->d_reclen, needswap), (u_long)DIRSIZ(FSFMT(dp), ep, needswap), - namlen, dp->v_mount->mnt_flag, entryoffsetinblock,dirblksiz); + namlen, dp->v_mount->mnt_flag, entryoffsetinblock, + dirblksiz); goto bad; } if (ep->d_ino == 0) @@ -681,10 +721,8 @@ bad: * argument ip is the inode to which the new directory entry will refer. */ void -ufs_makedirentry(ip, cnp, newdirp) - struct inode *ip; - struct componentname *cnp; - struct direct *newdirp; +ufs_makedirentry(struct inode *ip, struct componentname *cnp, + struct direct *newdirp) { #ifdef DIAGNOSTIC if ((cnp->cn_flags & SAVENAME) == 0) @@ -710,30 +748,28 @@ ufs_makedirentry(ip, cnp, newdirp) * soft dependency code). */ int -ufs_direnter(dvp, tvp, dirp, cnp, newdirbp) - struct vnode *dvp; - struct vnode *tvp; - struct direct *dirp; - struct componentname *cnp; - struct buf *newdirbp; +ufs_direnter(struct vnode *dvp, struct vnode *tvp, struct direct *dirp, + struct componentname *cnp, struct buf *newdirbp) { - struct ucred *cr; + kauth_cred_t cr; struct lwp *l; int newentrysize; struct inode *dp; struct buf *bp; u_int dsize; struct direct *ep, *nep; - int error, ret, blkoff, loc, spacefree, flags; + int error, ret, blkoff, loc, spacefree; char *dirbuf; struct timespec ts; struct ufsmount *ump = VFSTOUFS(dvp->v_mount); const int needswap = UFS_MPNEEDSWAP(ump); int dirblksiz = ump->um_dirblksiz; + UFS_WAPBL_JLOCK_ASSERT(dvp->v_mount); + error = 0; cr = cnp->cn_cred; - l = cnp->cn_lwp; + l = curlwp; dp = VTOI(dvp); newentrysize = DIRSIZ(0, dirp, 0); @@ -747,13 +783,8 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir */ if (dp->i_offset & (dirblksiz - 1)) panic("ufs_direnter: newblk"); - flags = B_CLRBUF; - if (!DOINGSOFTDEP(dvp)) - flags |= B_SYNC; - if ((error = VOP_BALLOC(dvp, (off_t)dp->i_offset, dirblksiz, - cr, flags, &bp)) != 0) { - if (DOINGSOFTDEP(dvp) && newdirbp != NULL) - bdwrite(newdirbp); + if ((error = UFS_BALLOC(dvp, (off_t)dp->i_offset, dirblksiz, + cr, B_CLRBUF | B_SYNC, &bp)) != 0) { return (error); } dp->i_size = dp->i_offset + dirblksiz; @@ -774,51 +805,18 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir } } blkoff = dp->i_offset & (ump->um_mountp->mnt_stat.f_iosize - 1); - memcpy((caddr_t)bp->b_data + blkoff, (caddr_t)dirp, - newentrysize); - if (DOINGSOFTDEP(dvp)) { - /* - * Ensure that the entire newly allocated block is a - * valid directory so that future growth within the - * block does not have to ensure that the block is - * written before the inode. - */ - blkoff += dirblksiz; - while (blkoff < bp->b_bcount) { - ((struct direct *) - (bp->b_data + blkoff))->d_reclen = dirblksiz; - blkoff += dirblksiz; - } - if (softdep_setup_directory_add(bp, dp, dp->i_offset, - ufs_rw32(dirp->d_ino, needswap), newdirbp, 1) == 0) { - bdwrite(bp); - TIMEVAL_TO_TIMESPEC(&time, &ts); - return VOP_UPDATE(dvp, &ts, &ts, UPDATE_DIROP); - } - /* We have just allocated a directory block in an - * indirect block. Rather than tracking when it gets - * claimed by the inode, we simply do a VOP_FSYNC - * now to ensure that it is there (in case the user - * does a future fsync). Note that we have to unlock - * the inode for the entry that we just entered, as - * the VOP_FSYNC may need to lock other inodes which - * can lead to deadlock if we also hold a lock on - * the newly entered node. - */ - error = VOP_BWRITE(bp); - if (error != 0) - return (error); - if (tvp != NULL) - VOP_UNLOCK(tvp, 0); - error = VOP_FSYNC(dvp, l->l_proc->p_ucred, FSYNC_WAIT, 0, 0, l); - if (tvp != 0) - vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY); - return (error); - } else { - error = VOP_BWRITE(bp); + memcpy((char *)bp->b_data + blkoff, dirp, newentrysize); +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL) { + ufsdirhash_newblk(dp, dp->i_offset); + ufsdirhash_add(dp, dirp, dp->i_offset); + ufsdirhash_checkblock(dp, (char *)bp->b_data + blkoff, + dp->i_offset); } - TIMEVAL_TO_TIMESPEC(&time, &ts); - ret = VOP_UPDATE(dvp, &ts, &ts, UPDATE_DIROP); +#endif + error = VOP_BWRITE(bp); + vfs_timestamp(&ts); + ret = UFS_UPDATE(dvp, &ts, &ts, UPDATE_DIROP); if (error == 0) return (ret); return (error); @@ -836,7 +834,7 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir /* * Increase size of directory if entry eats into new space. * This should never push the size past a new multiple of - * DIRBLKSIZE. + * DIRBLKSIZ. * * N.B. - THIS IS AN ARTIFACT OF 4.2 AND SHOULD NEVER HAPPEN. */ @@ -844,14 +842,13 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir dp->i_size = dp->i_offset + dp->i_count; DIP_ASSIGN(dp, size, dp->i_size); dp->i_flag |= IN_CHANGE | IN_UPDATE; + UFS_WAPBL_UPDATE(dvp, NULL, NULL, UPDATE_DIROP); } /* * Get the block containing the space for the new directory entry. */ - error = VOP_BLKATOFF(dvp, (off_t)dp->i_offset, &dirbuf, &bp); + error = ufs_blkatoff(dvp, (off_t)dp->i_offset, &dirbuf, &bp, true); if (error) { - if (DOINGSOFTDEP(dvp) && newdirbp != NULL) - bdwrite(newdirbp); return (error); } /* @@ -861,28 +858,49 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir * dp->i_offset + dp->i_count would yield the space. */ ep = (struct direct *)dirbuf; - dsize = DIRSIZ(FSFMT(dvp), ep, needswap); + dsize = (ep->d_ino != 0) ? DIRSIZ(FSFMT(dvp), ep, needswap) : 0; spacefree = ufs_rw16(ep->d_reclen, needswap) - dsize; for (loc = ufs_rw16(ep->d_reclen, needswap); loc < dp->i_count; ) { + uint16_t reclen; + nep = (struct direct *)(dirbuf + loc); - if (ep->d_ino) { - /* trim the existing slot */ - ep->d_reclen = ufs_rw16(dsize, needswap); - ep = (struct direct *)((char *)ep + dsize); - } else { - /* overwrite; nothing there; header is ours */ - spacefree += dsize; + + /* Trim the existing slot (NB: dsize may be zero). */ + ep->d_reclen = ufs_rw16(dsize, needswap); + ep = (struct direct *)((char *)ep + dsize); + + reclen = ufs_rw16(nep->d_reclen, needswap); + loc += reclen; + if (nep->d_ino == 0) { + /* + * A mid-block unused entry. Such entries are + * never created by the kernel, but fsck_ffs + * can create them (and it doesn't fix them). + * + * Add up the free space, and initialise the + * relocated entry since we don't memcpy it. + */ + spacefree += reclen; + ep->d_ino = 0; + dsize = 0; + continue; } dsize = DIRSIZ(FSFMT(dvp), nep, needswap); - spacefree += ufs_rw16(nep->d_reclen, needswap) - dsize; - loc += ufs_rw16(nep->d_reclen, needswap); - if (DOINGSOFTDEP(dvp)) - softdep_change_directoryentry_offset(dp, dirbuf, - (caddr_t)nep, (caddr_t)ep, dsize); - else - memcpy((caddr_t)ep, (caddr_t)nep, dsize); + spacefree += reclen - dsize; +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL) + ufsdirhash_move(dp, nep, + dp->i_offset + ((char *)nep - dirbuf), + dp->i_offset + ((char *)ep - dirbuf)); +#endif + memcpy((void *)ep, (void *)nep, dsize); } /* + * Here, `ep' points to a directory entry containing `dsize' in-use + * bytes followed by `spacefree' unused bytes. If ep->d_ino == 0, + * then the entry is completely unused (dsize == 0). The value + * of ep->d_reclen is always indeterminate. + * * Update the pointer fields in the previous entry (if any), * copy in the new entry, and write out the block. */ @@ -912,15 +930,19 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir dirp->d_type = tmp; } } - memcpy((caddr_t)ep, (caddr_t)dirp, (u_int)newentrysize); - if (DOINGSOFTDEP(dvp)) { - softdep_setup_directory_add(bp, dp, - dp->i_offset + (caddr_t)ep - dirbuf, - ufs_rw32(dirp->d_ino, needswap), newdirbp, 0); - bdwrite(bp); - } else { - error = VOP_BWRITE(bp); - } +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL && (ep->d_ino == 0 || + dirp->d_reclen == spacefree)) + ufsdirhash_add(dp, dirp, dp->i_offset + ((char *)ep - dirbuf)); +#endif + memcpy((void *)ep, (void *)dirp, (u_int)newentrysize); +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL) + ufsdirhash_checkblock(dp, dirbuf - + (dp->i_offset & (dirblksiz - 1)), + dp->i_offset & ~(dirblksiz - 1)); +#endif + error = VOP_BWRITE(bp); dp->i_flag |= IN_CHANGE | IN_UPDATE; /* * If all went well, and the directory can be shortened, proceed @@ -930,12 +952,13 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir * lock on the newly entered node. */ if (error == 0 && dp->i_endoff && dp->i_endoff < dp->i_size) { - if (DOINGSOFTDEP(dvp) && (tvp != NULL)) - VOP_UNLOCK(tvp, 0); - (void) VOP_TRUNCATE(dvp, (off_t)dp->i_endoff, IO_SYNC, cr, l); - if (DOINGSOFTDEP(dvp) && (tvp != NULL)) - vn_lock(tvp, LK_EXCLUSIVE | LK_RETRY); +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL) + ufsdirhash_dirtrunc(dp, dp->i_endoff); +#endif + (void) UFS_TRUNCATE(dvp, (off_t)dp->i_endoff, IO_SYNC, cr); } + UFS_WAPBL_UPDATE(dvp, NULL, NULL, UPDATE_DIROP); return (error); } @@ -952,11 +975,7 @@ ufs_direnter(dvp, tvp, dirp, cnp, newdir * to the size of the previous entry. */ int -ufs_dirremove(dvp, ip, flags, isrmdir) - struct vnode *dvp; - struct inode *ip; - int flags; - int isrmdir; +ufs_dirremove(struct vnode *dvp, struct inode *ip, int flags, int isrmdir) { struct inode *dp = VTOI(dvp); struct direct *ep; @@ -966,12 +985,14 @@ ufs_dirremove(dvp, ip, flags, isrmdir) const int needswap = UFS_MPNEEDSWAP(dp->i_ump); #endif + UFS_WAPBL_JLOCK_ASSERT(dvp->v_mount); + if (flags & DOWHITEOUT) { /* * Whiteout entry: set d_ino to WINO. */ - error = VOP_BLKATOFF(dvp, (off_t)dp->i_offset, (void *)&ep, - &bp); + error = ufs_blkatoff(dvp, (off_t)dp->i_offset, (void *)&ep, + &bp, true); if (error) return (error); ep->d_ino = ufs_rw32(WINO, needswap); @@ -979,10 +1000,21 @@ ufs_dirremove(dvp, ip, flags, isrmdir) goto out; } - if ((error = VOP_BLKATOFF(dvp, - (off_t)(dp->i_offset - dp->i_count), (void *)&ep, &bp)) != 0) + if ((error = ufs_blkatoff(dvp, + (off_t)(dp->i_offset - dp->i_count), (void *)&ep, &bp, true)) != 0) return (error); +#ifdef UFS_DIRHASH + /* + * Remove the dirhash entry. This is complicated by the fact + * that `ep' is the previous entry when dp->i_count != 0. + */ + if (dp->i_dirhash != NULL) + ufsdirhash_remove(dp, (dp->i_count == 0) ? ep : + (struct direct *)((char *)ep + + ufs_rw16(ep->d_reclen, needswap)), dp->i_offset); +#endif + if (dp->i_count == 0) { /* * First entry in block: set d_ino to zero. @@ -996,32 +1028,36 @@ ufs_dirremove(dvp, ip, flags, isrmdir) ufs_rw16(ufs_rw16(ep->d_reclen, needswap) + dp->i_reclen, needswap); } + +#ifdef UFS_DIRHASH + if (dp->i_dirhash != NULL) { + int dirblksiz = ip->i_ump->um_dirblksiz; + ufsdirhash_checkblock(dp, (char *)ep - + ((dp->i_offset - dp->i_count) & (dirblksiz - 1)), + dp->i_offset & ~(dirblksiz - 1)); + } +#endif + out: - if (DOINGSOFTDEP(dvp)) { - if (ip) { - ip->i_ffs_effnlink--; - softdep_change_linkcnt(ip); - softdep_setup_remove(bp, dp, ip, isrmdir); - } - bdwrite(bp); - } else { - if (ip) { - ip->i_ffs_effnlink--; - ip->i_nlink--; - DIP_ASSIGN(ip, nlink, ip->i_nlink); - ip->i_flag |= IN_CHANGE; - } - error = VOP_BWRITE(bp); + if (ip) { + ip->i_nlink--; + DIP_ASSIGN(ip, nlink, ip->i_nlink); + ip->i_flag |= IN_CHANGE; + UFS_WAPBL_UPDATE(ITOV(ip), NULL, NULL, 0); } + error = VOP_BWRITE(bp); dp->i_flag |= IN_CHANGE | IN_UPDATE; +#ifdef FFS /* * If the last named reference to a snapshot goes away, * drop its snapshot reference so that it will be reclaimed * when last open reference goes away. */ if (ip != 0 && (ip->i_flags & SF_SNAPSHOT) != 0 && - ip->i_ffs_effnlink == 0) + ip->i_nlink == 0) ffs_snapgone(ip); + UFS_WAPBL_UPDATE(dvp, NULL, NULL, 0); +#endif return (error); } @@ -1031,43 +1067,36 @@ out: * set up by a call to namei. */ int -ufs_dirrewrite(dp, oip, newinum, newtype, isrmdir, iflags) - struct inode *dp, *oip; - ino_t newinum; - int newtype; - int isrmdir; - int iflags; +ufs_dirrewrite(struct inode *dp, struct inode *oip, ino_t newinum, int newtype, + int isrmdir, int iflags) { struct buf *bp; struct direct *ep; struct vnode *vdp = ITOV(dp); int error; - error = VOP_BLKATOFF(vdp, (off_t)dp->i_offset, (void *)&ep, &bp); + error = ufs_blkatoff(vdp, (off_t)dp->i_offset, (void *)&ep, &bp, true); if (error) return (error); ep->d_ino = ufs_rw32(newinum, UFS_MPNEEDSWAP(dp->i_ump)); if (!FSFMT(vdp)) ep->d_type = newtype; - oip->i_ffs_effnlink--; - if (DOINGSOFTDEP(vdp)) { - softdep_change_linkcnt(oip); - softdep_setup_directory_change(bp, dp, oip, newinum, isrmdir); - bdwrite(bp); - } else { - oip->i_nlink--; - DIP_ASSIGN(oip, nlink, oip->i_nlink); - oip->i_flag |= IN_CHANGE; - error = VOP_BWRITE(bp); - } + oip->i_nlink--; + DIP_ASSIGN(oip, nlink, oip->i_nlink); + oip->i_flag |= IN_CHANGE; + UFS_WAPBL_UPDATE(ITOV(oip), NULL, NULL, UPDATE_DIROP); + error = VOP_BWRITE(bp); dp->i_flag |= iflags; +#ifdef FFS /* * If the last named reference to a snapshot goes away, * drop its snapshot reference so that it will be reclaimed * when last open reference goes away. */ - if ((oip->i_flags & SF_SNAPSHOT) != 0 && oip->i_ffs_effnlink == 0) + if ((oip->i_flags & SF_SNAPSHOT) != 0 && oip->i_nlink == 0) ffs_snapgone(oip); + UFS_WAPBL_UPDATE(vdp, NULL, NULL, UPDATE_DIROP); +#endif return (error); } @@ -1081,10 +1110,7 @@ ufs_dirrewrite(dp, oip, newinum, newtype * NB: does not handle corrupted directories. */ int -ufs_dirempty(ip, parentino, cred) - struct inode *ip; - ino_t parentino; - struct ucred *cred; +ufs_dirempty(struct inode *ip, ino_t parentino, kauth_cred_t cred) { doff_t off; struct dirtemplate dbuf; @@ -1096,7 +1122,7 @@ ufs_dirempty(ip, parentino, cred) for (off = 0; off < ip->i_size; off += ufs_rw16(dp->d_reclen, needswap)) { - error = vn_rdwr(UIO_READ, ITOV(ip), (caddr_t)dp, MINDIRSIZ, off, + error = vn_rdwr(UIO_READ, ITOV(ip), (void *)dp, MINDIRSIZ, off, UIO_SYSSPACE, IO_NODELOCKED, cred, &count, NULL); /* * Since we read MINDIRSIZ, residual must @@ -1148,10 +1174,7 @@ ufs_dirempty(ip, parentino, cred) * The target is always vput before returning. */ int -ufs_checkpath(source, target, cred, l) - struct inode *source, *target; - struct ucred *cred; - struct lwp *l; +ufs_checkpath(struct inode *source, struct inode *target, kauth_cred_t cred) { struct vnode *vp = ITOV(target); int error, rootino, namlen; @@ -1173,7 +1196,7 @@ ufs_checkpath(source, target, cred, l) error = ENOTDIR; break; } - error = vn_rdwr(UIO_READ, vp, (caddr_t)&dirbuf, + error = vn_rdwr(UIO_READ, vp, (void *)&dirbuf, sizeof (struct dirtemplate), (off_t)0, UIO_SYSSPACE, IO_NODELOCKED, cred, NULL, NULL); if (error != 0) @@ -1217,3 +1240,71 @@ out: vput(vp); return (error); } + +#define UFS_DIRRABLKS 0 +int ufs_dirrablks = UFS_DIRRABLKS; + +/* + * ufs_blkatoff: Return buffer with the contents of block "offset" from + * the beginning of directory "vp". If "res" is non-zero, fill it in with + * a pointer to the remaining space in the directory. If the caller intends + * to modify the buffer returned, "modify" must be true. + */ + +int +ufs_blkatoff(struct vnode *vp, off_t offset, char **res, struct buf **bpp, + bool modify) +{ + struct inode *ip; + struct buf *bp; + daddr_t lbn; + const int dirrablks = ufs_dirrablks; + daddr_t *blks; + int *blksizes; + int run, error; + struct mount *mp = vp->v_mount; + const int bshift = mp->mnt_fs_bshift; + const int bsize = 1 << bshift; + off_t eof; + + blks = kmem_alloc((1 + dirrablks) * sizeof(daddr_t), KM_SLEEP); + blksizes = kmem_alloc((1 + dirrablks) * sizeof(int), KM_SLEEP); + ip = VTOI(vp); + KASSERT(vp->v_size == ip->i_size); + GOP_SIZE(vp, vp->v_size, &eof, 0); + lbn = offset >> bshift; + + for (run = 0; run <= dirrablks;) { + const off_t curoff = lbn << bshift; + const int size = MIN(eof - curoff, bsize); + + if (size == 0) { + break; + } + KASSERT(curoff < eof); + blks[run] = lbn; + blksizes[run] = size; + lbn++; + run++; + if (size != bsize) { + break; + } + } + KASSERT(run >= 1); + error = breadn(vp, blks[0], blksizes[0], &blks[1], &blksizes[1], + run - 1, NOCRED, (modify ? B_MODIFY : 0), &bp); + if (error != 0) { + brelse(bp, 0); + *bpp = NULL; + goto out; + } + if (res) { + *res = (char *)bp->b_data + (offset & (bsize - 1)); + } + *bpp = bp; + + out: + kmem_free(blks, (1 + dirrablks) * sizeof(daddr_t)); + kmem_free(blksizes, (1 + dirrablks) * sizeof(int)); + return error; +}