[BACK]Return to uipc_mbuf.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / kern

Annotation of src/sys/kern/uipc_mbuf.c, Revision 1.126

1.126   ! thorpej     1: /*     $NetBSD: uipc_mbuf.c,v 1.125 2008/03/24 12:24:37 yamt Exp $     */
1.42      thorpej     2:
                      3: /*-
1.53      thorpej     4:  * Copyright (c) 1999, 2001 The NetBSD Foundation, Inc.
1.42      thorpej     5:  * All rights reserved.
                      6:  *
                      7:  * This code is derived from software contributed to The NetBSD Foundation
                      8:  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
                      9:  * NASA Ames Research Center.
                     10:  *
                     11:  * Redistribution and use in source and binary forms, with or without
                     12:  * modification, are permitted provided that the following conditions
                     13:  * are met:
                     14:  * 1. Redistributions of source code must retain the above copyright
                     15:  *    notice, this list of conditions and the following disclaimer.
                     16:  * 2. Redistributions in binary form must reproduce the above copyright
                     17:  *    notice, this list of conditions and the following disclaimer in the
                     18:  *    documentation and/or other materials provided with the distribution.
                     19:  * 3. All advertising materials mentioning features or use of this software
                     20:  *    must display the following acknowledgement:
                     21:  *     This product includes software developed by the NetBSD
                     22:  *     Foundation, Inc. and its contributors.
                     23:  * 4. Neither the name of The NetBSD Foundation nor the names of its
                     24:  *    contributors may be used to endorse or promote products derived
                     25:  *    from this software without specific prior written permission.
                     26:  *
                     27:  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     28:  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     29:  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     30:  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     31:  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     32:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     33:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     34:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     35:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     36:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     37:  * POSSIBILITY OF SUCH DAMAGE.
                     38:  */
1.10      cgd        39:
1.1       cgd        40: /*
1.9       mycroft    41:  * Copyright (c) 1982, 1986, 1988, 1991, 1993
                     42:  *     The Regents of the University of California.  All rights reserved.
1.1       cgd        43:  *
                     44:  * Redistribution and use in source and binary forms, with or without
                     45:  * modification, are permitted provided that the following conditions
                     46:  * are met:
                     47:  * 1. Redistributions of source code must retain the above copyright
                     48:  *    notice, this list of conditions and the following disclaimer.
                     49:  * 2. Redistributions in binary form must reproduce the above copyright
                     50:  *    notice, this list of conditions and the following disclaimer in the
                     51:  *    documentation and/or other materials provided with the distribution.
1.70      agc        52:  * 3. Neither the name of the University nor the names of its contributors
1.1       cgd        53:  *    may be used to endorse or promote products derived from this software
                     54:  *    without specific prior written permission.
                     55:  *
                     56:  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
                     57:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
                     58:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
                     59:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
                     60:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
                     61:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
                     62:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     63:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
                     64:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
                     65:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
                     66:  * SUCH DAMAGE.
                     67:  *
1.26      fvdl       68:  *     @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
1.1       cgd        69:  */
1.56      lukem      70:
                     71: #include <sys/cdefs.h>
1.126   ! thorpej    72: __KERNEL_RCSID(0, "$NetBSD: uipc_mbuf.c,v 1.125 2008/03/24 12:24:37 yamt Exp $");
1.69      martin     73:
                     74: #include "opt_mbuftrace.h"
1.105     yamt       75: #include "opt_ddb.h"
1.24      mrg        76:
1.6       mycroft    77: #include <sys/param.h>
                     78: #include <sys/systm.h>
1.125     yamt       79: #include <sys/atomic.h>
1.124     yamt       80: #include <sys/cpu.h>
1.6       mycroft    81: #include <sys/proc.h>
                     82: #include <sys/malloc.h>
1.1       cgd        83: #define MBTYPES
1.6       mycroft    84: #include <sys/mbuf.h>
                     85: #include <sys/kernel.h>
                     86: #include <sys/syslog.h>
                     87: #include <sys/domain.h>
                     88: #include <sys/protosw.h>
1.124     yamt       89: #include <sys/percpu.h>
1.28      thorpej    90: #include <sys/pool.h>
1.27      matt       91: #include <sys/socket.h>
1.55      simonb     92: #include <sys/sysctl.h>
                     93:
1.27      matt       94: #include <net/if.h>
1.14      christos   95:
1.65      thorpej    96: #include <uvm/uvm.h>
1.23      mrg        97:
1.122     ad         98: pool_cache_t mb_cache; /* mbuf cache */
                     99: pool_cache_t mcl_cache;        /* mbuf cluster cache */
1.53      thorpej   100:
1.18      thorpej   101: struct mbstat mbstat;
                    102: int    max_linkhdr;
                    103: int    max_protohdr;
                    104: int    max_hdr;
                    105: int    max_datalen;
                    106:
1.65      thorpej   107: static int mb_ctor(void *, void *, int);
                    108:
1.103     thorpej   109: static void    *mclpool_alloc(struct pool *, int);
                    110: static void    mclpool_release(struct pool *, void *);
1.58      thorpej   111:
1.103     thorpej   112: static struct pool_allocator mclpool_allocator = {
1.113     christos  113:        .pa_alloc = mclpool_alloc,
                    114:        .pa_free = mclpool_release,
1.58      thorpej   115: };
                    116:
1.79      junyoung  117: static struct mbuf *m_copym0(struct mbuf *, int, int, int, int);
1.85      yamt      118: static struct mbuf *m_split0(struct mbuf *, int, int, int);
1.86      yamt      119: static int m_copyback0(struct mbuf **, int, int, const void *, int, int);
1.85      yamt      120:
                    121: /* flags for m_copyback0 */
                    122: #define        M_COPYBACK0_COPYBACK    0x0001  /* copyback from cp */
                    123: #define        M_COPYBACK0_PRESERVE    0x0002  /* preserve original data */
                    124: #define        M_COPYBACK0_COW         0x0004  /* do copy-on-write */
                    125: #define        M_COPYBACK0_EXTEND      0x0008  /* extend chain */
1.28      thorpej   126:
1.103     thorpej   127: static const char mclpool_warnmsg[] =
1.42      thorpej   128:     "WARNING: mclpool limit reached; increase NMBCLUSTERS";
1.63      thorpej   129:
                    130: MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf");
1.42      thorpej   131:
1.124     yamt      132: static percpu_t *mbstat_percpu;
                    133:
1.64      matt      134: #ifdef MBUFTRACE
                    135: struct mownerhead mowners = LIST_HEAD_INITIALIZER(mowners);
                    136: struct mowner unknown_mowners[] = {
1.114     dogcow    137:        MOWNER_INIT("unknown", "free"),
                    138:        MOWNER_INIT("unknown", "data"),
                    139:        MOWNER_INIT("unknown", "header"),
                    140:        MOWNER_INIT("unknown", "soname"),
                    141:        MOWNER_INIT("unknown", "soopts"),
                    142:        MOWNER_INIT("unknown", "ftable"),
                    143:        MOWNER_INIT("unknown", "control"),
                    144:        MOWNER_INIT("unknown", "oobdata"),
1.64      matt      145: };
1.114     dogcow    146: struct mowner revoked_mowner = MOWNER_INIT("revoked", "");
1.64      matt      147: #endif
                    148:
1.125     yamt      149: #define        MEXT_ISEMBEDDED(m) ((m)->m_ext_ref == (m))
                    150:
                    151: #define        MCLADDREFERENCE(o, n)                                           \
                    152: do {                                                                   \
                    153:        KASSERT(((o)->m_flags & M_EXT) != 0);                           \
                    154:        KASSERT(((n)->m_flags & M_EXT) == 0);                           \
                    155:        KASSERT((o)->m_ext.ext_refcnt >= 1);                            \
                    156:        (n)->m_flags |= ((o)->m_flags & M_EXTCOPYFLAGS);                \
                    157:        atomic_inc_uint(&(o)->m_ext.ext_refcnt);                        \
                    158:        (n)->m_ext_ref = (o)->m_ext_ref;                                \
                    159:        mowner_ref((n), (n)->m_flags);                                  \
                    160:        MCLREFDEBUGN((n), __FILE__, __LINE__);                          \
                    161: } while (/* CONSTCOND */ 0)
                    162:
1.28      thorpej   163: /*
1.68      simonb    164:  * Initialize the mbuf allocator.
1.28      thorpej   165:  */
1.4       jtc       166: void
1.62      thorpej   167: mbinit(void)
1.1       cgd       168: {
1.65      thorpej   169:
                    170:        KASSERT(sizeof(struct _m_ext) <= MHLEN);
1.67      simonb    171:        KASSERT(sizeof(struct mbuf) == MSIZE);
1.65      thorpej   172:
1.111     yamt      173:        mclpool_allocator.pa_backingmap = mb_map;
1.53      thorpej   174:
1.122     ad        175:        mb_cache = pool_cache_init(msize, 0, 0, 0, "mbpl",
                    176:            NULL, IPL_VM, mb_ctor, NULL, NULL);
                    177:        KASSERT(mb_cache != NULL);
                    178:
                    179:        mcl_cache = pool_cache_init(mclbytes, 0, 0, 0, "mclpl",
                    180:            &mclpool_allocator, IPL_VM, NULL, NULL, NULL);
                    181:        KASSERT(mcl_cache != NULL);
1.59      thorpej   182:
1.122     ad        183:        pool_cache_set_drain_hook(mb_cache, m_reclaim, NULL);
                    184:        pool_cache_set_drain_hook(mcl_cache, m_reclaim, NULL);
1.37      thorpej   185:
                    186:        /*
1.39      thorpej   187:         * Set the hard limit on the mclpool to the number of
                    188:         * mbuf clusters the kernel is to support.  Log the limit
                    189:         * reached message max once a minute.
                    190:         */
1.122     ad        191:        pool_cache_sethardlimit(mcl_cache, nmbclusters, mclpool_warnmsg, 60);
1.42      thorpej   192:
1.124     yamt      193:        mbstat_percpu = percpu_alloc(sizeof(struct mbstat_cpu));
                    194:
1.39      thorpej   195:        /*
1.42      thorpej   196:         * Set a low water mark for both mbufs and clusters.  This should
                    197:         * help ensure that they can be allocated in a memory starvation
                    198:         * situation.  This is important for e.g. diskless systems which
                    199:         * must allocate mbufs in order for the pagedaemon to clean pages.
1.37      thorpej   200:         */
1.122     ad        201:        pool_cache_setlowat(mb_cache, mblowat);
                    202:        pool_cache_setlowat(mcl_cache, mcllowat);
1.64      matt      203:
                    204: #ifdef MBUFTRACE
                    205:        {
                    206:                /*
                    207:                 * Attach the unknown mowners.
                    208:                 */
                    209:                int i;
                    210:                MOWNER_ATTACH(&revoked_mowner);
                    211:                for (i = sizeof(unknown_mowners)/sizeof(unknown_mowners[0]);
                    212:                     i-- > 0; )
                    213:                        MOWNER_ATTACH(&unknown_mowners[i]);
                    214:        }
                    215: #endif
1.42      thorpej   216: }
                    217:
1.75      atatat    218: /*
                    219:  * sysctl helper routine for the kern.mbuf subtree.  nmbclusters may
                    220:  * or may not be writable, and mblowat and mcllowat need range
                    221:  * checking and pool tweaking after being reset.
                    222:  */
                    223: static int
                    224: sysctl_kern_mbuf(SYSCTLFN_ARGS)
1.42      thorpej   225: {
                    226:        int error, newval;
1.75      atatat    227:        struct sysctlnode node;
1.42      thorpej   228:
1.75      atatat    229:        node = *rnode;
                    230:        node.sysctl_data = &newval;
                    231:        switch (rnode->sysctl_num) {
1.42      thorpej   232:        case MBUF_NMBCLUSTERS:
1.76      atatat    233:                if (mb_map != NULL) {
1.80      atatat    234:                        node.sysctl_flags &= ~CTLFLAG_READWRITE;
                    235:                        node.sysctl_flags |= CTLFLAG_READONLY;
1.75      atatat    236:                }
                    237:                /* FALLTHROUGH */
1.42      thorpej   238:        case MBUF_MBLOWAT:
                    239:        case MBUF_MCLLOWAT:
1.75      atatat    240:                newval = *(int*)rnode->sysctl_data;
                    241:                break;
                    242:        default:
                    243:                return (EOPNOTSUPP);
                    244:        }
                    245:
                    246:        error = sysctl_lookup(SYSCTLFN_CALL(&node));
                    247:        if (error || newp == NULL)
1.42      thorpej   248:                return (error);
1.75      atatat    249:        if (newval < 0)
                    250:                return (EINVAL);
                    251:
                    252:        switch (node.sysctl_num) {
                    253:        case MBUF_NMBCLUSTERS:
                    254:                if (newval < nmbclusters)
                    255:                        return (EINVAL);
                    256:                nmbclusters = newval;
1.122     ad        257:                pool_cache_sethardlimit(mcl_cache, nmbclusters,
                    258:                    mclpool_warnmsg, 60);
1.75      atatat    259:                break;
                    260:        case MBUF_MBLOWAT:
                    261:                mblowat = newval;
1.122     ad        262:                pool_cache_setlowat(mb_cache, mblowat);
1.75      atatat    263:                break;
                    264:        case MBUF_MCLLOWAT:
1.76      atatat    265:                mcllowat = newval;
1.122     ad        266:                pool_cache_setlowat(mcl_cache, mcllowat);
1.75      atatat    267:                break;
                    268:        }
                    269:
                    270:        return (0);
                    271: }
                    272:
1.64      matt      273: #ifdef MBUFTRACE
1.124     yamt      274: static void
                    275: mowner_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci)
                    276: {
                    277:        struct mowner_counter *mc = v1;
                    278:        struct mowner_user *mo_user = v2;
                    279:        int i;
                    280:
                    281:        for (i = 0; i < MOWNER_COUNTER_NCOUNTERS; i++) {
                    282:                mo_user->mo_counter[i] += mc->mc_counter[i];
                    283:        }
                    284: }
                    285:
                    286: static void
                    287: mowner_convert_to_user(struct mowner *mo, struct mowner_user *mo_user)
                    288: {
                    289:
                    290:        memset(mo_user, 0, sizeof(*mo_user));
                    291:        KASSERT(sizeof(mo_user->mo_name) == sizeof(mo->mo_name));
                    292:        KASSERT(sizeof(mo_user->mo_descr) == sizeof(mo->mo_descr));
                    293:        memcpy(mo_user->mo_name, mo->mo_name, sizeof(mo->mo_name));
                    294:        memcpy(mo_user->mo_descr, mo->mo_descr, sizeof(mo->mo_descr));
                    295:        percpu_foreach(mo->mo_counters, mowner_conver_to_user_cb, mo_user);
                    296: }
                    297:
1.75      atatat    298: static int
                    299: sysctl_kern_mbuf_mowners(SYSCTLFN_ARGS)
                    300: {
                    301:        struct mowner *mo;
                    302:        size_t len = 0;
                    303:        int error = 0;
                    304:
                    305:        if (namelen != 0)
                    306:                return (EINVAL);
                    307:        if (newp != NULL)
                    308:                return (EPERM);
                    309:
                    310:        LIST_FOREACH(mo, &mowners, mo_link) {
1.124     yamt      311:                struct mowner_user mo_user;
                    312:
                    313:                mowner_convert_to_user(mo, &mo_user);
                    314:
1.75      atatat    315:                if (oldp != NULL) {
1.124     yamt      316:                        if (*oldlenp - len < sizeof(mo_user)) {
1.75      atatat    317:                                error = ENOMEM;
                    318:                                break;
                    319:                        }
1.124     yamt      320:                        error = copyout(&mo_user, (char *)oldp + len,
                    321:                            sizeof(mo_user));
1.75      atatat    322:                        if (error)
                    323:                                break;
1.64      matt      324:                }
1.124     yamt      325:                len += sizeof(mo_user);
1.75      atatat    326:        }
                    327:
                    328:        if (error == 0)
1.64      matt      329:                *oldlenp = len;
1.75      atatat    330:
                    331:        return (error);
                    332: }
                    333: #endif /* MBUFTRACE */
                    334:
1.124     yamt      335: static void
                    336: mbstat_conver_to_user_cb(void *v1, void *v2, struct cpu_info *ci)
                    337: {
                    338:        struct mbstat_cpu *mbsc = v1;
                    339:        struct mbstat *mbs = v2;
                    340:        int i;
                    341:
                    342:        for (i = 0; i < __arraycount(mbs->m_mtypes); i++) {
                    343:                mbs->m_mtypes[i] += mbsc->m_mtypes[i];
                    344:        }
                    345: }
                    346:
                    347: static void
                    348: mbstat_convert_to_user(struct mbstat *mbs)
                    349: {
                    350:
                    351:        memset(mbs, 0, sizeof(*mbs));
                    352:        mbs->m_drain = mbstat.m_drain;
                    353:        percpu_foreach(mbstat_percpu, mbstat_conver_to_user_cb, mbs);
                    354: }
                    355:
                    356: static int
                    357: sysctl_kern_mbuf_stats(SYSCTLFN_ARGS)
                    358: {
                    359:        struct sysctlnode node;
                    360:        struct mbstat mbs;
                    361:
                    362:        mbstat_convert_to_user(&mbs);
                    363:        node = *rnode;
                    364:        node.sysctl_data = &mbs;
                    365:        node.sysctl_size = sizeof(mbs);
                    366:        return sysctl_lookup(SYSCTLFN_CALL(&node));
                    367: }
                    368:
1.75      atatat    369: SYSCTL_SETUP(sysctl_kern_mbuf_setup, "sysctl kern.mbuf subtree setup")
                    370: {
                    371:
1.80      atatat    372:        sysctl_createv(clog, 0, NULL, NULL,
                    373:                       CTLFLAG_PERMANENT,
1.75      atatat    374:                       CTLTYPE_NODE, "kern", NULL,
                    375:                       NULL, 0, NULL, 0,
                    376:                       CTL_KERN, CTL_EOL);
1.80      atatat    377:        sysctl_createv(clog, 0, NULL, NULL,
                    378:                       CTLFLAG_PERMANENT,
1.82      atatat    379:                       CTLTYPE_NODE, "mbuf",
                    380:                       SYSCTL_DESCR("mbuf control variables"),
1.75      atatat    381:                       NULL, 0, NULL, 0,
                    382:                       CTL_KERN, KERN_MBUF, CTL_EOL);
                    383:
1.80      atatat    384:        sysctl_createv(clog, 0, NULL, NULL,
                    385:                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
1.82      atatat    386:                       CTLTYPE_INT, "msize",
                    387:                       SYSCTL_DESCR("mbuf base size"),
1.75      atatat    388:                       NULL, msize, NULL, 0,
                    389:                       CTL_KERN, KERN_MBUF, MBUF_MSIZE, CTL_EOL);
1.80      atatat    390:        sysctl_createv(clog, 0, NULL, NULL,
                    391:                       CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
1.82      atatat    392:                       CTLTYPE_INT, "mclbytes",
                    393:                       SYSCTL_DESCR("mbuf cluster size"),
1.75      atatat    394:                       NULL, mclbytes, NULL, 0,
                    395:                       CTL_KERN, KERN_MBUF, MBUF_MCLBYTES, CTL_EOL);
1.80      atatat    396:        sysctl_createv(clog, 0, NULL, NULL,
                    397:                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1.82      atatat    398:                       CTLTYPE_INT, "nmbclusters",
                    399:                       SYSCTL_DESCR("Limit on the number of mbuf clusters"),
1.75      atatat    400:                       sysctl_kern_mbuf, 0, &nmbclusters, 0,
                    401:                       CTL_KERN, KERN_MBUF, MBUF_NMBCLUSTERS, CTL_EOL);
1.80      atatat    402:        sysctl_createv(clog, 0, NULL, NULL,
                    403:                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1.82      atatat    404:                       CTLTYPE_INT, "mblowat",
                    405:                       SYSCTL_DESCR("mbuf low water mark"),
1.75      atatat    406:                       sysctl_kern_mbuf, 0, &mblowat, 0,
                    407:                       CTL_KERN, KERN_MBUF, MBUF_MBLOWAT, CTL_EOL);
1.80      atatat    408:        sysctl_createv(clog, 0, NULL, NULL,
                    409:                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1.82      atatat    410:                       CTLTYPE_INT, "mcllowat",
                    411:                       SYSCTL_DESCR("mbuf cluster low water mark"),
1.75      atatat    412:                       sysctl_kern_mbuf, 0, &mcllowat, 0,
                    413:                       CTL_KERN, KERN_MBUF, MBUF_MCLLOWAT, CTL_EOL);
1.80      atatat    414:        sysctl_createv(clog, 0, NULL, NULL,
                    415:                       CTLFLAG_PERMANENT,
1.82      atatat    416:                       CTLTYPE_STRUCT, "stats",
                    417:                       SYSCTL_DESCR("mbuf allocation statistics"),
1.124     yamt      418:                       sysctl_kern_mbuf_stats, 0, NULL, 0,
1.75      atatat    419:                       CTL_KERN, KERN_MBUF, MBUF_STATS, CTL_EOL);
                    420: #ifdef MBUFTRACE
1.80      atatat    421:        sysctl_createv(clog, 0, NULL, NULL,
                    422:                       CTLFLAG_PERMANENT,
1.82      atatat    423:                       CTLTYPE_STRUCT, "mowners",
                    424:                       SYSCTL_DESCR("Information about mbuf owners"),
1.75      atatat    425:                       sysctl_kern_mbuf_mowners, 0, NULL, 0,
                    426:                       CTL_KERN, KERN_MBUF, MBUF_MOWNERS, CTL_EOL);
                    427: #endif /* MBUFTRACE */
1.28      thorpej   428: }
                    429:
1.103     thorpej   430: static void *
1.116     yamt      431: mclpool_alloc(struct pool *pp, int flags)
1.28      thorpej   432: {
1.118     thorpej   433:        bool waitok = (flags & PR_WAITOK) ? true : false;
1.28      thorpej   434:
1.93      yamt      435:        return ((void *)uvm_km_alloc_poolpage(mb_map, waitok));
1.1       cgd       436: }
                    437:
1.103     thorpej   438: static void
1.116     yamt      439: mclpool_release(struct pool *pp, void *v)
1.1       cgd       440: {
                    441:
1.93      yamt      442:        uvm_km_free_poolpage(mb_map, (vaddr_t)v);
1.65      thorpej   443: }
                    444:
                    445: /*ARGSUSED*/
                    446: static int
1.116     yamt      447: mb_ctor(void *arg, void *object, int flags)
1.65      thorpej   448: {
                    449:        struct mbuf *m = object;
                    450:
                    451: #ifdef POOL_VTOPHYS
                    452:        m->m_paddr = POOL_VTOPHYS(m);
                    453: #else
                    454:        m->m_paddr = M_PADDR_INVALID;
                    455: #endif
                    456:        return (0);
1.1       cgd       457: }
                    458:
1.14      christos  459: void
1.116     yamt      460: m_reclaim(void *arg, int flags)
1.1       cgd       461: {
1.27      matt      462:        struct domain *dp;
1.81      matt      463:        const struct protosw *pr;
1.27      matt      464:        struct ifnet *ifp;
1.122     ad        465:        int s;
1.1       cgd       466:
1.122     ad        467:        KERNEL_LOCK(1, NULL);
                    468:        s = splvm();
1.91      matt      469:        DOMAIN_FOREACH(dp) {
1.33      thorpej   470:                for (pr = dp->dom_protosw;
                    471:                     pr < dp->dom_protoswNPROTOSW; pr++)
                    472:                        if (pr->pr_drain)
                    473:                                (*pr->pr_drain)();
1.91      matt      474:        }
1.92      matt      475:        IFNET_FOREACH(ifp) {
1.27      matt      476:                if (ifp->if_drain)
                    477:                        (*ifp->if_drain)(ifp);
1.92      matt      478:        }
1.1       cgd       479:        splx(s);
                    480:        mbstat.m_drain++;
1.122     ad        481:        KERNEL_UNLOCK_ONE(NULL);
1.1       cgd       482: }
                    483:
                    484: /*
                    485:  * Space allocation routines.
                    486:  * These are also available as macros
                    487:  * for critical paths.
                    488:  */
                    489: struct mbuf *
1.62      thorpej   490: m_get(int nowait, int type)
1.1       cgd       491: {
1.27      matt      492:        struct mbuf *m;
1.1       cgd       493:
1.124     yamt      494:        m = pool_cache_get(mb_cache,
                    495:            nowait == M_WAIT ? PR_WAITOK|PR_LIMITFAIL : 0);
                    496:        if (m == NULL)
                    497:                return NULL;
                    498:
                    499:        mbstat_type_add(type, 1);
                    500:        mowner_init(m, type);
1.125     yamt      501:        m->m_ext_ref = m;
1.124     yamt      502:        m->m_type = type;
                    503:        m->m_next = NULL;
                    504:        m->m_nextpkt = NULL;
                    505:        m->m_data = m->m_dat;
                    506:        m->m_flags = 0;
                    507:
                    508:        return m;
1.1       cgd       509: }
                    510:
                    511: struct mbuf *
1.62      thorpej   512: m_gethdr(int nowait, int type)
1.1       cgd       513: {
1.27      matt      514:        struct mbuf *m;
1.1       cgd       515:
1.124     yamt      516:        m = m_get(nowait, type);
                    517:        if (m == NULL)
                    518:                return NULL;
                    519:
                    520:        m->m_data = m->m_pktdat;
                    521:        m->m_flags = M_PKTHDR;
                    522:        m->m_pkthdr.rcvif = NULL;
                    523:        m->m_pkthdr.csum_flags = 0;
                    524:        m->m_pkthdr.csum_data = 0;
                    525:        SLIST_INIT(&m->m_pkthdr.tags);
                    526:
                    527:        return m;
1.1       cgd       528: }
                    529:
                    530: struct mbuf *
1.62      thorpej   531: m_getclr(int nowait, int type)
1.1       cgd       532: {
1.27      matt      533:        struct mbuf *m;
1.1       cgd       534:
1.5       cgd       535:        MGET(m, nowait, type);
1.1       cgd       536:        if (m == 0)
1.71      simonb    537:                return (NULL);
1.119     christos  538:        memset(mtod(m, void *), 0, MLEN);
1.1       cgd       539:        return (m);
                    540: }
                    541:
1.64      matt      542: void
                    543: m_clget(struct mbuf *m, int nowait)
                    544: {
1.71      simonb    545:
1.64      matt      546:        MCLGET(m, nowait);
                    547: }
                    548:
1.1       cgd       549: struct mbuf *
1.62      thorpej   550: m_free(struct mbuf *m)
1.1       cgd       551: {
1.27      matt      552:        struct mbuf *n;
1.1       cgd       553:
                    554:        MFREE(m, n);
                    555:        return (n);
                    556: }
                    557:
1.9       mycroft   558: void
1.62      thorpej   559: m_freem(struct mbuf *m)
1.1       cgd       560: {
1.27      matt      561:        struct mbuf *n;
1.1       cgd       562:
                    563:        if (m == NULL)
                    564:                return;
                    565:        do {
                    566:                MFREE(m, n);
1.18      thorpej   567:                m = n;
                    568:        } while (m);
1.1       cgd       569: }
                    570:
1.64      matt      571: #ifdef MBUFTRACE
1.83      jonathan  572: /*
                    573:  * Walk a chain of mbufs, claiming ownership of each mbuf in the chain.
                    574:  */
1.64      matt      575: void
1.83      jonathan  576: m_claimm(struct mbuf *m, struct mowner *mo)
1.64      matt      577: {
1.71      simonb    578:
1.64      matt      579:        for (; m != NULL; m = m->m_next)
                    580:                MCLAIM(m, mo);
                    581: }
                    582: #endif
                    583:
1.1       cgd       584: /*
                    585:  * Mbuffer utility routines.
                    586:  */
                    587:
                    588: /*
                    589:  * Lesser-used path for M_PREPEND:
                    590:  * allocate new mbuf to prepend to chain,
                    591:  * copy junk along.
                    592:  */
                    593: struct mbuf *
1.62      thorpej   594: m_prepend(struct mbuf *m, int len, int how)
1.1       cgd       595: {
                    596:        struct mbuf *mn;
                    597:
1.9       mycroft   598:        MGET(mn, how, m->m_type);
1.1       cgd       599:        if (mn == (struct mbuf *)NULL) {
                    600:                m_freem(m);
                    601:                return ((struct mbuf *)NULL);
                    602:        }
                    603:        if (m->m_flags & M_PKTHDR) {
1.101     yamt      604:                M_MOVE_PKTHDR(mn, m);
1.64      matt      605:        } else {
                    606:                MCLAIM(mn, m->m_owner);
1.1       cgd       607:        }
                    608:        mn->m_next = m;
                    609:        m = mn;
                    610:        if (len < MHLEN)
                    611:                MH_ALIGN(m, len);
                    612:        m->m_len = len;
                    613:        return (m);
                    614: }
                    615:
                    616: /*
                    617:  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
                    618:  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
                    619:  * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
                    620:  */
                    621: int MCFail;
                    622:
                    623: struct mbuf *
1.62      thorpej   624: m_copym(struct mbuf *m, int off0, int len, int wait)
1.1       cgd       625: {
1.71      simonb    626:
1.44      itojun    627:        return m_copym0(m, off0, len, wait, 0); /* shallow copy on M_EXT */
                    628: }
                    629:
                    630: struct mbuf *
1.62      thorpej   631: m_dup(struct mbuf *m, int off0, int len, int wait)
1.44      itojun    632: {
1.71      simonb    633:
1.44      itojun    634:        return m_copym0(m, off0, len, wait, 1); /* deep copy */
                    635: }
                    636:
                    637: static struct mbuf *
1.62      thorpej   638: m_copym0(struct mbuf *m, int off0, int len, int wait, int deep)
1.44      itojun    639: {
1.27      matt      640:        struct mbuf *n, **np;
                    641:        int off = off0;
1.1       cgd       642:        struct mbuf *top;
                    643:        int copyhdr = 0;
                    644:
                    645:        if (off < 0 || len < 0)
1.43      thorpej   646:                panic("m_copym: off %d, len %d", off, len);
1.1       cgd       647:        if (off == 0 && m->m_flags & M_PKTHDR)
                    648:                copyhdr = 1;
                    649:        while (off > 0) {
                    650:                if (m == 0)
1.90      matt      651:                        panic("m_copym: m == 0, off %d", off);
1.1       cgd       652:                if (off < m->m_len)
                    653:                        break;
                    654:                off -= m->m_len;
                    655:                m = m->m_next;
                    656:        }
                    657:        np = &top;
                    658:        top = 0;
                    659:        while (len > 0) {
                    660:                if (m == 0) {
                    661:                        if (len != M_COPYALL)
1.90      matt      662:                                panic("m_copym: m == 0, len %d [!COPYALL]",
                    663:                                    len);
1.1       cgd       664:                        break;
                    665:                }
                    666:                MGET(n, wait, m->m_type);
                    667:                *np = n;
                    668:                if (n == 0)
                    669:                        goto nospace;
1.64      matt      670:                MCLAIM(n, m->m_owner);
1.1       cgd       671:                if (copyhdr) {
                    672:                        M_COPY_PKTHDR(n, m);
                    673:                        if (len == M_COPYALL)
                    674:                                n->m_pkthdr.len -= off0;
                    675:                        else
                    676:                                n->m_pkthdr.len = len;
                    677:                        copyhdr = 0;
                    678:                }
1.9       mycroft   679:                n->m_len = min(len, m->m_len - off);
1.1       cgd       680:                if (m->m_flags & M_EXT) {
1.44      itojun    681:                        if (!deep) {
                    682:                                n->m_data = m->m_data + off;
                    683:                                MCLADDREFERENCE(m, n);
                    684:                        } else {
1.48      itojun    685:                                /*
1.50      itojun    686:                                 * we are unsure about the way m was allocated.
                    687:                                 * copy into multiple MCLBYTES cluster mbufs.
1.48      itojun    688:                                 */
1.44      itojun    689:                                MCLGET(n, wait);
1.50      itojun    690:                                n->m_len = 0;
                    691:                                n->m_len = M_TRAILINGSPACE(n);
                    692:                                n->m_len = min(n->m_len, len);
                    693:                                n->m_len = min(n->m_len, m->m_len - off);
1.119     christos  694:                                memcpy(mtod(n, void *), mtod(m, char *) + off,
1.44      itojun    695:                                    (unsigned)n->m_len);
                    696:                        }
1.1       cgd       697:                } else
1.119     christos  698:                        memcpy(mtod(n, void *), mtod(m, char *) + off,
1.1       cgd       699:                            (unsigned)n->m_len);
                    700:                if (len != M_COPYALL)
                    701:                        len -= n->m_len;
1.50      itojun    702:                off += n->m_len;
                    703: #ifdef DIAGNOSTIC
                    704:                if (off > m->m_len)
                    705:                        panic("m_copym0 overrun");
                    706: #endif
                    707:                if (off == m->m_len) {
                    708:                        m = m->m_next;
                    709:                        off = 0;
                    710:                }
1.1       cgd       711:                np = &n->m_next;
                    712:        }
                    713:        if (top == 0)
                    714:                MCFail++;
                    715:        return (top);
                    716: nospace:
                    717:        m_freem(top);
                    718:        MCFail++;
1.71      simonb    719:        return (NULL);
1.1       cgd       720: }
                    721:
                    722: /*
1.18      thorpej   723:  * Copy an entire packet, including header (which must be present).
                    724:  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
                    725:  */
                    726: struct mbuf *
1.62      thorpej   727: m_copypacket(struct mbuf *m, int how)
1.18      thorpej   728: {
                    729:        struct mbuf *top, *n, *o;
                    730:
                    731:        MGET(n, how, m->m_type);
                    732:        top = n;
                    733:        if (!n)
                    734:                goto nospace;
                    735:
1.64      matt      736:        MCLAIM(n, m->m_owner);
1.18      thorpej   737:        M_COPY_PKTHDR(n, m);
                    738:        n->m_len = m->m_len;
                    739:        if (m->m_flags & M_EXT) {
                    740:                n->m_data = m->m_data;
                    741:                MCLADDREFERENCE(m, n);
                    742:        } else {
1.30      perry     743:                memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
1.18      thorpej   744:        }
                    745:
                    746:        m = m->m_next;
                    747:        while (m) {
                    748:                MGET(o, how, m->m_type);
                    749:                if (!o)
                    750:                        goto nospace;
                    751:
1.64      matt      752:                MCLAIM(o, m->m_owner);
1.18      thorpej   753:                n->m_next = o;
                    754:                n = n->m_next;
                    755:
                    756:                n->m_len = m->m_len;
                    757:                if (m->m_flags & M_EXT) {
                    758:                        n->m_data = m->m_data;
                    759:                        MCLADDREFERENCE(m, n);
                    760:                } else {
1.30      perry     761:                        memcpy(mtod(n, char *), mtod(m, char *), n->m_len);
1.18      thorpej   762:                }
                    763:
                    764:                m = m->m_next;
                    765:        }
                    766:        return top;
                    767: nospace:
                    768:        m_freem(top);
                    769:        MCFail++;
1.71      simonb    770:        return NULL;
1.18      thorpej   771: }
                    772:
                    773: /*
1.1       cgd       774:  * Copy data from an mbuf chain starting "off" bytes from the beginning,
                    775:  * continuing for "len" bytes, into the indicated buffer.
                    776:  */
1.14      christos  777: void
1.100     martin    778: m_copydata(struct mbuf *m, int off, int len, void *vp)
1.1       cgd       779: {
1.94      tron      780:        unsigned        count;
1.119     christos  781:        void *          cp = vp;
1.1       cgd       782:
                    783:        if (off < 0 || len < 0)
1.90      matt      784:                panic("m_copydata: off %d, len %d", off, len);
1.1       cgd       785:        while (off > 0) {
1.94      tron      786:                if (m == NULL)
                    787:                        panic("m_copydata: m == NULL, off %d", off);
1.1       cgd       788:                if (off < m->m_len)
                    789:                        break;
                    790:                off -= m->m_len;
                    791:                m = m->m_next;
                    792:        }
                    793:        while (len > 0) {
1.94      tron      794:                if (m == NULL)
                    795:                        panic("m_copydata: m == NULL, len %d", len);
1.9       mycroft   796:                count = min(m->m_len - off, len);
1.119     christos  797:                memcpy(cp, mtod(m, char *) + off, count);
1.1       cgd       798:                len -= count;
1.119     christos  799:                cp = (char *)cp + count;
1.1       cgd       800:                off = 0;
                    801:                m = m->m_next;
                    802:        }
                    803: }
                    804:
                    805: /*
                    806:  * Concatenate mbuf chain n to m.
1.72      itojun    807:  * n might be copied into m (when n->m_len is small), therefore data portion of
                    808:  * n could be copied into an mbuf of different mbuf type.
1.1       cgd       809:  * Any m_pkthdr is not updated.
                    810:  */
1.14      christos  811: void
1.62      thorpej   812: m_cat(struct mbuf *m, struct mbuf *n)
1.1       cgd       813: {
1.73      yamt      814:
1.1       cgd       815:        while (m->m_next)
                    816:                m = m->m_next;
                    817:        while (n) {
1.77      itojun    818:                if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) {
1.1       cgd       819:                        /* just join the two chains */
                    820:                        m->m_next = n;
                    821:                        return;
                    822:                }
                    823:                /* splat the data from one into the other */
1.119     christos  824:                memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
1.1       cgd       825:                    (u_int)n->m_len);
                    826:                m->m_len += n->m_len;
                    827:                n = m_free(n);
                    828:        }
                    829: }
                    830:
1.11      mycroft   831: void
1.62      thorpej   832: m_adj(struct mbuf *mp, int req_len)
1.1       cgd       833: {
1.27      matt      834:        int len = req_len;
                    835:        struct mbuf *m;
                    836:        int count;
1.1       cgd       837:
                    838:        if ((m = mp) == NULL)
                    839:                return;
                    840:        if (len >= 0) {
                    841:                /*
                    842:                 * Trim from head.
                    843:                 */
                    844:                while (m != NULL && len > 0) {
                    845:                        if (m->m_len <= len) {
                    846:                                len -= m->m_len;
                    847:                                m->m_len = 0;
                    848:                                m = m->m_next;
                    849:                        } else {
                    850:                                m->m_len -= len;
                    851:                                m->m_data += len;
                    852:                                len = 0;
                    853:                        }
                    854:                }
                    855:                m = mp;
                    856:                if (mp->m_flags & M_PKTHDR)
                    857:                        m->m_pkthdr.len -= (req_len - len);
                    858:        } else {
                    859:                /*
                    860:                 * Trim from tail.  Scan the mbuf chain,
                    861:                 * calculating its length and finding the last mbuf.
                    862:                 * If the adjustment only affects this mbuf, then just
                    863:                 * adjust and return.  Otherwise, rescan and truncate
                    864:                 * after the remaining size.
                    865:                 */
                    866:                len = -len;
                    867:                count = 0;
                    868:                for (;;) {
                    869:                        count += m->m_len;
                    870:                        if (m->m_next == (struct mbuf *)0)
                    871:                                break;
                    872:                        m = m->m_next;
                    873:                }
                    874:                if (m->m_len >= len) {
                    875:                        m->m_len -= len;
1.8       deraadt   876:                        if (mp->m_flags & M_PKTHDR)
                    877:                                mp->m_pkthdr.len -= len;
1.1       cgd       878:                        return;
                    879:                }
                    880:                count -= len;
                    881:                if (count < 0)
                    882:                        count = 0;
                    883:                /*
                    884:                 * Correct length for chain is "count".
                    885:                 * Find the mbuf with last data, adjust its length,
                    886:                 * and toss data from remaining mbufs on chain.
                    887:                 */
                    888:                m = mp;
                    889:                if (m->m_flags & M_PKTHDR)
                    890:                        m->m_pkthdr.len = count;
                    891:                for (; m; m = m->m_next) {
                    892:                        if (m->m_len >= count) {
                    893:                                m->m_len = count;
                    894:                                break;
                    895:                        }
                    896:                        count -= m->m_len;
                    897:                }
1.110     christos  898:                if (m)
                    899:                        while (m->m_next)
                    900:                                (m = m->m_next)->m_len = 0;
1.1       cgd       901:        }
                    902: }
                    903:
                    904: /*
1.102     skrll     905:  * Rearrange an mbuf chain so that len bytes are contiguous
1.1       cgd       906:  * and in the data area of an mbuf (so that mtod and dtom
                    907:  * will work for a structure of size len).  Returns the resulting
                    908:  * mbuf chain on success, frees it and returns null on failure.
                    909:  * If there is room, it will add up to max_protohdr-len extra bytes to the
                    910:  * contiguous region in an attempt to avoid being called next time.
                    911:  */
                    912: int MPFail;
                    913:
                    914: struct mbuf *
1.62      thorpej   915: m_pullup(struct mbuf *n, int len)
1.1       cgd       916: {
1.27      matt      917:        struct mbuf *m;
                    918:        int count;
1.1       cgd       919:        int space;
                    920:
                    921:        /*
                    922:         * If first mbuf has no cluster, and has room for len bytes
                    923:         * without shifting current data, pullup into it,
                    924:         * otherwise allocate a new mbuf to prepend to the chain.
                    925:         */
                    926:        if ((n->m_flags & M_EXT) == 0 &&
                    927:            n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
                    928:                if (n->m_len >= len)
                    929:                        return (n);
                    930:                m = n;
                    931:                n = n->m_next;
                    932:                len -= m->m_len;
                    933:        } else {
                    934:                if (len > MHLEN)
                    935:                        goto bad;
                    936:                MGET(m, M_DONTWAIT, n->m_type);
                    937:                if (m == 0)
                    938:                        goto bad;
1.64      matt      939:                MCLAIM(m, n->m_owner);
1.1       cgd       940:                m->m_len = 0;
                    941:                if (n->m_flags & M_PKTHDR) {
1.101     yamt      942:                        M_MOVE_PKTHDR(m, n);
1.1       cgd       943:                }
                    944:        }
                    945:        space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
                    946:        do {
                    947:                count = min(min(max(len, max_protohdr), space), n->m_len);
1.119     christos  948:                memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
1.1       cgd       949:                  (unsigned)count);
                    950:                len -= count;
                    951:                m->m_len += count;
                    952:                n->m_len -= count;
                    953:                space -= count;
                    954:                if (n->m_len)
                    955:                        n->m_data += count;
                    956:                else
                    957:                        n = m_free(n);
                    958:        } while (len > 0 && n);
                    959:        if (len > 0) {
                    960:                (void) m_free(m);
                    961:                goto bad;
                    962:        }
                    963:        m->m_next = n;
                    964:        return (m);
                    965: bad:
                    966:        m_freem(n);
                    967:        MPFail++;
1.71      simonb    968:        return (NULL);
1.60      thorpej   969: }
                    970:
                    971: /*
                    972:  * Like m_pullup(), except a new mbuf is always allocated, and we allow
                    973:  * the amount of empty space before the data in the new mbuf to be specified
                    974:  * (in the event that the caller expects to prepend later).
                    975:  */
                    976: int MSFail;
                    977:
                    978: struct mbuf *
                    979: m_copyup(struct mbuf *n, int len, int dstoff)
                    980: {
                    981:        struct mbuf *m;
                    982:        int count, space;
                    983:
                    984:        if (len > (MHLEN - dstoff))
                    985:                goto bad;
                    986:        MGET(m, M_DONTWAIT, n->m_type);
                    987:        if (m == NULL)
                    988:                goto bad;
1.64      matt      989:        MCLAIM(m, n->m_owner);
1.60      thorpej   990:        m->m_len = 0;
                    991:        if (n->m_flags & M_PKTHDR) {
1.101     yamt      992:                M_MOVE_PKTHDR(m, n);
1.60      thorpej   993:        }
                    994:        m->m_data += dstoff;
                    995:        space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
                    996:        do {
                    997:                count = min(min(max(len, max_protohdr), space), n->m_len);
1.119     christos  998:                memcpy(mtod(m, char *) + m->m_len, mtod(n, void *),
1.60      thorpej   999:                    (unsigned)count);
                   1000:                len -= count;
                   1001:                m->m_len += count;
                   1002:                n->m_len -= count;
                   1003:                space -= count;
                   1004:                if (n->m_len)
                   1005:                        n->m_data += count;
                   1006:                else
                   1007:                        n = m_free(n);
                   1008:        } while (len > 0 && n);
                   1009:        if (len > 0) {
                   1010:                (void) m_free(m);
                   1011:                goto bad;
                   1012:        }
                   1013:        m->m_next = n;
                   1014:        return (m);
                   1015:  bad:
                   1016:        m_freem(n);
                   1017:        MSFail++;
                   1018:        return (NULL);
1.9       mycroft  1019: }
                   1020:
                   1021: /*
                   1022:  * Partition an mbuf chain in two pieces, returning the tail --
                   1023:  * all but the first len0 bytes.  In case of failure, it returns NULL and
                   1024:  * attempts to restore the chain to its original state.
                   1025:  */
                   1026: struct mbuf *
1.62      thorpej  1027: m_split(struct mbuf *m0, int len0, int wait)
1.9       mycroft  1028: {
1.85      yamt     1029:
1.87      yamt     1030:        return m_split0(m0, len0, wait, 1);
1.85      yamt     1031: }
                   1032:
                   1033: static struct mbuf *
                   1034: m_split0(struct mbuf *m0, int len0, int wait, int copyhdr)
                   1035: {
1.27      matt     1036:        struct mbuf *m, *n;
1.22      thorpej  1037:        unsigned len = len0, remain, len_save;
1.9       mycroft  1038:
                   1039:        for (m = m0; m && len > m->m_len; m = m->m_next)
                   1040:                len -= m->m_len;
                   1041:        if (m == 0)
1.71      simonb   1042:                return (NULL);
1.9       mycroft  1043:        remain = m->m_len - len;
1.85      yamt     1044:        if (copyhdr && (m0->m_flags & M_PKTHDR)) {
1.9       mycroft  1045:                MGETHDR(n, wait, m0->m_type);
                   1046:                if (n == 0)
1.71      simonb   1047:                        return (NULL);
1.112     pavel    1048:                MCLAIM(n, m0->m_owner);
1.9       mycroft  1049:                n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
                   1050:                n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1.22      thorpej  1051:                len_save = m0->m_pkthdr.len;
1.9       mycroft  1052:                m0->m_pkthdr.len = len0;
                   1053:                if (m->m_flags & M_EXT)
                   1054:                        goto extpacket;
                   1055:                if (remain > MHLEN) {
                   1056:                        /* m can't be the lead packet */
                   1057:                        MH_ALIGN(n, 0);
                   1058:                        n->m_next = m_split(m, len, wait);
                   1059:                        if (n->m_next == 0) {
                   1060:                                (void) m_free(n);
1.22      thorpej  1061:                                m0->m_pkthdr.len = len_save;
1.71      simonb   1062:                                return (NULL);
1.9       mycroft  1063:                        } else
                   1064:                                return (n);
                   1065:                } else
                   1066:                        MH_ALIGN(n, remain);
                   1067:        } else if (remain == 0) {
                   1068:                n = m->m_next;
                   1069:                m->m_next = 0;
                   1070:                return (n);
                   1071:        } else {
                   1072:                MGET(n, wait, m->m_type);
                   1073:                if (n == 0)
1.71      simonb   1074:                        return (NULL);
1.64      matt     1075:                MCLAIM(n, m->m_owner);
1.9       mycroft  1076:                M_ALIGN(n, remain);
                   1077:        }
                   1078: extpacket:
                   1079:        if (m->m_flags & M_EXT) {
1.125     yamt     1080:                n->m_data = m->m_data + len;
1.18      thorpej  1081:                MCLADDREFERENCE(m, n);
1.9       mycroft  1082:        } else {
1.119     christos 1083:                memcpy(mtod(n, void *), mtod(m, char *) + len, remain);
1.9       mycroft  1084:        }
                   1085:        n->m_len = remain;
                   1086:        m->m_len = len;
                   1087:        n->m_next = m->m_next;
                   1088:        m->m_next = 0;
                   1089:        return (n);
                   1090: }
                   1091: /*
                   1092:  * Routine to copy from device local memory into mbufs.
                   1093:  */
                   1094: struct mbuf *
1.62      thorpej  1095: m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
                   1096:     void (*copy)(const void *from, void *to, size_t len))
1.9       mycroft  1097: {
1.27      matt     1098:        struct mbuf *m;
1.9       mycroft  1099:        struct mbuf *top = 0, **mp = &top;
1.27      matt     1100:        int off = off0, len;
                   1101:        char *cp;
1.9       mycroft  1102:        char *epkt;
                   1103:
                   1104:        cp = buf;
                   1105:        epkt = cp + totlen;
                   1106:        if (off) {
1.13      cgd      1107:                /*
                   1108:                 * If 'off' is non-zero, packet is trailer-encapsulated,
                   1109:                 * so we have to skip the type and length fields.
                   1110:                 */
1.104     perry    1111:                cp += off + 2 * sizeof(uint16_t);
                   1112:                totlen -= 2 * sizeof(uint16_t);
1.9       mycroft  1113:        }
                   1114:        MGETHDR(m, M_DONTWAIT, MT_DATA);
                   1115:        if (m == 0)
1.71      simonb   1116:                return (NULL);
1.9       mycroft  1117:        m->m_pkthdr.rcvif = ifp;
                   1118:        m->m_pkthdr.len = totlen;
                   1119:        m->m_len = MHLEN;
                   1120:
                   1121:        while (totlen > 0) {
                   1122:                if (top) {
                   1123:                        MGET(m, M_DONTWAIT, MT_DATA);
                   1124:                        if (m == 0) {
                   1125:                                m_freem(top);
1.71      simonb   1126:                                return (NULL);
1.9       mycroft  1127:                        }
                   1128:                        m->m_len = MLEN;
                   1129:                }
                   1130:                len = min(totlen, epkt - cp);
                   1131:                if (len >= MINCLSIZE) {
                   1132:                        MCLGET(m, M_DONTWAIT);
1.19      mycroft  1133:                        if ((m->m_flags & M_EXT) == 0) {
1.20      mycroft  1134:                                m_free(m);
1.19      mycroft  1135:                                m_freem(top);
1.71      simonb   1136:                                return (NULL);
1.19      mycroft  1137:                        }
                   1138:                        m->m_len = len = min(len, MCLBYTES);
1.9       mycroft  1139:                } else {
                   1140:                        /*
                   1141:                         * Place initial small packet/header at end of mbuf.
                   1142:                         */
                   1143:                        if (len < m->m_len) {
                   1144:                                if (top == 0 && len + max_linkhdr <= m->m_len)
                   1145:                                        m->m_data += max_linkhdr;
                   1146:                                m->m_len = len;
                   1147:                        } else
                   1148:                                len = m->m_len;
                   1149:                }
                   1150:                if (copy)
1.119     christos 1151:                        copy(cp, mtod(m, void *), (size_t)len);
1.9       mycroft  1152:                else
1.119     christos 1153:                        memcpy(mtod(m, void *), cp, (size_t)len);
1.9       mycroft  1154:                cp += len;
                   1155:                *mp = m;
                   1156:                mp = &m->m_next;
                   1157:                totlen -= len;
                   1158:                if (cp == epkt)
                   1159:                        cp = buf;
                   1160:        }
                   1161:        return (top);
1.18      thorpej  1162: }
                   1163:
                   1164: /*
                   1165:  * Copy data from a buffer back into the indicated mbuf chain,
                   1166:  * starting "off" bytes from the beginning, extending the mbuf
                   1167:  * chain if necessary.
                   1168:  */
                   1169: void
1.86      yamt     1170: m_copyback(struct mbuf *m0, int off, int len, const void *cp)
1.18      thorpej  1171: {
1.85      yamt     1172: #if defined(DEBUG)
                   1173:        struct mbuf *origm = m0;
                   1174:        int error;
                   1175: #endif /* defined(DEBUG) */
                   1176:
                   1177:        if (m0 == NULL)
                   1178:                return;
                   1179:
                   1180: #if defined(DEBUG)
                   1181:        error =
                   1182: #endif /* defined(DEBUG) */
                   1183:        m_copyback0(&m0, off, len, cp,
                   1184:            M_COPYBACK0_COPYBACK|M_COPYBACK0_EXTEND, M_DONTWAIT);
                   1185:
                   1186: #if defined(DEBUG)
                   1187:        if (error != 0 || (m0 != NULL && origm != m0))
                   1188:                panic("m_copyback");
                   1189: #endif /* defined(DEBUG) */
                   1190: }
                   1191:
                   1192: struct mbuf *
1.86      yamt     1193: m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how)
1.85      yamt     1194: {
                   1195:        int error;
                   1196:
                   1197:        /* don't support chain expansion */
                   1198:        KDASSERT(off + len <= m_length(m0));
                   1199:
                   1200:        error = m_copyback0(&m0, off, len, cp,
                   1201:            M_COPYBACK0_COPYBACK|M_COPYBACK0_COW, how);
                   1202:        if (error) {
                   1203:                /*
                   1204:                 * no way to recover from partial success.
                   1205:                 * just free the chain.
                   1206:                 */
                   1207:                m_freem(m0);
                   1208:                return NULL;
                   1209:        }
                   1210:        return m0;
                   1211: }
                   1212:
                   1213: /*
                   1214:  * m_makewritable: ensure the specified range writable.
                   1215:  */
                   1216: int
                   1217: m_makewritable(struct mbuf **mp, int off, int len, int how)
                   1218: {
                   1219:        int error;
                   1220: #if defined(DEBUG)
                   1221:        struct mbuf *n;
                   1222:        int origlen, reslen;
                   1223:
                   1224:        origlen = m_length(*mp);
                   1225: #endif /* defined(DEBUG) */
                   1226:
                   1227: #if 0 /* M_COPYALL is large enough */
                   1228:        if (len == M_COPYALL)
                   1229:                len = m_length(*mp) - off; /* XXX */
                   1230: #endif
                   1231:
                   1232:        error = m_copyback0(mp, off, len, NULL,
                   1233:            M_COPYBACK0_PRESERVE|M_COPYBACK0_COW, how);
                   1234:
                   1235: #if defined(DEBUG)
                   1236:        reslen = 0;
                   1237:        for (n = *mp; n; n = n->m_next)
                   1238:                reslen += n->m_len;
                   1239:        if (origlen != reslen)
                   1240:                panic("m_makewritable: length changed");
                   1241:        if (((*mp)->m_flags & M_PKTHDR) != 0 && reslen != (*mp)->m_pkthdr.len)
                   1242:                panic("m_makewritable: inconsist");
                   1243: #endif /* defined(DEBUG) */
                   1244:
                   1245:        return error;
                   1246: }
                   1247:
                   1248: int
1.86      yamt     1249: m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags,
                   1250:     int how)
1.85      yamt     1251: {
1.27      matt     1252:        int mlen;
1.85      yamt     1253:        struct mbuf *m, *n;
                   1254:        struct mbuf **mp;
1.18      thorpej  1255:        int totlen = 0;
1.86      yamt     1256:        const char *cp = vp;
1.18      thorpej  1257:
1.85      yamt     1258:        KASSERT(mp0 != NULL);
                   1259:        KASSERT(*mp0 != NULL);
                   1260:        KASSERT((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL);
                   1261:        KASSERT((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL);
                   1262:
1.106     yamt     1263:        /*
                   1264:         * we don't bother to update "totlen" in the case of M_COPYBACK0_COW,
                   1265:         * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive.
                   1266:         */
                   1267:
                   1268:        KASSERT((~flags & (M_COPYBACK0_EXTEND|M_COPYBACK0_COW)) != 0);
                   1269:
1.85      yamt     1270:        mp = mp0;
                   1271:        m = *mp;
1.18      thorpej  1272:        while (off > (mlen = m->m_len)) {
                   1273:                off -= mlen;
                   1274:                totlen += mlen;
1.109     yamt     1275:                if (m->m_next == NULL) {
                   1276:                        int tspace;
                   1277: extend:
1.85      yamt     1278:                        if ((flags & M_COPYBACK0_EXTEND) == 0)
                   1279:                                goto out;
1.109     yamt     1280:
                   1281:                        /*
                   1282:                         * try to make some space at the end of "m".
                   1283:                         */
                   1284:
                   1285:                        mlen = m->m_len;
                   1286:                        if (off + len >= MINCLSIZE &&
                   1287:                            (m->m_flags & M_EXT) == 0 && m->m_len == 0) {
                   1288:                                MCLGET(m, how);
                   1289:                        }
                   1290:                        tspace = M_TRAILINGSPACE(m);
                   1291:                        if (tspace > 0) {
                   1292:                                tspace = min(tspace, off + len);
                   1293:                                KASSERT(tspace > 0);
                   1294:                                memset(mtod(m, char *) + m->m_len, 0,
                   1295:                                    min(off, tspace));
                   1296:                                m->m_len += tspace;
                   1297:                                off += mlen;
                   1298:                                totlen -= mlen;
                   1299:                                continue;
                   1300:                        }
                   1301:
                   1302:                        /*
                   1303:                         * need to allocate an mbuf.
                   1304:                         */
                   1305:
                   1306:                        if (off + len >= MINCLSIZE) {
                   1307:                                n = m_getcl(how, m->m_type, 0);
                   1308:                        } else {
                   1309:                                n = m_get(how, m->m_type);
                   1310:                        }
                   1311:                        if (n == NULL) {
1.18      thorpej  1312:                                goto out;
1.109     yamt     1313:                        }
                   1314:                        n->m_len = 0;
                   1315:                        n->m_len = min(M_TRAILINGSPACE(n), off + len);
                   1316:                        memset(mtod(n, char *), 0, min(n->m_len, off));
1.18      thorpej  1317:                        m->m_next = n;
                   1318:                }
1.85      yamt     1319:                mp = &m->m_next;
1.18      thorpej  1320:                m = m->m_next;
                   1321:        }
                   1322:        while (len > 0) {
1.85      yamt     1323:                mlen = m->m_len - off;
                   1324:                if (mlen != 0 && M_READONLY(m)) {
                   1325:                        char *datap;
                   1326:                        int eatlen;
                   1327:
                   1328:                        /*
                   1329:                         * this mbuf is read-only.
                   1330:                         * allocate a new writable mbuf and try again.
                   1331:                         */
                   1332:
                   1333: #if defined(DIAGNOSTIC)
                   1334:                        if ((flags & M_COPYBACK0_COW) == 0)
                   1335:                                panic("m_copyback0: read-only");
                   1336: #endif /* defined(DIAGNOSTIC) */
                   1337:
                   1338:                        /*
                   1339:                         * if we're going to write into the middle of
                   1340:                         * a mbuf, split it first.
                   1341:                         */
                   1342:                        if (off > 0 && len < mlen) {
                   1343:                                n = m_split0(m, off, how, 0);
                   1344:                                if (n == NULL)
                   1345:                                        goto enobufs;
                   1346:                                m->m_next = n;
                   1347:                                mp = &m->m_next;
                   1348:                                m = n;
                   1349:                                off = 0;
                   1350:                                continue;
                   1351:                        }
                   1352:
                   1353:                        /*
                   1354:                         * XXX TODO coalesce into the trailingspace of
                   1355:                         * the previous mbuf when possible.
                   1356:                         */
                   1357:
                   1358:                        /*
                   1359:                         * allocate a new mbuf.  copy packet header if needed.
                   1360:                         */
                   1361:                        MGET(n, how, m->m_type);
                   1362:                        if (n == NULL)
                   1363:                                goto enobufs;
                   1364:                        MCLAIM(n, m->m_owner);
                   1365:                        if (off == 0 && (m->m_flags & M_PKTHDR) != 0) {
1.101     yamt     1366:                                M_MOVE_PKTHDR(n, m);
1.85      yamt     1367:                                n->m_len = MHLEN;
                   1368:                        } else {
                   1369:                                if (len >= MINCLSIZE)
                   1370:                                        MCLGET(n, M_DONTWAIT);
                   1371:                                n->m_len =
                   1372:                                    (n->m_flags & M_EXT) ? MCLBYTES : MLEN;
                   1373:                        }
                   1374:                        if (n->m_len > len)
                   1375:                                n->m_len = len;
                   1376:
                   1377:                        /*
                   1378:                         * free the region which has been overwritten.
                   1379:                         * copying data from old mbufs if requested.
                   1380:                         */
                   1381:                        if (flags & M_COPYBACK0_PRESERVE)
                   1382:                                datap = mtod(n, char *);
                   1383:                        else
                   1384:                                datap = NULL;
                   1385:                        eatlen = n->m_len;
                   1386:                        KDASSERT(off == 0 || eatlen >= mlen);
                   1387:                        if (off > 0) {
                   1388:                                KDASSERT(len >= mlen);
                   1389:                                m->m_len = off;
                   1390:                                m->m_next = n;
                   1391:                                if (datap) {
                   1392:                                        m_copydata(m, off, mlen, datap);
                   1393:                                        datap += mlen;
                   1394:                                }
                   1395:                                eatlen -= mlen;
                   1396:                                mp = &m->m_next;
                   1397:                                m = m->m_next;
                   1398:                        }
                   1399:                        while (m != NULL && M_READONLY(m) &&
                   1400:                            n->m_type == m->m_type && eatlen > 0) {
                   1401:                                mlen = min(eatlen, m->m_len);
                   1402:                                if (datap) {
                   1403:                                        m_copydata(m, 0, mlen, datap);
                   1404:                                        datap += mlen;
                   1405:                                }
                   1406:                                m->m_data += mlen;
                   1407:                                m->m_len -= mlen;
                   1408:                                eatlen -= mlen;
                   1409:                                if (m->m_len == 0)
                   1410:                                        *mp = m = m_free(m);
                   1411:                        }
                   1412:                        if (eatlen > 0)
                   1413:                                n->m_len -= eatlen;
                   1414:                        n->m_next = m;
                   1415:                        *mp = m = n;
                   1416:                        continue;
                   1417:                }
                   1418:                mlen = min(mlen, len);
                   1419:                if (flags & M_COPYBACK0_COPYBACK) {
1.119     christos 1420:                        memcpy(mtod(m, char *) + off, cp, (unsigned)mlen);
1.85      yamt     1421:                        cp += mlen;
                   1422:                }
1.18      thorpej  1423:                len -= mlen;
                   1424:                mlen += off;
                   1425:                off = 0;
                   1426:                totlen += mlen;
                   1427:                if (len == 0)
                   1428:                        break;
1.109     yamt     1429:                if (m->m_next == NULL) {
                   1430:                        goto extend;
1.18      thorpej  1431:                }
1.85      yamt     1432:                mp = &m->m_next;
1.18      thorpej  1433:                m = m->m_next;
                   1434:        }
1.106     yamt     1435: out:   if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) {
                   1436:                KASSERT((flags & M_COPYBACK0_EXTEND) != 0);
1.18      thorpej  1437:                m->m_pkthdr.len = totlen;
1.106     yamt     1438:        }
1.85      yamt     1439:
                   1440:        return 0;
                   1441:
                   1442: enobufs:
                   1443:        return ENOBUFS;
1.66      thorpej  1444: }
                   1445:
1.101     yamt     1446: void
                   1447: m_move_pkthdr(struct mbuf *to, struct mbuf *from)
                   1448: {
                   1449:
                   1450:        KASSERT((to->m_flags & M_EXT) == 0);
                   1451:        KASSERT((to->m_flags & M_PKTHDR) == 0 || m_tag_first(to) == NULL);
                   1452:        KASSERT((from->m_flags & M_PKTHDR) != 0);
                   1453:
                   1454:        to->m_pkthdr = from->m_pkthdr;
                   1455:        to->m_flags = from->m_flags & M_COPYFLAGS;
                   1456:        to->m_data = to->m_pktdat;
                   1457:
                   1458:        from->m_flags &= ~M_PKTHDR;
                   1459: }
                   1460:
1.66      thorpej  1461: /*
                   1462:  * Apply function f to the data in an mbuf chain starting "off" bytes from the
                   1463:  * beginning, continuing for "len" bytes.
                   1464:  */
                   1465: int
                   1466: m_apply(struct mbuf *m, int off, int len,
1.119     christos 1467:     int (*f)(void *, void *, unsigned int), void *arg)
1.66      thorpej  1468: {
                   1469:        unsigned int count;
                   1470:        int rval;
                   1471:
                   1472:        KASSERT(len >= 0);
                   1473:        KASSERT(off >= 0);
                   1474:
                   1475:        while (off > 0) {
                   1476:                KASSERT(m != NULL);
                   1477:                if (off < m->m_len)
                   1478:                        break;
                   1479:                off -= m->m_len;
                   1480:                m = m->m_next;
                   1481:        }
                   1482:        while (len > 0) {
                   1483:                KASSERT(m != NULL);
                   1484:                count = min(m->m_len - off, len);
                   1485:
1.119     christos 1486:                rval = (*f)(arg, mtod(m, char *) + off, count);
1.66      thorpej  1487:                if (rval)
                   1488:                        return (rval);
                   1489:
                   1490:                len -= count;
                   1491:                off = 0;
                   1492:                m = m->m_next;
                   1493:        }
                   1494:
                   1495:        return (0);
                   1496: }
                   1497:
                   1498: /*
                   1499:  * Return a pointer to mbuf/offset of location in mbuf chain.
                   1500:  */
                   1501: struct mbuf *
                   1502: m_getptr(struct mbuf *m, int loc, int *off)
                   1503: {
                   1504:
                   1505:        while (loc >= 0) {
                   1506:                /* Normal end of search */
                   1507:                if (m->m_len > loc) {
                   1508:                        *off = loc;
                   1509:                        return (m);
                   1510:                } else {
                   1511:                        loc -= m->m_len;
                   1512:
                   1513:                        if (m->m_next == NULL) {
                   1514:                                if (loc == 0) {
                   1515:                                        /* Point at the end of valid data */
                   1516:                                        *off = m->m_len;
                   1517:                                        return (m);
                   1518:                                } else
                   1519:                                        return (NULL);
                   1520:                        } else
                   1521:                                m = m->m_next;
                   1522:                }
                   1523:        }
                   1524:
                   1525:        return (NULL);
1.1       cgd      1526: }
1.105     yamt     1527:
1.125     yamt     1528: /*
                   1529:  * m_ext_free: release a reference to the mbuf external storage.
                   1530:  *
                   1531:  * => free the mbuf m itsself as well.
                   1532:  */
                   1533:
                   1534: void
                   1535: m_ext_free(struct mbuf *m)
                   1536: {
                   1537:        bool embedded = MEXT_ISEMBEDDED(m);
                   1538:        bool dofree = true;
                   1539:        u_int refcnt;
                   1540:
                   1541:        KASSERT((m->m_flags & M_EXT) != 0);
                   1542:        KASSERT(MEXT_ISEMBEDDED(m->m_ext_ref));
                   1543:        KASSERT((m->m_ext_ref->m_flags & M_EXT) != 0);
                   1544:        KASSERT((m->m_flags & M_EXT_CLUSTER) ==
                   1545:            (m->m_ext_ref->m_flags & M_EXT_CLUSTER));
                   1546:
                   1547:        if (__predict_true(m->m_ext.ext_refcnt == 1)) {
                   1548:                refcnt = m->m_ext.ext_refcnt = 0;
                   1549:        } else {
                   1550:                refcnt = atomic_dec_uint_nv(&m->m_ext.ext_refcnt);
                   1551:        }
                   1552:        if (refcnt > 0) {
                   1553:                if (embedded) {
                   1554:                        /*
                   1555:                         * other mbuf's m_ext_ref still points to us.
                   1556:                         */
                   1557:                        dofree = false;
                   1558:                } else {
                   1559:                        m->m_ext_ref = m;
                   1560:                }
                   1561:        } else {
                   1562:                /*
                   1563:                 * dropping the last reference
                   1564:                 */
                   1565:                if (!embedded) {
                   1566:                        m->m_ext.ext_refcnt++; /* XXX */
                   1567:                        m_ext_free(m->m_ext_ref);
                   1568:                        m->m_ext_ref = m;
                   1569:                } else if ((m->m_flags & M_EXT_CLUSTER) != 0) {
                   1570:                        pool_cache_put_paddr((struct pool_cache *)
                   1571:                            m->m_ext.ext_arg,
                   1572:                            m->m_ext.ext_buf, m->m_ext.ext_paddr);
                   1573:                } else if (m->m_ext.ext_free) {
                   1574:                        (*m->m_ext.ext_free)(m,
                   1575:                            m->m_ext.ext_buf, m->m_ext.ext_size,
                   1576:                            m->m_ext.ext_arg);
                   1577:                        /*
                   1578:                         * 'm' is already freed by the ext_free callback.
                   1579:                         */
                   1580:                        dofree = false;
                   1581:                } else {
                   1582:                        free(m->m_ext.ext_buf, m->m_ext.ext_type);
                   1583:                }
                   1584:        }
                   1585:        if (dofree) {
                   1586:                pool_cache_put(mb_cache, m);
                   1587:        }
                   1588: }
                   1589:
1.105     yamt     1590: #if defined(DDB)
                   1591: void
                   1592: m_print(const struct mbuf *m, const char *modif, void (*pr)(const char *, ...))
                   1593: {
                   1594:        char ch;
1.118     thorpej  1595:        bool opt_c = false;
1.105     yamt     1596:        char buf[512];
                   1597:
                   1598:        while ((ch = *(modif++)) != '\0') {
                   1599:                switch (ch) {
                   1600:                case 'c':
1.118     thorpej  1601:                        opt_c = true;
1.105     yamt     1602:                        break;
                   1603:                }
                   1604:        }
                   1605:
                   1606: nextchain:
                   1607:        (*pr)("MBUF %p\n", m);
1.123     yamt     1608:        bitmask_snprintf((u_int)m->m_flags, M_FLAGS_BITS, buf, sizeof(buf));
1.105     yamt     1609:        (*pr)("  data=%p, len=%d, type=%d, flags=0x%s\n",
                   1610:            m->m_data, m->m_len, m->m_type, buf);
                   1611:        (*pr)("  owner=%p, next=%p, nextpkt=%p\n", m->m_owner, m->m_next,
                   1612:            m->m_nextpkt);
                   1613:        (*pr)("  leadingspace=%u, trailingspace=%u, readonly=%u\n",
                   1614:            (int)M_LEADINGSPACE(m), (int)M_TRAILINGSPACE(m),
                   1615:            (int)M_READONLY(m));
                   1616:        if ((m->m_flags & M_PKTHDR) != 0) {
                   1617:                bitmask_snprintf(m->m_pkthdr.csum_flags, M_CSUM_BITS, buf,
                   1618:                    sizeof(buf));
                   1619:                (*pr)("  pktlen=%d, rcvif=%p, csum_flags=0x%s, csum_data=0x%"
                   1620:                    PRIx32 ", segsz=%u\n",
                   1621:                    m->m_pkthdr.len, m->m_pkthdr.rcvif,
                   1622:                    buf, m->m_pkthdr.csum_data, m->m_pkthdr.segsz);
                   1623:        }
                   1624:        if ((m->m_flags & M_EXT)) {
1.125     yamt     1625:                (*pr)("  ext_refcnt=%u, ext_buf=%p, ext_size=%zd, "
1.105     yamt     1626:                    "ext_free=%p, ext_arg=%p\n",
1.125     yamt     1627:                    m->m_ext.ext_refcnt,
1.105     yamt     1628:                    m->m_ext.ext_buf, m->m_ext.ext_size,
                   1629:                    m->m_ext.ext_free, m->m_ext.ext_arg);
                   1630:        }
                   1631:        if ((~m->m_flags & (M_EXT|M_EXT_PAGES)) == 0) {
1.108     yamt     1632:                vaddr_t sva = (vaddr_t)m->m_ext.ext_buf;
                   1633:                vaddr_t eva = sva + m->m_ext.ext_size;
                   1634:                int n = (round_page(eva) - trunc_page(sva)) >> PAGE_SHIFT;
                   1635:                int i;
1.105     yamt     1636:
                   1637:                (*pr)("  pages:");
1.108     yamt     1638:                for (i = 0; i < n; i ++) {
                   1639:                        (*pr)(" %p", m->m_ext.ext_pgs[i]);
1.105     yamt     1640:                }
                   1641:                (*pr)("\n");
                   1642:        }
                   1643:
                   1644:        if (opt_c) {
                   1645:                m = m->m_next;
                   1646:                if (m != NULL) {
                   1647:                        goto nextchain;
                   1648:                }
                   1649:        }
                   1650: }
                   1651: #endif /* defined(DDB) */
1.124     yamt     1652:
                   1653: void
                   1654: mbstat_type_add(int type, int diff)
                   1655: {
                   1656:        struct mbstat_cpu *mb;
                   1657:        int s;
                   1658:
                   1659:        s = splvm();
1.126   ! thorpej  1660:        mb = percpu_getref(mbstat_percpu);
1.124     yamt     1661:        mb->m_mtypes[type] += diff;
1.126   ! thorpej  1662:        percpu_putref(mbstat_percpu);
1.124     yamt     1663:        splx(s);
                   1664: }
                   1665:
                   1666: #if defined(MBUFTRACE)
                   1667: void
                   1668: mowner_attach(struct mowner *mo)
                   1669: {
                   1670:
                   1671:        KASSERT(mo->mo_counters == NULL);
                   1672:        mo->mo_counters = percpu_alloc(sizeof(struct mowner_counter));
                   1673:
                   1674:        /* XXX lock */
                   1675:        LIST_INSERT_HEAD(&mowners, mo, mo_link);
                   1676: }
                   1677:
                   1678: void
                   1679: mowner_detach(struct mowner *mo)
                   1680: {
                   1681:
                   1682:        KASSERT(mo->mo_counters != NULL);
                   1683:
                   1684:        /* XXX lock */
                   1685:        LIST_REMOVE(mo, mo_link);
                   1686:
                   1687:        percpu_free(mo->mo_counters, sizeof(struct mowner_counter));
                   1688:        mo->mo_counters = NULL;
                   1689: }
                   1690:
                   1691: void
                   1692: mowner_init(struct mbuf *m, int type)
                   1693: {
                   1694:        struct mowner_counter *mc;
                   1695:        struct mowner *mo;
                   1696:        int s;
                   1697:
                   1698:        m->m_owner = mo = &unknown_mowners[type];
                   1699:        s = splvm();
1.126   ! thorpej  1700:        mc = percpu_getref(mo->mo_counters);
1.124     yamt     1701:        mc->mc_counter[MOWNER_COUNTER_CLAIMS]++;
1.126   ! thorpej  1702:        percpu_putref(mo->mo_counters);
1.124     yamt     1703:        splx(s);
                   1704: }
                   1705:
                   1706: void
                   1707: mowner_ref(struct mbuf *m, int flags)
                   1708: {
                   1709:        struct mowner *mo = m->m_owner;
                   1710:        struct mowner_counter *mc;
                   1711:        int s;
                   1712:
                   1713:        s = splvm();
1.126   ! thorpej  1714:        mc = percpu_getref(mo->mo_counters);
1.124     yamt     1715:        if ((flags & M_EXT) != 0)
                   1716:                mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++;
                   1717:        if ((flags & M_CLUSTER) != 0)
                   1718:                mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++;
1.126   ! thorpej  1719:        percpu_putref(mo->mo_counters);
1.124     yamt     1720:        splx(s);
                   1721: }
                   1722:
                   1723: void
                   1724: mowner_revoke(struct mbuf *m, bool all, int flags)
                   1725: {
                   1726:        struct mowner *mo = m->m_owner;
                   1727:        struct mowner_counter *mc;
                   1728:        int s;
                   1729:
                   1730:        s = splvm();
1.126   ! thorpej  1731:        mc = percpu_getref(mo->mo_counters);
1.124     yamt     1732:        if ((flags & M_EXT) != 0)
                   1733:                mc->mc_counter[MOWNER_COUNTER_EXT_RELEASES]++;
                   1734:        if ((flags & M_CLUSTER) != 0)
                   1735:                mc->mc_counter[MOWNER_COUNTER_CLUSTER_RELEASES]++;
                   1736:        if (all)
                   1737:                mc->mc_counter[MOWNER_COUNTER_RELEASES]++;
1.126   ! thorpej  1738:        percpu_putref(mo->mo_counters);
1.124     yamt     1739:        splx(s);
                   1740:        if (all)
                   1741:                m->m_owner = &revoked_mowner;
                   1742: }
                   1743:
                   1744: static void
                   1745: mowner_claim(struct mbuf *m, struct mowner *mo)
                   1746: {
                   1747:        struct mowner_counter *mc;
                   1748:        int flags = m->m_flags;
                   1749:        int s;
                   1750:
                   1751:        s = splvm();
1.126   ! thorpej  1752:        mc = percpu_getref(mo->mo_counters);
1.124     yamt     1753:        mc->mc_counter[MOWNER_COUNTER_CLAIMS]++;
                   1754:        if ((flags & M_EXT) != 0)
                   1755:                mc->mc_counter[MOWNER_COUNTER_EXT_CLAIMS]++;
                   1756:        if ((flags & M_CLUSTER) != 0)
                   1757:                mc->mc_counter[MOWNER_COUNTER_CLUSTER_CLAIMS]++;
1.126   ! thorpej  1758:        percpu_putref(mo->mo_counters);
1.124     yamt     1759:        splx(s);
                   1760:        m->m_owner = mo;
                   1761: }
                   1762:
                   1763: void
                   1764: m_claim(struct mbuf *m, struct mowner *mo)
                   1765: {
                   1766:
                   1767:        if (m->m_owner == mo || mo == NULL)
                   1768:                return;
                   1769:
                   1770:        mowner_revoke(m, true, m->m_flags);
                   1771:        mowner_claim(m, mo);
                   1772: }
                   1773: #endif /* defined(MBUFTRACE) */

CVSweb <webmaster@jp.NetBSD.org>