Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/netinet6/ip6_input.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/netinet6/ip6_input.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.154 retrieving revision 1.167 diff -u -p -r1.154 -r1.167 --- src/sys/netinet6/ip6_input.c 2016/01/08 03:55:39 1.154 +++ src/sys/netinet6/ip6_input.c 2016/08/31 09:14:47 1.167 @@ -1,4 +1,4 @@ -/* $NetBSD: ip6_input.c,v 1.154 2016/01/08 03:55:39 knakahara Exp $ */ +/* $NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $ */ /* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */ /* @@ -62,7 +62,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.154 2016/01/08 03:55:39 knakahara Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ip6_input.c,v 1.167 2016/08/31 09:14:47 ozaki-r Exp $"); #ifdef _KERNEL_OPT #include "opt_gateway.h" @@ -87,6 +87,7 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_input.c, #include #include #include +#include #include #include @@ -133,11 +134,8 @@ __KERNEL_RCSID(0, "$NetBSD: ip6_input.c, extern struct domain inet6domain; u_char ip6_protox[IPPROTO_MAX]; -struct in6_ifaddr *in6_ifaddr; pktqueue_t *ip6_pktq __read_mostly; -extern callout_t in6_tmpaddrtimer_ch; - int ip6_forward_srcrt; /* XXX */ int ip6_sourcecheck; /* XXX */ int ip6_sourcecheck_interval; /* XXX */ @@ -146,7 +144,9 @@ pfil_head_t *inet6_pfil_hook; percpu_t *ip6stat_percpu; -static void ip6_init2(void *); +percpu_t *ip6_forward_rt_percpu __cacheline_aligned; + +static void ip6_init2(void); static void ip6intr(void *); static struct m_tag *ip6_setdstifaddr(struct mbuf *, const struct in6_ifaddr *); @@ -165,6 +165,8 @@ ip6_init(void) const struct ip6protosw *pr; int i; + in6_init(); + sysctl_net_inet6_ip6_setup(NULL); pr = (const struct ip6protosw *)pffindproto(PF_INET6, IPPROTO_RAW, SOCK_RAW); if (pr == 0) @@ -186,7 +188,7 @@ ip6_init(void) frag6_init(); ip6_desync_factor = cprng_fast32() % MAX_TEMP_DESYNC_FACTOR; - ip6_init2(NULL); + ip6_init2(); #ifdef GATEWAY ip6flow_init(ip6_hashsize); #endif @@ -195,16 +197,16 @@ ip6_init(void) KASSERT(inet6_pfil_hook != NULL); ip6stat_percpu = percpu_alloc(sizeof(uint64_t) * IP6_NSTATS); + + ip6_forward_rt_percpu = percpu_alloc(sizeof(struct route)); + if (ip6_forward_rt_percpu == NULL) + panic("failed to alllocate ip6_forward_rt_percpu"); } static void -ip6_init2(void *dummy) +ip6_init2(void) { - /* nd6_timer_init */ - callout_init(&nd6_timer_ch, CALLOUT_MPSAFE); - callout_reset(&nd6_timer_ch, hz, nd6_timer, NULL); - /* timer for regeneranation of temporary addresses randomize ID */ callout_init(&in6_tmpaddrtimer_ch, CALLOUT_MPSAFE); callout_reset(&in6_tmpaddrtimer_ch, @@ -223,24 +225,29 @@ ip6intr(void *arg __unused) mutex_enter(softnet_lock); while ((m = pktq_dequeue(ip6_pktq)) != NULL) { - const ifnet_t *ifp = m->m_pkthdr.rcvif; + struct psref psref; + struct ifnet *rcvif = m_get_rcvif_psref(m, &psref); + if (rcvif == NULL) { + m_freem(m); + continue; + } /* * Drop the packet if IPv6 is disabled on the interface. */ - if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)) { + if ((ND_IFINFO(rcvif)->flags & ND6_IFF_IFDISABLED)) { + m_put_rcvif_psref(rcvif, &psref); m_freem(m); continue; } - ip6_input(m); + ip6_input(m, rcvif); + m_put_rcvif_psref(rcvif, &psref); } mutex_exit(softnet_lock); } -extern struct route ip6_forward_rt; - void -ip6_input(struct mbuf *m) +ip6_input(struct mbuf *m, struct ifnet *rcvif) { struct ip6_hdr *ip6; int hit, off = sizeof(struct ip6_hdr), nest; @@ -254,6 +261,7 @@ ip6_input(struct mbuf *m) struct sockaddr dst; struct sockaddr_in6 dst6; } u; + struct route *ro; /* * make sure we don't have onion peering information into m_tag. @@ -271,19 +279,18 @@ ip6_input(struct mbuf *m) } else { #define M2MMAX 32 if (m->m_next) { - if (m->m_flags & M_LOOP) { + if (m->m_flags & M_LOOP) /*XXX*/ IP6_STATINC(IP6_STAT_M2M + lo0ifp->if_index); - } else if (m->m_pkthdr.rcvif->if_index < M2MMAX) { - IP6_STATINC(IP6_STAT_M2M + - m->m_pkthdr.rcvif->if_index); - } else + else if (rcvif->if_index < M2MMAX) + IP6_STATINC(IP6_STAT_M2M + rcvif->if_index); + else IP6_STATINC(IP6_STAT_M2M); } else IP6_STATINC(IP6_STAT_M1); #undef M2MMAX } - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_receive); + in6_ifstat_inc(rcvif, ifs6_in_receive); IP6_STATINC(IP6_STAT_TOTAL); /* @@ -293,19 +300,17 @@ ip6_input(struct mbuf *m) * IPv6 header is in the first mbuf of the chain. */ if (IP6_HDR_ALIGNED_P(mtod(m, void *)) == 0) { - struct ifnet *inifp = m->m_pkthdr.rcvif; if ((m = m_copyup(m, sizeof(struct ip6_hdr), (max_linkhdr + 3) & ~3)) == NULL) { /* XXXJRT new stat, please */ IP6_STATINC(IP6_STAT_TOOSMALL); - in6_ifstat_inc(inifp, ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); return; } } else if (__predict_false(m->m_len < sizeof(struct ip6_hdr))) { - struct ifnet *inifp = m->m_pkthdr.rcvif; if ((m = m_pullup(m, sizeof(struct ip6_hdr))) == NULL) { IP6_STATINC(IP6_STAT_TOOSMALL); - in6_ifstat_inc(inifp, ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); return; } } @@ -314,7 +319,7 @@ ip6_input(struct mbuf *m) if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION) { IP6_STATINC(IP6_STAT_BADVERS); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); goto bad; } @@ -344,8 +349,7 @@ ip6_input(struct mbuf *m) struct in6_addr odst; odst = ip6->ip6_dst; - if (pfil_run_hooks(inet6_pfil_hook, &m, m->m_pkthdr.rcvif, - PFIL_IN) != 0) + if (pfil_run_hooks(inet6_pfil_hook, &m, rcvif, PFIL_IN) != 0) return; if (m == NULL) return; @@ -371,7 +375,7 @@ ip6_input(struct mbuf *m) * XXX: "badscope" is not very suitable for a multicast source. */ IP6_STATINC(IP6_STAT_BADSCOPE); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_addrerr); + in6_ifstat_inc(rcvif, ifs6_in_addrerr); goto bad; } /* @@ -389,7 +393,7 @@ ip6_input(struct mbuf *m) if (IN6_IS_ADDR_V4MAPPED(&ip6->ip6_src) || IN6_IS_ADDR_V4MAPPED(&ip6->ip6_dst)) { IP6_STATINC(IP6_STAT_BADSCOPE); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_addrerr); + in6_ifstat_inc(rcvif, ifs6_in_addrerr); goto bad; } #if 0 @@ -403,7 +407,7 @@ ip6_input(struct mbuf *m) if (IN6_IS_ADDR_V4COMPAT(&ip6->ip6_src) || IN6_IS_ADDR_V4COMPAT(&ip6->ip6_dst)) { IP6_STATINC(IP6_STAT_BADSCOPE); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_addrerr); + in6_ifstat_inc(rcvif, ifs6_in_addrerr); goto bad; } #endif @@ -426,8 +430,8 @@ ip6_input(struct mbuf *m) IP6_STATINC(IP6_STAT_BADSCOPE); /* XXX */ goto bad; } - if (in6_setscope(&ip6->ip6_src, m->m_pkthdr.rcvif, NULL) || - in6_setscope(&ip6->ip6_dst, m->m_pkthdr.rcvif, NULL)) { + if (in6_setscope(&ip6->ip6_src, rcvif, NULL) || + in6_setscope(&ip6->ip6_dst, rcvif, NULL)) { IP6_STATINC(IP6_STAT_BADSCOPE); goto bad; } @@ -438,12 +442,12 @@ ip6_input(struct mbuf *m) if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst)) { struct in6_multi *in6m = 0; - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_mcast); + in6_ifstat_inc(rcvif, ifs6_in_mcast); /* * See if we belong to the destination multicast group on the * arrival interface. */ - IN6_LOOKUP_MULTI(ip6->ip6_dst, m->m_pkthdr.rcvif, in6m); + IN6_LOOKUP_MULTI(ip6->ip6_dst, rcvif, in6m); if (in6m) ours = 1; else if (!ip6_mrouter) { @@ -451,10 +455,10 @@ ip6_input(struct mbuf *m) ip6s[IP6_STAT_NOTMEMBER]++; ip6s[IP6_STAT_CANTFORWARD]++; IP6_STAT_PUTREF(); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_discard); + in6_ifstat_inc(rcvif, ifs6_in_discard); goto bad; } - deliverifp = m->m_pkthdr.rcvif; + deliverifp = rcvif; goto hbhcheck; } @@ -463,7 +467,9 @@ ip6_input(struct mbuf *m) /* * Unicast check */ - rt = rtcache_lookup2(&ip6_forward_rt, &u.dst, 1, &hit); + ro = percpu_getref(ip6_forward_rt_percpu); + rt = rtcache_lookup2(ro, &u.dst, 1, &hit); + percpu_putref(ip6_forward_rt_percpu); if (hit) IP6_STATINC(IP6_STAT_FORWARD_CACHEHIT); else @@ -482,7 +488,6 @@ ip6_input(struct mbuf *m) */ if (rt != NULL && (rt->rt_flags & (RTF_HOST|RTF_GATEWAY)) == RTF_HOST && - !(rt->rt_flags & RTF_CLONED) && #if 0 /* * The check below is redundant since the comparison of @@ -506,10 +511,9 @@ ip6_input(struct mbuf *m) goto hbhcheck; } else { /* address is not ready, so discard the packet. */ - nd6log((LOG_INFO, - "ip6_input: packet to an unready address %s->%s\n", + nd6log(LOG_INFO, "packet to an unready address %s->%s\n", ip6_sprintf(&ip6->ip6_src), - ip6_sprintf(&ip6->ip6_dst))); + ip6_sprintf(&ip6->ip6_dst)); goto bad; } @@ -538,9 +542,7 @@ ip6_input(struct mbuf *m) * working right. */ struct ifaddr *ifa; - IFADDR_FOREACH(ifa, m->m_pkthdr.rcvif) { - if (ifa->ifa_addr == NULL) - continue; /* just for safety */ + IFADDR_READER_FOREACH(ifa, rcvif) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; if (IN6_ARE_ADDR_EQUAL(IFA_IN6(ifa), &ip6->ip6_dst)) { @@ -558,7 +560,7 @@ ip6_input(struct mbuf *m) */ if (!ip6_forwarding) { IP6_STATINC(IP6_STAT_CANTFORWARD); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_discard); + in6_ifstat_inc(rcvif, ifs6_in_discard); goto bad; } @@ -571,8 +573,10 @@ ip6_input(struct mbuf *m) */ if (deliverifp && ip6_getdstifaddr(m) == NULL) { struct in6_ifaddr *ia6; + int s = pserialize_read_enter(); ia6 = in6_ifawithifp(deliverifp, &ip6->ip6_dst); + /* Depends on ip6_setdstifaddr never sleep */ if (ia6 != NULL && ip6_setdstifaddr(m, ia6) == NULL) { /* * XXX maybe we should drop the packet here, @@ -580,6 +584,7 @@ ip6_input(struct mbuf *m) * to the upper layers. */ } + pserialize_read_exit(s); } /* @@ -593,7 +598,7 @@ ip6_input(struct mbuf *m) if (ip6_hopopts_input(&plen, &rtalert, &m, &off)) { #if 0 /*touches NULL pointer*/ - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_discard); + in6_ifstat_inc(rcvif, ifs6_in_discard); #endif return; /* m have already been freed */ } @@ -613,8 +618,8 @@ ip6_input(struct mbuf *m) * (non-zero) payload length to the variable plen. */ IP6_STATINC(IP6_STAT_BADOPTIONS); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_discard); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_discard); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); icmp6_error(m, ICMP6_PARAM_PROB, ICMP6_PARAMPROB_HEADER, (char *)&ip6->ip6_plen - (char *)ip6); @@ -646,7 +651,7 @@ ip6_input(struct mbuf *m) */ if (m->m_pkthdr.len - sizeof(struct ip6_hdr) < plen) { IP6_STATINC(IP6_STAT_TOOSHORT); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_truncated); + in6_ifstat_inc(rcvif, ifs6_in_truncated); goto bad; } if (m->m_pkthdr.len > sizeof(struct ip6_hdr) + plen) { @@ -669,7 +674,7 @@ ip6_input(struct mbuf *m) * ip6_mforward() returns a non-zero value, the packet * must be discarded, else it may be accepted below. */ - if (ip6_mrouter && ip6_mforward(ip6, m->m_pkthdr.rcvif, m)) { + if (ip6_mrouter && ip6_mforward(ip6, rcvif, m)) { IP6_STATINC(IP6_STAT_CANTFORWARD); m_freem(m); return; @@ -697,7 +702,7 @@ ip6_input(struct mbuf *m) if (IN6_IS_ADDR_V4MAPPED(&ip6->ip6_src) || IN6_IS_ADDR_V4MAPPED(&ip6->ip6_dst)) { IP6_STATINC(IP6_STAT_BADSCOPE); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_addrerr); + in6_ifstat_inc(rcvif, ifs6_in_addrerr); goto bad; } @@ -707,9 +712,11 @@ ip6_input(struct mbuf *m) #ifdef IFA_STATS if (deliverifp != NULL) { struct in6_ifaddr *ia6; + int s = pserialize_read_enter(); ia6 = in6_ifawithifp(deliverifp, &ip6->ip6_dst); if (ia6) ia6->ia_ifa.ifa_data.ifad_inbytes += m->m_pkthdr.len; + pserialize_read_exit(s); } #endif IP6_STATINC(IP6_STAT_DELIVERED); @@ -720,7 +727,7 @@ ip6_input(struct mbuf *m) while (nxt != IPPROTO_DONE) { if (ip6_hdrnestlimit && (++nest > ip6_hdrnestlimit)) { IP6_STATINC(IP6_STAT_TOOMANYHDR); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); goto bad; } @@ -730,14 +737,13 @@ ip6_input(struct mbuf *m) */ if (m->m_pkthdr.len < off) { IP6_STATINC(IP6_STAT_TOOSHORT); - in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_truncated); + in6_ifstat_inc(rcvif, ifs6_in_truncated); goto bad; } if (nxt == IPPROTO_ROUTING) { if (rh_present++) { - in6_ifstat_inc(m->m_pkthdr.rcvif, - ifs6_in_hdrerr); + in6_ifstat_inc(rcvif, ifs6_in_hdrerr); IP6_STATINC(IP6_STAT_BADOPTIONS); goto bad; } @@ -1070,8 +1076,7 @@ ip6_savecontrol(struct in6pcb *in6p, str memcpy(&pi6.ipi6_addr, &ip6->ip6_dst, sizeof(struct in6_addr)); in6_clearscope(&pi6.ipi6_addr); /* XXX */ - pi6.ipi6_ifindex = m->m_pkthdr.rcvif ? - m->m_pkthdr.rcvif->if_index : 0; + pi6.ipi6_ifindex = m->m_pkthdr.rcvif_index; *mp = sbcreatecontrol((void *) &pi6, sizeof(struct in6_pktinfo), IS2292(IPV6_2292PKTINFO, IPV6_PKTINFO), IPPROTO_IPV6); @@ -1533,66 +1538,6 @@ ip6_delaux(struct mbuf *m) m_tag_delete(m, mtag); } -#ifdef GATEWAY -/* - * sysctl helper routine for net.inet.ip6.maxflows. Since - * we could reduce this value, call ip6flow_reap(); - */ -static int -sysctl_net_inet6_ip6_maxflows(SYSCTLFN_ARGS) -{ - int error; - - error = sysctl_lookup(SYSCTLFN_CALL(rnode)); - if (error || newp == NULL) - return (error); - - mutex_enter(softnet_lock); - KERNEL_LOCK(1, NULL); - - ip6flow_reap(0); - - KERNEL_UNLOCK_ONE(NULL); - mutex_exit(softnet_lock); - - return (0); -} - -static int -sysctl_net_inet6_ip6_hashsize(SYSCTLFN_ARGS) -{ - int error, tmp; - struct sysctlnode node; - - node = *rnode; - tmp = ip6_hashsize; - node.sysctl_data = &tmp; - error = sysctl_lookup(SYSCTLFN_CALL(&node)); - if (error || newp == NULL) - return (error); - - if ((tmp & (tmp - 1)) == 0 && tmp != 0) { - /* - * Can only fail due to malloc() - */ - mutex_enter(softnet_lock); - KERNEL_LOCK(1, NULL); - - error = ip6flow_invalidate_all(tmp); - - KERNEL_UNLOCK_ONE(NULL); - mutex_exit(softnet_lock); - } else { - /* - * EINVAL if not a power of 2 - */ - error = EINVAL; - } - - return error; -} -#endif /* GATEWAY */ - /* * System control for IP6 */ @@ -1911,22 +1856,6 @@ sysctl_net_inet6_ip6_setup(struct sysctl NULL, 0, &ip6_mcast_pmtu, 0, CTL_NET, PF_INET6, IPPROTO_IPV6, CTL_CREATE, CTL_EOL); -#ifdef GATEWAY - sysctl_createv(clog, 0, NULL, NULL, - CTLFLAG_PERMANENT|CTLFLAG_READWRITE, - CTLTYPE_INT, "maxflows", - SYSCTL_DESCR("Number of flows for fast forwarding (IPv6)"), - sysctl_net_inet6_ip6_maxflows, 0, &ip6_maxflows, 0, - CTL_NET, PF_INET6, IPPROTO_IPV6, - CTL_CREATE, CTL_EOL); - sysctl_createv(clog, 0, NULL, NULL, - CTLFLAG_PERMANENT|CTLFLAG_READWRITE, - CTLTYPE_INT, "hashsize", - SYSCTL_DESCR("Size of hash table for fast forwarding (IPv6)"), - sysctl_net_inet6_ip6_hashsize, 0, &ip6_hashsize, 0, - CTL_NET, PF_INET6, IPPROTO_IPV6, - CTL_CREATE, CTL_EOL); -#endif /* anonportalgo RFC6056 subtree */ const struct sysctlnode *portalgo_node; sysctl_createv(clog, 0, NULL, &portalgo_node,