Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/netinet/ip_input.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/netinet/ip_input.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.319.4.8 retrieving revision 1.333 diff -u -p -r1.319.4.8 -r1.333 --- src/sys/netinet/ip_input.c 2016/12/05 10:55:28 1.319.4.8 +++ src/sys/netinet/ip_input.c 2016/07/04 08:10:50 1.333 @@ -1,4 +1,4 @@ -/* $NetBSD: ip_input.c,v 1.319.4.8 2016/12/05 10:55:28 skrll Exp $ */ +/* $NetBSD: ip_input.c,v 1.333 2016/07/04 08:10:50 ozaki-r Exp $ */ /* * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -91,7 +91,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.319.4.8 2016/12/05 10:55:28 skrll Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ip_input.c,v 1.333 2016/07/04 08:10:50 ozaki-r Exp $"); #ifdef _KERNEL_OPT #include "opt_inet.h" @@ -101,7 +101,6 @@ __KERNEL_RCSID(0, "$NetBSD: ip_input.c,v #include "opt_mrouting.h" #include "opt_mbuftrace.h" #include "opt_inet_csum.h" -#include "opt_net_mpsafe.h" #endif #include "arp.h" @@ -226,7 +225,7 @@ pfil_head_t * inet_pfil_hook __read_mo ipid_state_t * ip_ids __read_mostly; percpu_t * ipstat_percpu __read_mostly; -static percpu_t *ipforward_rt_percpu __cacheline_aligned; +static struct route ipforward_rt __cacheline_aligned; uint16_t ip_id; @@ -295,13 +294,9 @@ static struct in_ifaddr *ip_match_our_ad static struct in_ifaddr *ip_match_our_address_broadcast(struct ifnet *, struct ip *); -#ifdef NET_MPSAFE -#define SOFTNET_LOCK() mutex_enter(softnet_lock) -#define SOFTNET_UNLOCK() mutex_exit(softnet_lock) -#else +/* XXX: Not yet enabled. */ #define SOFTNET_LOCK() KASSERT(mutex_owned(softnet_lock)) #define SOFTNET_UNLOCK() KASSERT(mutex_owned(softnet_lock)) -#endif /* * IP initialization: fill in IP protocol switch table. @@ -350,10 +345,6 @@ ip_init(void) #endif /* MBUFTRACE */ ipstat_percpu = percpu_alloc(sizeof(uint64_t) * IP_NSTATS); - - ipforward_rt_percpu = percpu_alloc(sizeof(struct route)); - if (ipforward_rt_percpu == NULL) - panic("failed to allocate ipforward_rt_percpu"); } static struct in_ifaddr * @@ -381,17 +372,16 @@ ip_match_our_address(struct ifnet *ifp, checkif = ip_checkinterface && (ipforwarding == 0) && (ifp->if_flags & IFF_LOOPBACK) == 0; - IN_ADDRHASH_READER_FOREACH(ia, ip->ip_dst.s_addr) { + LIST_FOREACH(ia, &IN_IFADDR_HASH(ip->ip_dst.s_addr), ia_hash) { if (in_hosteq(ia->ia_addr.sin_addr, ip->ip_dst)) { if (ia->ia4_flags & IN_IFF_NOTREADY) continue; if (checkif && ia->ia_ifp != ifp) continue; - if ((ia->ia_ifp->if_flags & IFF_UP) != 0 && - (ia->ia4_flags & IN_IFF_DETACHED) == 0) + if ((ia->ia_ifp->if_flags & IFF_UP) != 0) break; else - (*downmatch)++; + downmatch++; } } @@ -404,11 +394,11 @@ ip_match_our_address_broadcast(struct if struct in_ifaddr *ia = NULL; struct ifaddr *ifa; - IFADDR_READER_FOREACH(ifa, ifp) { + IFADDR_FOREACH(ifa, ifp) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); - if (ia->ia4_flags & (IN_IFF_NOTREADY | IN_IFF_DETACHED)) + if (ia->ia4_flags & IN_IFF_NOTREADY) continue; if (in_hosteq(ip->ip_dst, ia->ia_broadaddr.sin_addr) || in_hosteq(ip->ip_dst, ia->ia_netbroadcast) || @@ -442,15 +432,11 @@ ipintr(void *arg __unused) KASSERT(cpu_softintr_p()); -#ifndef NET_MPSAFE mutex_enter(softnet_lock); -#endif while ((m = pktq_dequeue(ip_pktq)) != NULL) { ip_input(m); } -#ifndef NET_MPSAFE mutex_exit(softnet_lock); -#endif } /* @@ -461,13 +447,12 @@ static void ip_input(struct mbuf *m) { struct ip *ip = NULL; - struct in_ifaddr *ia = NULL; + struct in_ifaddr *ia; int hlen = 0, len; int downmatch; int srcrt = 0; ifnet_t *ifp; struct psref psref; - int s; KASSERTMSG(cpu_softintr_p(), "ip_input: not in the software " "interrupt handler; synchronization assumptions violated"); @@ -484,8 +469,9 @@ ip_input(struct mbuf *m) * are receiving, can't do anything with incoming packets yet. * Note: we pre-check without locks held. */ - if (IN_ADDRLIST_READER_EMPTY()) + if (!TAILQ_FIRST(&in_ifaddrhead)) { goto out; + } IP_STATINC(IP_STAT_TOTAL); /* @@ -623,9 +609,10 @@ ip_input(struct mbuf *m) struct in_addr odst = ip->ip_dst; bool freed; + SOFTNET_LOCK(); freed = pfil_run_hooks(inet_pfil_hook, &m, ifp, PFIL_IN) != 0; + SOFTNET_UNLOCK(); if (freed || m == NULL) { - m = NULL; goto out; } ip = mtod(m, struct ip *); @@ -655,7 +642,6 @@ ip_input(struct mbuf *m) if ((*altq_input)(m, AF_INET) == 0) { /* Packet dropped by traffic conditioner. */ SOFTNET_UNLOCK(); - m = NULL; goto out; } SOFTNET_UNLOCK(); @@ -669,10 +655,8 @@ ip_input(struct mbuf *m) * to be sent and the original packet to be freed). */ ip_nhops = 0; /* for source routed packets */ - if (hlen > sizeof (struct ip) && ip_dooptions(m)) { - m = NULL; + if (hlen > sizeof (struct ip) && ip_dooptions(m)) goto out; - } /* * Check our list of addresses, to see if the packet is for us. @@ -682,21 +666,15 @@ ip_input(struct mbuf *m) * or IN_IFF_NOTREADY addresses as not mine. */ downmatch = 0; - s = pserialize_read_enter(); ia = ip_match_our_address(ifp, ip, &downmatch); - if (ia != NULL) { - pserialize_read_exit(s); + if (ia != NULL) goto ours; - } if (ifp->if_flags & IFF_BROADCAST) { ia = ip_match_our_address_broadcast(ifp, ip); - if (ia != NULL) { - pserialize_read_exit(s); + if (ia != NULL) goto ours; - } } - pserialize_read_exit(s); if (IN_MULTICAST(ip->ip_dst.s_addr)) { #ifdef MROUTING @@ -833,17 +811,8 @@ ours: * Switch out to protocol's input routine. */ #if IFA_STATS - if (ia && ip) { - struct in_ifaddr *_ia; - /* - * Keep a reference from ip_match_our_address with psref - * is expensive, so explore ia here again. - */ - s = pserialize_read_enter(); - _ia = in_get_ia(ip->ip_dst); - _ia->ia_ifa.ifa_data.ifad_inbytes += ntohs(ip->ip_len); - pserialize_read_exit(s); - } + if (ia && ip) + ia->ia_ifa.ifa_data.ifad_inbytes += ntohs(ip->ip_len); #endif IP_STATINC(IP_STAT_DELIVERED); @@ -905,8 +874,6 @@ ip_dooptions(struct mbuf *m) int opt, optlen, cnt, off, code, type = ICMP_PARAMPROB, forward = 0; struct in_addr dst; n_time ntime; - struct ifaddr *ifa; - int s; dst = ip->ip_dst; cp = (u_char *)(ip + 1); @@ -958,11 +925,8 @@ ip_dooptions(struct mbuf *m) goto bad; } ipaddr.sin_addr = ip->ip_dst; - - s = pserialize_read_enter(); - ifa = ifa_ifwithaddr(sintosa(&ipaddr)); - if (ifa == NULL) { - pserialize_read_exit(s); + ia = ifatoia(ifa_ifwithaddr(sintosa(&ipaddr))); + if (ia == 0) { if (opt == IPOPT_SSRR) { type = ICMP_UNREACH; code = ICMP_UNREACH_SRCFAIL; @@ -974,8 +938,6 @@ ip_dooptions(struct mbuf *m) */ break; } - pserialize_read_exit(s); - off--; /* 0 origin */ if ((off + sizeof(struct in_addr)) > optlen) { /* @@ -989,26 +951,18 @@ ip_dooptions(struct mbuf *m) */ memcpy((void *)&ipaddr.sin_addr, (void *)(cp + off), sizeof(ipaddr.sin_addr)); - s = pserialize_read_enter(); - if (opt == IPOPT_SSRR) { - ifa = ifa_ifwithladdr(sintosa(&ipaddr)); - if (ifa != NULL) - ia = ifatoia(ifa); - else - ia = NULL; - } else { + if (opt == IPOPT_SSRR) + ia = ifatoia(ifa_ifwithladdr(sintosa(&ipaddr))); + else ia = ip_rtaddr(ipaddr.sin_addr); - } - if (ia == NULL) { + if (ia == 0) { type = ICMP_UNREACH; code = ICMP_UNREACH_SRCFAIL; - pserialize_read_exit(s); goto bad; } ip->ip_dst = ipaddr.sin_addr; bcopy((void *)&ia->ia_addr.sin_addr, (void *)(cp + off), sizeof(struct in_addr)); - pserialize_read_exit(s); cp[IPOPT_OFFSET] += sizeof(struct in_addr); /* * Let ip_intr's mcast routing check handle mcast pkts @@ -1037,22 +991,15 @@ ip_dooptions(struct mbuf *m) * locate outgoing interface; if we're the destination, * use the incoming interface (should be same). */ - s = pserialize_read_enter(); - ifa = ifa_ifwithaddr(sintosa(&ipaddr)); - if (ifa == NULL) { - ia = ip_rtaddr(ipaddr.sin_addr); - if (ia == NULL) { - pserialize_read_exit(s); - type = ICMP_UNREACH; - code = ICMP_UNREACH_HOST; - goto bad; - } - } else { - ia = ifatoia(ifa); + if ((ia = ifatoia(ifa_ifwithaddr(sintosa(&ipaddr)))) + == NULL && + (ia = ip_rtaddr(ipaddr.sin_addr)) == NULL) { + type = ICMP_UNREACH; + code = ICMP_UNREACH_HOST; + goto bad; } bcopy((void *)&ia->ia_addr.sin_addr, (void *)(cp + off), sizeof(struct in_addr)); - pserialize_read_exit(s); cp[IPOPT_OFFSET] += sizeof(struct in_addr); break; @@ -1083,7 +1030,7 @@ ip_dooptions(struct mbuf *m) case IPOPT_TS_TSANDADDR: { struct ifnet *rcvif; - int _s, _ss; + int s; if (ipt->ipt_ptr - 1 + sizeof(n_time) + sizeof(struct in_addr) > ipt->ipt_len) { @@ -1092,18 +1039,14 @@ ip_dooptions(struct mbuf *m) goto bad; } ipaddr.sin_addr = dst; - _ss = pserialize_read_enter(); - rcvif = m_get_rcvif(m, &_s); - ifa = ifaof_ifpforaddr(sintosa(&ipaddr), rcvif); - m_put_rcvif(rcvif, &_s); - if (ifa == NULL) { - pserialize_read_exit(_ss); - break; - } - ia = ifatoia(ifa); + rcvif = m_get_rcvif(m, &s); + ia = ifatoia(ifaof_ifpforaddr(sintosa(&ipaddr), + rcvif)); + m_put_rcvif(rcvif, &s); + if (ia == 0) + continue; bcopy(&ia->ia_addr.sin_addr, cp0, sizeof(struct in_addr)); - pserialize_read_exit(_ss); ipt->ipt_ptr += sizeof(struct in_addr); break; } @@ -1117,13 +1060,9 @@ ip_dooptions(struct mbuf *m) } memcpy(&ipaddr.sin_addr, cp0, sizeof(struct in_addr)); - s = pserialize_read_enter(); - ifa = ifa_ifwithaddr(sintosa(&ipaddr)); - if (ifa == NULL) { - pserialize_read_exit(s); + if (ifatoia(ifa_ifwithaddr(sintosa(&ipaddr))) + == NULL) continue; - } - pserialize_read_exit(s); ipt->ipt_ptr += sizeof(struct in_addr); break; @@ -1142,7 +1081,7 @@ ip_dooptions(struct mbuf *m) } if (forward) { struct ifnet *rcvif; - struct psref _psref; + struct psref psref; if (ip_forwsrcrt == 0) { type = ICMP_UNREACH; @@ -1150,14 +1089,14 @@ ip_dooptions(struct mbuf *m) goto bad; } - rcvif = m_get_rcvif_psref(m, &_psref); + rcvif = m_get_rcvif_psref(m, &psref); if (__predict_false(rcvif == NULL)) { type = ICMP_UNREACH; code = ICMP_UNREACH_HOST; goto bad; } ip_forward(m, 1, rcvif); - m_put_rcvif_psref(rcvif, &_psref); + m_put_rcvif_psref(rcvif, &psref); return true; } return false; @@ -1179,14 +1118,11 @@ ip_rtaddr(struct in_addr dst) struct sockaddr dst; struct sockaddr_in dst4; } u; - struct route *ro; sockaddr_in_init(&u.dst4, &dst, 0); SOFTNET_LOCK(); - ro = percpu_getref(ipforward_rt_percpu); - rt = rtcache_lookup(ro, &u.dst); - percpu_putref(ipforward_rt_percpu); + rt = rtcache_lookup(&ipforward_rt, &u.dst); SOFTNET_UNLOCK(); if (rt == NULL) return NULL; @@ -1319,7 +1255,6 @@ ip_forward(struct mbuf *m, int srcrt, st struct sockaddr_in dst4; } u; uint64_t *ips; - struct route *ro; KASSERTMSG(cpu_softintr_p(), "ip_forward: not in the software " "interrupt handler; synchronization assumptions violated"); @@ -1341,17 +1276,19 @@ ip_forward(struct mbuf *m, int srcrt, st return; } + SOFTNET_LOCK(); + if (ip->ip_ttl <= IPTTLDEC) { icmp_error(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, dest, 0); + SOFTNET_UNLOCK(); return; } sockaddr_in_init(&u.dst4, &ip->ip_dst, 0); - ro = percpu_getref(ipforward_rt_percpu); - if ((rt = rtcache_lookup(ro, &u.dst)) == NULL) { - percpu_putref(ipforward_rt_percpu); + if ((rt = rtcache_lookup(&ipforward_rt, &u.dst)) == NULL) { icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, dest, 0); + SOFTNET_UNLOCK(); return; } @@ -1394,7 +1331,7 @@ ip_forward(struct mbuf *m, int srcrt, st } } - error = ip_output(m, NULL, ro, + error = ip_output(m, NULL, &ipforward_rt, (IP_FORWARDING | (ip_directedbcast ? IP_ALLOWBROADCAST : 0)), NULL, NULL); @@ -1416,18 +1353,18 @@ ip_forward(struct mbuf *m, int srcrt, st if (mcopy) { #ifdef GATEWAY if (mcopy->m_flags & M_CANFASTFWD) - ipflow_create(ro, mcopy); + ipflow_create(&ipforward_rt, mcopy); #endif m_freem(mcopy); } - percpu_putref(ipforward_rt_percpu); + SOFTNET_UNLOCK(); return; redirect: error: if (mcopy == NULL) { - percpu_putref(ipforward_rt_percpu); + SOFTNET_UNLOCK(); return; } @@ -1450,7 +1387,7 @@ error: type = ICMP_UNREACH; code = ICMP_UNREACH_NEEDFRAG; - if ((rt = rtcache_validate(ro)) != NULL) + if ((rt = rtcache_validate(&ipforward_rt)) != NULL) destmtu = rt->rt_ifp->if_mtu; #ifdef IPSEC if (ipsec_used) @@ -1468,11 +1405,11 @@ error: */ if (mcopy) m_freem(mcopy); - percpu_putref(ipforward_rt_percpu); + SOFTNET_UNLOCK(); return; } icmp_error(mcopy, type, code, dest, destmtu); - percpu_putref(ipforward_rt_percpu); + SOFTNET_UNLOCK(); } void @@ -1536,8 +1473,8 @@ ip_savecontrol(struct inpcb *inp, struct if (inpflags & INP_RECVIF) { struct sockaddr_dl sdl; - sockaddr_dl_init(&sdl, sizeof(sdl), ifp->if_index, 0, NULL, 0, - NULL, 0); + sockaddr_dl_init(&sdl, sizeof(sdl), ifp ? + ifp->if_index : 0, 0, NULL, 0, NULL, 0); *mp = sbcreatecontrol(&sdl, sdl.sdl_len, IP_RECVIF, IPPROTO_IP); if (*mp) mp = &(*mp)->m_next;