Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/net/route.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/net/route.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.194 retrieving revision 1.194.6.10 diff -u -p -r1.194 -r1.194.6.10 --- src/sys/net/route.c 2017/03/24 03:45:02 1.194 +++ src/sys/net/route.c 2018/06/08 10:14:33 1.194.6.10 @@ -1,4 +1,4 @@ -/* $NetBSD: route.c,v 1.194 2017/03/24 03:45:02 ozaki-r Exp $ */ +/* $NetBSD: route.c,v 1.194.6.10 2018/06/08 10:14:33 martin Exp $ */ /*- * Copyright (c) 1998, 2008 The NetBSD Foundation, Inc. @@ -97,7 +97,7 @@ #endif #include -__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.194 2017/03/24 03:45:02 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: route.c,v 1.194.6.10 2018/06/08 10:14:33 martin Exp $"); #include #ifdef RTFLUSH_DEBUG @@ -123,10 +123,15 @@ __KERNEL_RCSID(0, "$NetBSD: route.c,v 1. #include #include #include +#if defined(INET) || defined(INET6) +#include +#endif #include #include +#define PRESERVED_RTF (RTF_UP | RTF_GATEWAY | RTF_HOST | RTF_DONE | RTF_MASK) + #ifdef RTFLUSH_DEBUG #define rtcache_debug() __predict_false(_rtcache_debug) #else /* RTFLUSH_DEBUG */ @@ -140,7 +145,7 @@ __KERNEL_RCSID(0, "$NetBSD: route.c,v 1. #define RT_REFCNT_TRACE(rt) do {} while (0) #endif -#ifdef DEBUG +#ifdef RT_DEBUG #define dlog(level, fmt, args...) log(level, fmt, ##args) #else #define dlog(level, fmt, args...) do {} while (0) @@ -166,8 +171,12 @@ static void rt_timer_timer(void *); * Locking notes: * - The routing table is protected by a global rwlock * - API: RT_RLOCK and friends - * - rtcaches are protected by a global rwlock - * - API: RTCACHE_RLOCK and friends + * - rtcaches are NOT protected by the framework + * - Callers must guarantee a rtcache isn't accessed simultaneously + * - How the constraint is guranteed in the wild + * - Protect a rtcache by a mutex (e.g., inp_route) + * - Make rtcache per-CPU and allow only accesses from softint + * (e.g., ipforward_rt_percpu) * - References to a rtentry is managed by reference counting and psref * - Reference couting is used for temporal reference when a rtentry * is fetched from the routing table @@ -200,41 +209,34 @@ static void rt_timer_timer(void *); * - if the caller runs in softint, the caller fails to fetch * - otherwise, the caller waits for the update completed and retries * to fetch (probably succeed to fetch for the second time) + * - rtcache invalidation + * - There is a global generation counter that is incremented when + * any routes have been added or deleted + * - When a rtcache caches a rtentry into itself, it also stores + * a snapshot of the generation counter + * - If the snapshot equals to the global counter, the cache is valid, + * otherwise the cache is invalidated */ /* - * Global locks for the routing table and rtcaches. - * Locking order: rtcache_lock => rt_lock + * Global lock for the routing table. */ static krwlock_t rt_lock __cacheline_aligned; #ifdef NET_MPSAFE #define RT_RLOCK() rw_enter(&rt_lock, RW_READER) #define RT_WLOCK() rw_enter(&rt_lock, RW_WRITER) #define RT_UNLOCK() rw_exit(&rt_lock) -#define RT_LOCKED() rw_lock_held(&rt_lock) +#define RT_WLOCKED() rw_write_held(&rt_lock) #define RT_ASSERT_WLOCK() KASSERT(rw_write_held(&rt_lock)) #else #define RT_RLOCK() do {} while (0) #define RT_WLOCK() do {} while (0) #define RT_UNLOCK() do {} while (0) -#define RT_LOCKED() false +#define RT_WLOCKED() true #define RT_ASSERT_WLOCK() do {} while (0) #endif -static krwlock_t rtcache_lock __cacheline_aligned; -#ifdef NET_MPSAFE -#define RTCACHE_RLOCK() rw_enter(&rtcache_lock, RW_READER) -#define RTCACHE_WLOCK() rw_enter(&rtcache_lock, RW_WRITER) -#define RTCACHE_UNLOCK() rw_exit(&rtcache_lock) -#define RTCACHE_ASSERT_WLOCK() KASSERT(rw_write_held(&rtcache_lock)) -#define RTCACHE_WLOCKED() rw_write_held(&rtcache_lock) -#else -#define RTCACHE_RLOCK() do {} while (0) -#define RTCACHE_WLOCK() do {} while (0) -#define RTCACHE_UNLOCK() do {} while (0) -#define RTCACHE_ASSERT_WLOCK() do {} while (0) -#define RTCACHE_WLOCKED() false -#endif +static uint64_t rtcache_generation; /* * mutex and cv that are used to wait for references to a rtentry left @@ -255,7 +257,8 @@ static struct { struct workqueue *wq; struct work wk; kmutex_t lock; - struct rtentry *queue[10]; + SLIST_HEAD(, rtentry) queue; + bool enqueued; } rt_free_global __cacheline_aligned; /* psref for rtentry */ @@ -268,23 +271,21 @@ static int _rtcache_debug = 0; static kauth_listener_t route_listener; static int rtdeletemsg(struct rtentry *); -static void rtflushall(int); static void rt_maskedcopy(const struct sockaddr *, struct sockaddr *, const struct sockaddr *); -static void rtcache_clear(struct route *); -static void rtcache_clear_rtentry(int, struct rtentry *); -static void rtcache_invalidate(struct dom_rtlist *); +static void rtcache_invalidate(void); static void rt_ref(struct rtentry *); static struct rtentry * - rtalloc1_locked(const struct sockaddr *, int, bool); -static struct rtentry * - rtcache_validate_locked(struct route *); -static void rtcache_free_locked(struct route *); -static int rtcache_setdst_locked(struct route *, const struct sockaddr *); + rtalloc1_locked(const struct sockaddr *, int, bool, bool); + +static struct ifaddr *rt_getifa(struct rt_addrinfo *, struct psref *); +static struct ifnet *rt_getifp(struct rt_addrinfo *, struct psref *); +static struct ifaddr *ifa_ifwithroute_psref(int, const struct sockaddr *, + const struct sockaddr *, struct psref *); static void rtcache_ref(struct rtentry *, struct route *); @@ -465,6 +466,9 @@ rt_init(void) #endif mutex_init(&rt_free_global.lock, MUTEX_DEFAULT, IPL_SOFTNET); + SLIST_INIT(&rt_free_global.queue); + rt_free_global.enqueued = false; + rt_psref_class = psref_class_create("rtentry", IPL_SOFTNET); error = workqueue_create(&rt_free_global.wq, "rt_free", @@ -488,38 +492,15 @@ rt_init(void) } static void -rtflushall(int family) +rtcache_invalidate(void) { - struct domain *dom; + + RT_ASSERT_WLOCK(); if (rtcache_debug()) printf("%s: enter\n", __func__); - if ((dom = pffinddomain(family)) == NULL) - return; - - RTCACHE_WLOCK(); - rtcache_invalidate(&dom->dom_rtcache); - RTCACHE_UNLOCK(); -} - -static void -rtcache(struct route *ro) -{ - struct domain *dom; - - RTCACHE_ASSERT_WLOCK(); - - rtcache_invariants(ro); - KASSERT(ro->_ro_rt != NULL); - KASSERT(ro->ro_invalid == false); - KASSERT(rtcache_getdst(ro) != NULL); - - if ((dom = pffinddomain(rtcache_getdst(ro)->sa_family)) == NULL) - return; - - LIST_INSERT_HEAD(&dom->dom_rtcache, ro, ro_rtcache_next); - rtcache_invariants(ro); + rtcache_generation++; } #ifdef RT_DEBUG @@ -557,7 +538,8 @@ dump_rt(const struct rtentry *rt) * will be incremented. The caller has to rtfree it by itself. */ struct rtentry * -rtalloc1_locked(const struct sockaddr *dst, int report, bool wait_ok) +rtalloc1_locked(const struct sockaddr *dst, int report, bool wait_ok, + bool wlock) { rtbl_t *rtbl; struct rtentry *rt; @@ -582,23 +564,18 @@ retry: if (ISSET(rt->rt_flags, RTF_UPDATING) && /* XXX updater should be always able to acquire */ curlwp != rt_update_global.lwp) { - bool need_lock = false; if (!wait_ok || !rt_wait_ok()) goto miss; RT_UNLOCK(); splx(s); - /* XXX need more proper solution */ - if (RTCACHE_WLOCKED()) { - RTCACHE_UNLOCK(); - need_lock = true; - } - /* We can wait until the update is complete */ rt_update_wait(); - if (need_lock) - RTCACHE_WLOCK(); + if (wlock) + RT_WLOCK(); + else + RT_RLOCK(); goto retry; } #endif /* NET_MPSAFE */ @@ -627,7 +604,7 @@ rtalloc1(const struct sockaddr *dst, int struct rtentry *rt; RT_RLOCK(); - rt = rtalloc1_locked(dst, report, true); + rt = rtalloc1_locked(dst, report, true, false); RT_UNLOCK(); return rt; @@ -720,23 +697,21 @@ _rt_free(struct rtentry *rt) static void rt_free_work(struct work *wk, void *arg) { - int i; - struct rtentry *rt; -restart: - mutex_enter(&rt_free_global.lock); - for (i = 0; i < sizeof(rt_free_global.queue); i++) { - if (rt_free_global.queue[i] == NULL) - continue; - rt = rt_free_global.queue[i]; - rt_free_global.queue[i] = NULL; - mutex_exit(&rt_free_global.lock); + for (;;) { + struct rtentry *rt; + mutex_enter(&rt_free_global.lock); + rt_free_global.enqueued = false; + if ((rt = SLIST_FIRST(&rt_free_global.queue)) == NULL) { + mutex_exit(&rt_free_global.lock); + return; + } + SLIST_REMOVE_HEAD(&rt_free_global.queue, rt_free); + mutex_exit(&rt_free_global.lock); atomic_dec_uint(&rt->rt_refcnt); _rt_free(rt); - goto restart; } - mutex_exit(&rt_free_global.lock); } void @@ -744,23 +719,20 @@ rt_free(struct rtentry *rt) { KASSERT(rt->rt_refcnt > 0); - if (!rt_wait_ok()) { - int i; - mutex_enter(&rt_free_global.lock); - for (i = 0; i < sizeof(rt_free_global.queue); i++) { - if (rt_free_global.queue[i] == NULL) { - rt_free_global.queue[i] = rt; - break; - } - } - KASSERT(i < sizeof(rt_free_global.queue)); - rt_ref(rt); - mutex_exit(&rt_free_global.lock); - workqueue_enqueue(rt_free_global.wq, &rt_free_global.wk, NULL); - } else { + if (rt_wait_ok()) { atomic_dec_uint(&rt->rt_refcnt); _rt_free(rt); + return; } + + mutex_enter(&rt_free_global.lock); + rt_ref(rt); + SLIST_INSERT_HEAD(&rt_free_global.queue, rt, rt_free); + if (!rt_free_global.enqueued) { + workqueue_enqueue(rt_free_global.wq, &rt_free_global.wk, NULL); + rt_free_global.enqueued = true; + } + mutex_exit(&rt_free_global.lock); } #ifdef NET_MPSAFE @@ -784,17 +756,14 @@ rt_update_prepare(struct rtentry *rt) dlog(LOG_DEBUG, "%s: updating rt=%p lwp=%p\n", __func__, rt, curlwp); - RTCACHE_WLOCK(); RT_WLOCK(); /* If the entry is being destroyed, don't proceed the update. */ if (!ISSET(rt->rt_flags, RTF_UP)) { RT_UNLOCK(); - RTCACHE_UNLOCK(); - return -1; + return ESRCH; } rt->rt_flags |= RTF_UPDATING; RT_UNLOCK(); - RTCACHE_UNLOCK(); mutex_enter(&rt_update_global.lock); while (rt_update_global.ongoing) { @@ -819,11 +788,9 @@ void rt_update_finish(struct rtentry *rt) { - RTCACHE_WLOCK(); RT_WLOCK(); rt->rt_flags &= ~RTF_UPDATING; RT_UNLOCK(); - RTCACHE_UNLOCK(); mutex_enter(&rt_update_global.lock); rt_update_global.ongoing = false; @@ -924,11 +891,13 @@ rtredirect(const struct sockaddr *dst, c error = rt_update_prepare(rt); if (error == 0) { #endif + RT_WLOCK(); error = rt_setgate(rt, gateway); if (error == 0) { rt->rt_flags |= RTF_MODIFIED; flags |= RTF_MODIFIED; } + RT_UNLOCK(); #ifdef NET_MPSAFE rt_update_finish(rt); } else { @@ -992,9 +961,9 @@ rtdeletemsg(struct rtentry *rt) return error; } -struct ifaddr * +static struct ifaddr * ifa_ifwithroute_psref(int flags, const struct sockaddr *dst, - const struct sockaddr *gateway, struct psref *psref) + const struct sockaddr *gateway, struct psref *psref) { struct ifaddr *ifa = NULL; @@ -1024,11 +993,7 @@ ifa_ifwithroute_psref(int flags, const s int s; struct rtentry *rt; - /* XXX we cannot call rtalloc1 if holding the rt lock */ - if (RT_LOCKED()) - rt = rtalloc1_locked(gateway, 0, true); - else - rt = rtalloc1(gateway, 0); + rt = rtalloc1_locked(gateway, 0, true, true); if (rt == NULL) return NULL; if (rt->rt_flags & RTF_GATEWAY) { @@ -1114,7 +1079,7 @@ rtrequest_newmsg(const int req, const st return 0; } -struct ifnet * +static struct ifnet * rt_getifp(struct rt_addrinfo *info, struct psref *psref) { const struct sockaddr *ifpaddr = info->rti_info[RTAX_IFP]; @@ -1139,7 +1104,7 @@ rt_getifp(struct rt_addrinfo *info, stru return info->rti_ifp; } -struct ifaddr * +static struct ifaddr * rt_getifa(struct rt_addrinfo *info, struct psref *psref) { struct ifaddr *ifa = NULL; @@ -1240,10 +1205,14 @@ rtrequest1(int req, struct rt_addrinfo * rt_ref(rt); RT_REFCNT_TRACE(rt); } + rtcache_invalidate(); RT_UNLOCK(); need_unlock = false; rt_timer_remove_all(rt); - rtcache_clear_rtentry(dst->sa_family, rt); +#if defined(INET) || defined(INET6) + if (netmask != NULL) + lltable_prefix_free(dst->sa_family, dst, netmask, 0); +#endif if (ret_nrt == NULL) { /* Adjust the refcount */ rt_ref(rt); @@ -1332,9 +1301,9 @@ rtrequest1(int req, struct rt_addrinfo * rt_ref(rt); RT_REFCNT_TRACE(rt); } + rtcache_invalidate(); RT_UNLOCK(); need_unlock = false; - rtflushall(dst->sa_family); break; case RTM_GET: if (netmask != NULL) { @@ -1367,6 +1336,7 @@ rt_setgate(struct rtentry *rt, const str { struct sockaddr *new, *old; + KASSERT(RT_WLOCKED()); KASSERT(rt->_rt_key != NULL); RT_DPRINTF("rt->_rt_key = %p\n", (void *)rt->_rt_key); @@ -1385,11 +1355,7 @@ rt_setgate(struct rtentry *rt, const str if (rt->rt_flags & RTF_GATEWAY) { struct rtentry *gwrt; - /* XXX we cannot call rtalloc1 if holding the rt lock */ - if (RT_LOCKED()) - gwrt = rtalloc1_locked(gate, 1, false); - else - gwrt = rtalloc1(gate, 1); + gwrt = rtalloc1_locked(gate, 1, false, true); /* * If we switched gateways, grab the MTU from the new * gateway route if the current MTU, if the current MTU is @@ -1413,6 +1379,154 @@ rt_setgate(struct rtentry *rt, const str return 0; } +static struct ifaddr * +rt_update_get_ifa(const struct rt_addrinfo info, const struct rtentry *rt, + struct ifnet **ifp, struct psref *psref_ifp, struct psref *psref) +{ + struct ifaddr *ifa = NULL; + + *ifp = NULL; + if (info.rti_info[RTAX_IFP] != NULL) { + ifa = ifa_ifwithnet_psref(info.rti_info[RTAX_IFP], psref); + if (ifa == NULL) + goto next; + *ifp = ifa->ifa_ifp; + if_acquire(*ifp, psref_ifp); + if (info.rti_info[RTAX_IFA] == NULL && + info.rti_info[RTAX_GATEWAY] == NULL) + goto next; + ifa_release(ifa, psref); + if (info.rti_info[RTAX_IFA] == NULL) { + /* route change -ifp */ + ifa = ifaof_ifpforaddr_psref(info.rti_info[RTAX_GATEWAY], + *ifp, psref); + } else { + /* route change -ifp -ifa */ + ifa = ifa_ifwithaddr_psref(info.rti_info[RTAX_IFA], psref); + if (ifa != NULL) + goto out; + ifa = ifaof_ifpforaddr_psref(info.rti_info[RTAX_IFA], + *ifp, psref); + } + goto out; + } +next: + if (info.rti_info[RTAX_IFA] != NULL) { + /* route change -ifa */ + ifa = ifa_ifwithaddr_psref(info.rti_info[RTAX_IFA], psref); + if (ifa != NULL) + goto out; + } + if (info.rti_info[RTAX_GATEWAY] != NULL) { + /* route change */ + ifa = ifa_ifwithroute_psref(rt->rt_flags, rt_getkey(rt), + info.rti_info[RTAX_GATEWAY], psref); + } +out: + if (ifa != NULL && *ifp == NULL) { + *ifp = ifa->ifa_ifp; + if_acquire(*ifp, psref_ifp); + } + if (ifa == NULL && *ifp != NULL) { + if_put(*ifp, psref_ifp); + *ifp = NULL; + } + return ifa; +} + +int +rt_update(struct rtentry *rt, struct rt_addrinfo *info, void *rtm) +{ + int error = 0; + struct ifnet *ifp = NULL, *new_ifp = NULL; + struct ifaddr *ifa = NULL, *new_ifa; + struct psref psref_ifa, psref_new_ifa, psref_ifp, psref_new_ifp; + bool newgw, ifp_changed = false; + + RT_WLOCK(); + /* + * New gateway could require new ifaddr, ifp; + * flags may also be different; ifp may be specified + * by ll sockaddr when protocol address is ambiguous + */ + newgw = info->rti_info[RTAX_GATEWAY] != NULL && + sockaddr_cmp(info->rti_info[RTAX_GATEWAY], rt->rt_gateway) != 0; + + if (newgw || info->rti_info[RTAX_IFP] != NULL || + info->rti_info[RTAX_IFA] != NULL) { + ifp = rt_getifp(info, &psref_ifp); + /* info refers ifp so we need to keep a reference */ + ifa = rt_getifa(info, &psref_ifa); + if (ifa == NULL) { + error = ENETUNREACH; + goto out; + } + } + if (newgw) { + error = rt_setgate(rt, info->rti_info[RTAX_GATEWAY]); + if (error != 0) + goto out; + } + if (info->rti_info[RTAX_TAG]) { + const struct sockaddr *tag; + tag = rt_settag(rt, info->rti_info[RTAX_TAG]); + if (tag == NULL) { + error = ENOBUFS; + goto out; + } + } + /* + * New gateway could require new ifaddr, ifp; + * flags may also be different; ifp may be specified + * by ll sockaddr when protocol address is ambiguous + */ + new_ifa = rt_update_get_ifa(*info, rt, &new_ifp, &psref_new_ifp, + &psref_new_ifa); + if (new_ifa != NULL) { + ifa_release(ifa, &psref_ifa); + ifa = new_ifa; + } + if (ifa) { + struct ifaddr *oifa = rt->rt_ifa; + if (oifa != ifa && !ifa_is_destroying(ifa) && + new_ifp != NULL && !if_is_deactivated(new_ifp)) { + if (oifa && oifa->ifa_rtrequest) + oifa->ifa_rtrequest(RTM_DELETE, rt, info); + rt_replace_ifa(rt, ifa); + rt->rt_ifp = new_ifp; + ifp_changed = true; + } + if (new_ifa == NULL) + ifa_release(ifa, &psref_ifa); + } + ifa_release(new_ifa, &psref_new_ifa); + if (new_ifp && rt->rt_ifp != new_ifp && !if_is_deactivated(new_ifp)) { + rt->rt_ifp = new_ifp; + ifp_changed = true; + } + rt_setmetrics(rtm, rt); + if (rt->rt_flags != info->rti_flags) { + rt->rt_flags = (info->rti_flags & ~PRESERVED_RTF) | + (rt->rt_flags & PRESERVED_RTF); + } + if (rt->rt_ifa && rt->rt_ifa->ifa_rtrequest) + rt->rt_ifa->ifa_rtrequest(RTM_ADD, rt, info); +#if defined(INET) || defined(INET6) + if (ifp_changed && rt_mask(rt) != NULL) + lltable_prefix_free(rt_getkey(rt)->sa_family, rt_getkey(rt), + rt_mask(rt), 0); +#else + (void)ifp_changed; /* XXX gcc */ +#endif +out: + if_put(new_ifp, &psref_new_ifp); + if_put(ifp, &psref_ifp); + + RT_UNLOCK(); + + return error; +} + static void rt_maskedcopy(const struct sockaddr *src, struct sockaddr *dst, const struct sockaddr *netmask) @@ -1582,8 +1696,6 @@ rt_ifa_addlocal(struct ifaddr *ifa) memset(&info, 0, sizeof(info)); info.rti_flags = RTF_HOST | RTF_LOCAL; - if (!(ifa->ifa_ifp->if_flags & (IFF_LOOPBACK|IFF_POINTOPOINT))) - info.rti_flags |= RTF_LLDATA; info.rti_info[RTAX_DST] = ifa->ifa_addr; info.rti_info[RTAX_GATEWAY] = (const struct sockaddr *)ifa->ifa_ifp->if_sadl; @@ -1686,7 +1798,6 @@ rt_timer_init(void) /* XXX should be in rt_init */ rw_init(&rt_lock); - rw_init(&rtcache_lock); LIST_INIT(&rttimer_queue_head); callout_init(&rt_timer_ch, CALLOUT_MPSAFE); @@ -1848,7 +1959,12 @@ rt_timer_work(struct work *wk, void *arg (r->rtt_time + rtq->rtq_timeout) < time_uptime) { LIST_REMOVE(r, rtt_link); TAILQ_REMOVE(&rtq->rtq_head, r, rtt_next); - rt_ref(r->rtt_rt); /* XXX */ + /* + * Take a reference to avoid the rtentry is freed + * accidentally after RT_UNLOCK. The callback + * (rtt_func) must rt_unref it by itself. + */ + rt_ref(r->rtt_rt); RT_REFCNT_TRACE(r->rtt_rt); RT_UNLOCK(); (*r->rtt_func)(r->rtt_rt, r); @@ -1879,20 +1995,20 @@ _rtcache_init(struct route *ro, int flag rtcache_invariants(ro); KASSERT(ro->_ro_rt == NULL); - RTCACHE_ASSERT_WLOCK(); if (rtcache_getdst(ro) == NULL) return NULL; - ro->ro_invalid = false; rt = rtalloc1(rtcache_getdst(ro), flag); - if (rt != NULL && ISSET(rt->rt_flags, RTF_UP)) { - ro->_ro_rt = rt; - KASSERT(!ISSET(rt->rt_flags, RTF_UPDATING)); - rtcache_ref(rt, ro); - rt_unref(rt); - rtcache(ro); - } else if (rt != NULL) + if (rt != NULL) { + RT_RLOCK(); + if (ISSET(rt->rt_flags, RTF_UP)) { + ro->_ro_rt = rt; + ro->ro_rtcache_generation = rtcache_generation; + rtcache_ref(rt, ro); + } + RT_UNLOCK(); rt_unref(rt); + } rtcache_invariants(ro); return ro->_ro_rt; @@ -1901,32 +2017,23 @@ _rtcache_init(struct route *ro, int flag struct rtentry * rtcache_init(struct route *ro) { - struct rtentry *rt; - RTCACHE_WLOCK(); - rt = _rtcache_init(ro, 1); - RTCACHE_UNLOCK(); - return rt; + + return _rtcache_init(ro, 1); } struct rtentry * rtcache_init_noclone(struct route *ro) { - struct rtentry *rt; - RTCACHE_WLOCK(); - rt = _rtcache_init(ro, 0); - RTCACHE_UNLOCK(); - return rt; + + return _rtcache_init(ro, 0); } struct rtentry * rtcache_update(struct route *ro, int clone) { - struct rtentry *rt; - RTCACHE_WLOCK(); - rtcache_clear(ro); - rt = _rtcache_init(ro, clone); - RTCACHE_UNLOCK(); - return rt; + + ro->_ro_rt = NULL; + return _rtcache_init(ro, clone); } void @@ -1947,19 +2054,16 @@ rtcache_copy(struct route *new_ro, struc if (ret != 0) goto out; - RTCACHE_WLOCK(); - new_ro->ro_invalid = false; - if ((new_ro->_ro_rt = rt) != NULL) - rtcache(new_ro); + RT_RLOCK(); + new_ro->_ro_rt = rt; + new_ro->ro_rtcache_generation = rtcache_generation; + RT_UNLOCK(); rtcache_invariants(new_ro); - RTCACHE_UNLOCK(); out: rtcache_unref(rt, old_ro); return; } -static struct dom_rtlist invalid_routes = LIST_HEAD_INITIALIZER(dom_rtlist); - #if defined(RT_DEBUG) && defined(NET_MPSAFE) static void rtcache_trace(const char *func, struct rtentry *rt, struct route *ro) @@ -2002,162 +2106,92 @@ rtcache_unref(struct rtentry *rt, struct #endif } -static struct rtentry * -rtcache_validate_locked(struct route *ro) +struct rtentry * +rtcache_validate(struct route *ro) { struct rtentry *rt = NULL; #ifdef NET_MPSAFE retry: #endif - rt = ro->_ro_rt; rtcache_invariants(ro); - - if (ro->ro_invalid) { + RT_RLOCK(); + if (ro->ro_rtcache_generation != rtcache_generation) { + /* The cache is invalidated */ rt = NULL; goto out; } - RT_RLOCK(); - if (rt != NULL && (rt->rt_flags & RTF_UP) != 0 && rt->rt_ifp != NULL) { + rt = ro->_ro_rt; + if (rt == NULL) + goto out; + + if ((rt->rt_flags & RTF_UP) == 0) { + rt = NULL; + goto out; + } #ifdef NET_MPSAFE - if (ISSET(rt->rt_flags, RTF_UPDATING)) { - if (rt_wait_ok()) { - RT_UNLOCK(); - RTCACHE_UNLOCK(); - /* We can wait until the update is complete */ - rt_update_wait(); - RTCACHE_RLOCK(); - goto retry; - } else { - rt = NULL; - } - } else -#endif - rtcache_ref(rt, ro); + if (ISSET(rt->rt_flags, RTF_UPDATING)) { + if (rt_wait_ok()) { + RT_UNLOCK(); + + /* We can wait until the update is complete */ + rt_update_wait(); + goto retry; + } else { + rt = NULL; + } } else - rt = NULL; - RT_UNLOCK(); +#endif + rtcache_ref(rt, ro); out: + RT_UNLOCK(); return rt; } struct rtentry * -rtcache_validate(struct route *ro) -{ - struct rtentry *rt; - - RTCACHE_RLOCK(); - rt = rtcache_validate_locked(ro); - RTCACHE_UNLOCK(); - return rt; -} - -static void -rtcache_invalidate(struct dom_rtlist *rtlist) -{ - struct route *ro; - - RTCACHE_ASSERT_WLOCK(); - - while ((ro = LIST_FIRST(rtlist)) != NULL) { - rtcache_invariants(ro); - KASSERT(ro->_ro_rt != NULL); - ro->ro_invalid = true; - LIST_REMOVE(ro, ro_rtcache_next); - LIST_INSERT_HEAD(&invalid_routes, ro, ro_rtcache_next); - rtcache_invariants(ro); - } -} - -static void -rtcache_clear_rtentry(int family, struct rtentry *rt) -{ - struct domain *dom; - struct route *ro, *nro; - - if ((dom = pffinddomain(family)) == NULL) - return; - - RTCACHE_WLOCK(); - LIST_FOREACH_SAFE(ro, &dom->dom_rtcache, ro_rtcache_next, nro) { - if (ro->_ro_rt == rt) - rtcache_clear(ro); - } - RTCACHE_UNLOCK(); -} - -static void -rtcache_clear(struct route *ro) -{ - - RTCACHE_ASSERT_WLOCK(); - - rtcache_invariants(ro); - if (ro->_ro_rt == NULL) - return; - - LIST_REMOVE(ro, ro_rtcache_next); - - ro->_ro_rt = NULL; - ro->ro_invalid = false; - rtcache_invariants(ro); -} - -struct rtentry * rtcache_lookup2(struct route *ro, const struct sockaddr *dst, int clone, int *hitp) { const struct sockaddr *odst; struct rtentry *rt = NULL; - RTCACHE_RLOCK(); odst = rtcache_getdst(ro); - if (odst == NULL) { - RTCACHE_UNLOCK(); - RTCACHE_WLOCK(); + if (odst == NULL) goto miss; - } if (sockaddr_cmp(odst, dst) != 0) { - RTCACHE_UNLOCK(); - RTCACHE_WLOCK(); - rtcache_free_locked(ro); + rtcache_free(ro); goto miss; } - rt = rtcache_validate_locked(ro); + rt = rtcache_validate(ro); if (rt == NULL) { - RTCACHE_UNLOCK(); - RTCACHE_WLOCK(); - rtcache_clear(ro); + ro->_ro_rt = NULL; goto miss; } rtcache_invariants(ro); - RTCACHE_UNLOCK(); if (hitp != NULL) *hitp = 1; return rt; miss: if (hitp != NULL) *hitp = 0; - if (rtcache_setdst_locked(ro, dst) == 0) + if (rtcache_setdst(ro, dst) == 0) rt = _rtcache_init(ro, clone); rtcache_invariants(ro); - RTCACHE_UNLOCK(); return rt; } -static void -rtcache_free_locked(struct route *ro) +void +rtcache_free(struct route *ro) { - RTCACHE_ASSERT_WLOCK(); - rtcache_clear(ro); + ro->_ro_rt = NULL; if (ro->ro_sa != NULL) { sockaddr_free(ro->ro_sa); ro->ro_sa = NULL; @@ -2165,32 +2199,21 @@ rtcache_free_locked(struct route *ro) rtcache_invariants(ro); } -void -rtcache_free(struct route *ro) -{ - - RTCACHE_WLOCK(); - rtcache_free_locked(ro); - RTCACHE_UNLOCK(); -} - -static int -rtcache_setdst_locked(struct route *ro, const struct sockaddr *sa) +int +rtcache_setdst(struct route *ro, const struct sockaddr *sa) { KASSERT(sa != NULL); - RTCACHE_ASSERT_WLOCK(); - rtcache_invariants(ro); if (ro->ro_sa != NULL) { if (ro->ro_sa->sa_family == sa->sa_family) { - rtcache_clear(ro); + ro->_ro_rt = NULL; sockaddr_copy(ro->ro_sa, ro->ro_sa->sa_len, sa); rtcache_invariants(ro); return 0; } /* free ro_sa, wrong family */ - rtcache_free_locked(ro); + rtcache_free(ro); } KASSERT(ro->_ro_rt == NULL); @@ -2203,18 +2226,6 @@ rtcache_setdst_locked(struct route *ro, return 0; } -int -rtcache_setdst(struct route *ro, const struct sockaddr *sa) -{ - int error; - - RTCACHE_WLOCK(); - error = rtcache_setdst_locked(ro, sa); - RTCACHE_UNLOCK(); - - return error; -} - const struct sockaddr * rt_settag(struct rtentry *rt, const struct sockaddr *tag) { @@ -2268,7 +2279,7 @@ rt_delete_matched_entries(sa_family_t fa RT_UNLOCK(); return; } - rt->rt_refcnt++; + rt_ref(rt); splx(s); RT_UNLOCK();