summaryrefslogtreecommitdiffstats
path: root/net/ipv4/route.c
diff options
context:
space:
mode:
authorEric Dumazet <edumazet@google.com>2012-07-31 05:45:30 +0000
committerDavid S. Miller <davem@davemloft.net>2012-07-31 14:41:39 -0700
commitd26b3a7c4b3b26319f18bb645de93eba8f4bdcd5 (patch)
treeca86c03450fafdc89dac98ce403b1906fcaa025d /net/ipv4/route.c
parent54764bb647b2e847c512acf8d443df965da35000 (diff)
downloadop-kernel-dev-d26b3a7c4b3b26319f18bb645de93eba8f4bdcd5.zip
op-kernel-dev-d26b3a7c4b3b26319f18bb645de93eba8f4bdcd5.tar.gz
ipv4: percpu nh_rth_output cache
Input path is mostly run under RCU and doesnt touch dst refcnt But output path on forwarding or UDP workloads hits badly dst refcount, and we have lot of false sharing, for example in ipv4_mtu() when reading rt->rt_pmtu Using a percpu cache for nh_rth_output gives a nice performance increase at a small cost. 24 udpflood test on my 24 cpu machine (dummy0 output device) (each process sends 1.000.000 udp frames, 24 processes are started) before : 5.24 s after : 2.06 s For reference, time on linux-3.5 : 6.60 s Signed-off-by: Eric Dumazet <edumazet@google.com> Tested-by: Alexander Duyck <alexander.h.duyck@intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/ipv4/route.c')
-rw-r--r--net/ipv4/route.c18
1 files changed, 13 insertions, 5 deletions
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 2bd1074..4f6276c 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -1206,11 +1206,15 @@ static inline void rt_free(struct rtable *rt)
static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
{
- struct rtable *orig, *prev, **p = (struct rtable **)&nh->nh_rth_output;
+ struct rtable *orig, *prev, **p;
- if (rt_is_input_route(rt))
+ if (rt_is_input_route(rt)) {
p = (struct rtable **)&nh->nh_rth_input;
-
+ } else {
+ if (!nh->nh_pcpu_rth_output)
+ goto nocache;
+ p = (struct rtable **)__this_cpu_ptr(nh->nh_pcpu_rth_output);
+ }
orig = *p;
prev = cmpxchg(p, orig, rt);
@@ -1223,6 +1227,7 @@ static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
* unsuccessful at storing this route into the cache
* we really need to set it.
*/
+nocache:
rt->dst.flags |= DST_NOCACHE;
}
}
@@ -1749,8 +1754,11 @@ static struct rtable *__mkroute_output(const struct fib_result *res,
fnhe = NULL;
if (fi) {
fnhe = find_exception(&FIB_RES_NH(*res), fl4->daddr);
- if (!fnhe) {
- rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_output);
+ if (!fnhe && FIB_RES_NH(*res).nh_pcpu_rth_output) {
+ struct rtable __rcu **prth;
+
+ prth = __this_cpu_ptr(FIB_RES_NH(*res).nh_pcpu_rth_output);
+ rth = rcu_dereference(*prth);
if (rt_cache_valid(rth)) {
dst_hold(&rth->dst);
return rth;
OpenPOWER on IntegriCloud