diff options
Diffstat (limited to 'include/net')
| -rw-r--r-- | include/net/af_unix.h | 3 | ||||
| -rw-r--r-- | include/net/inet_connection_sock.h | 2 | ||||
| -rw-r--r-- | include/net/inetpeer.h | 87 | ||||
| -rw-r--r-- | include/net/ip6_fib.h | 33 | ||||
| -rw-r--r-- | include/net/ip6_route.h | 21 | ||||
| -rw-r--r-- | include/net/ip_fib.h | 12 | ||||
| -rw-r--r-- | include/net/netfilter/nf_conntrack_core.h | 4 | ||||
| -rw-r--r-- | include/net/netfilter/nf_conntrack_l3proto.h | 11 | ||||
| -rw-r--r-- | include/net/netfilter/nf_conntrack_l4proto.h | 22 | ||||
| -rw-r--r-- | include/net/netns/conntrack.h | 55 | ||||
| -rw-r--r-- | include/net/netns/ipv4.h | 2 | ||||
| -rw-r--r-- | include/net/netns/ipv6.h | 1 | ||||
| -rw-r--r-- | include/net/route.h | 56 | ||||
| -rw-r--r-- | include/net/tcp.h | 3 | ||||
| -rw-r--r-- | include/net/timewait_sock.h | 8 |
15 files changed, 263 insertions, 57 deletions
diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 2ee33da..b5f8988 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -14,10 +14,11 @@ extern struct sock *unix_get_socket(struct file *filp); extern struct sock *unix_peer_get(struct sock *); #define UNIX_HASH_SIZE 256 +#define UNIX_HASH_BITS 8 extern unsigned int unix_tot_inflight; extern spinlock_t unix_table_lock; -extern struct hlist_head unix_socket_table[UNIX_HASH_SIZE + 1]; +extern struct hlist_head unix_socket_table[2 * UNIX_HASH_SIZE]; struct unix_address { atomic_t refcnt; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 7d83f90..e1b7734 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -43,7 +43,7 @@ struct inet_connection_sock_af_ops { struct sock *(*syn_recv_sock)(struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst); - struct inet_peer *(*get_peer)(struct sock *sk, bool *release_it); + struct inet_peer *(*get_peer)(struct sock *sk); u16 net_header_len; u16 net_frag_header_len; u16 sockaddr_len; diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h index b94765e..c27c8f1 100644 --- a/include/net/inetpeer.h +++ b/include/net/inetpeer.h @@ -40,7 +40,10 @@ struct inet_peer { u32 pmtu_orig; u32 pmtu_learned; struct inetpeer_addr_base redirect_learned; - struct list_head gc_list; + union { + struct list_head gc_list; + struct rcu_head gc_rcu; + }; /* * Once inet_peer is queued for deletion (refcnt == -1), following fields * are not available: rid, ip_id_count, tcp_ts, tcp_ts_stamp @@ -62,6 +65,69 @@ struct inet_peer { atomic_t refcnt; }; +struct inet_peer_base { + struct inet_peer __rcu *root; + seqlock_t lock; + u32 flush_seq; + int total; +}; + +#define INETPEER_BASE_BIT 0x1UL + +static inline struct inet_peer *inetpeer_ptr(unsigned long val) +{ + BUG_ON(val & INETPEER_BASE_BIT); + return (struct inet_peer *) val; +} + +static inline struct inet_peer_base *inetpeer_base_ptr(unsigned long val) +{ + if (!(val & INETPEER_BASE_BIT)) + return NULL; + val &= ~INETPEER_BASE_BIT; + return (struct inet_peer_base *) val; +} + +static inline bool inetpeer_ptr_is_peer(unsigned long val) +{ + return !(val & INETPEER_BASE_BIT); +} + +static inline void __inetpeer_ptr_set_peer(unsigned long *val, struct inet_peer *peer) +{ + /* This implicitly clears INETPEER_BASE_BIT */ + *val = (unsigned long) peer; +} + +static inline bool inetpeer_ptr_set_peer(unsigned long *ptr, struct inet_peer *peer) +{ + unsigned long val = (unsigned long) peer; + unsigned long orig = *ptr; + + if (!(orig & INETPEER_BASE_BIT) || + cmpxchg(ptr, orig, val) != orig) + return false; + return true; +} + +static inline void inetpeer_init_ptr(unsigned long *ptr, struct inet_peer_base *base) +{ + *ptr = (unsigned long) base | INETPEER_BASE_BIT; +} + +static inline void inetpeer_transfer_peer(unsigned long *to, unsigned long *from) +{ + unsigned long val = *from; + + *to = val; + if (inetpeer_ptr_is_peer(val)) { + struct inet_peer *peer = inetpeer_ptr(val); + atomic_inc(&peer->refcnt); + } +} + +extern void inet_peer_base_init(struct inet_peer_base *); + void inet_initpeers(void) __init; #define INETPEER_METRICS_NEW (~(u32) 0) @@ -72,31 +138,38 @@ static inline bool inet_metrics_new(const struct inet_peer *p) } /* can be called with or without local BH being disabled */ -struct inet_peer *inet_getpeer(const struct inetpeer_addr *daddr, int create); +struct inet_peer *inet_getpeer(struct inet_peer_base *base, + const struct inetpeer_addr *daddr, + int create); -static inline struct inet_peer *inet_getpeer_v4(__be32 v4daddr, int create) +static inline struct inet_peer *inet_getpeer_v4(struct inet_peer_base *base, + __be32 v4daddr, + int create) { struct inetpeer_addr daddr; daddr.addr.a4 = v4daddr; daddr.family = AF_INET; - return inet_getpeer(&daddr, create); + return inet_getpeer(base, &daddr, create); } -static inline struct inet_peer *inet_getpeer_v6(const struct in6_addr *v6daddr, int create) +static inline struct inet_peer *inet_getpeer_v6(struct inet_peer_base *base, + const struct in6_addr *v6daddr, + int create) { struct inetpeer_addr daddr; *(struct in6_addr *)daddr.addr.a6 = *v6daddr; daddr.family = AF_INET6; - return inet_getpeer(&daddr, create); + return inet_getpeer(base, &daddr, create); } /* can be called from BH context or outside */ extern void inet_putpeer(struct inet_peer *p); extern bool inet_peer_xrlim_allow(struct inet_peer *peer, int timeout); -extern void inetpeer_invalidate_tree(int family); +extern void inetpeer_invalidate_tree(struct inet_peer_base *); +extern void inetpeer_invalidate_family(int family); /* * temporary check to make sure we dont access rid, ip_id_count, tcp_ts, diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index 0ae759a..a192f78 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -107,7 +107,7 @@ struct rt6_info { u32 rt6i_peer_genid; struct inet6_dev *rt6i_idev; - struct inet_peer *rt6i_peer; + unsigned long _rt6i_peer; #ifdef CONFIG_XFRM u32 rt6i_flow_cache_genid; @@ -118,6 +118,36 @@ struct rt6_info { u8 rt6i_protocol; }; +static inline struct inet_peer *rt6_peer_ptr(struct rt6_info *rt) +{ + return inetpeer_ptr(rt->_rt6i_peer); +} + +static inline bool rt6_has_peer(struct rt6_info *rt) +{ + return inetpeer_ptr_is_peer(rt->_rt6i_peer); +} + +static inline void __rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer) +{ + __inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer); +} + +static inline bool rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer) +{ + return inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer); +} + +static inline void rt6_init_peer(struct rt6_info *rt, struct inet_peer_base *base) +{ + inetpeer_init_ptr(&rt->_rt6i_peer, base); +} + +static inline void rt6_transfer_peer(struct rt6_info *rt, struct rt6_info *ort) +{ + inetpeer_transfer_peer(&rt->_rt6i_peer, &ort->_rt6i_peer); +} + static inline struct inet6_dev *ip6_dst_idev(struct dst_entry *dst) { return ((struct rt6_info *)dst)->rt6i_idev; @@ -207,6 +237,7 @@ struct fib6_table { u32 tb6_id; rwlock_t tb6_lock; struct fib6_node tb6_root; + struct inet_peer_base tb6_peers; }; #define RT6_TABLE_UNSPEC RT_TABLE_UNSPEC diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 37c1a1e..a2cda24 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -53,16 +53,25 @@ static inline unsigned int rt6_flags2srcprefs(int flags) return (flags >> 3) & 7; } -extern void rt6_bind_peer(struct rt6_info *rt, - int create); +extern void rt6_bind_peer(struct rt6_info *rt, int create); + +static inline struct inet_peer *__rt6_get_peer(struct rt6_info *rt, int create) +{ + if (rt6_has_peer(rt)) + return rt6_peer_ptr(rt); + + rt6_bind_peer(rt, create); + return (rt6_has_peer(rt) ? rt6_peer_ptr(rt) : NULL); +} static inline struct inet_peer *rt6_get_peer(struct rt6_info *rt) { - if (rt->rt6i_peer) - return rt->rt6i_peer; + return __rt6_get_peer(rt, 0); +} - rt6_bind_peer(rt, 0); - return rt->rt6i_peer; +static inline struct inet_peer *rt6_get_peer_create(struct rt6_info *rt) +{ + return __rt6_get_peer(rt, 1); } extern void ip6_route_input(struct sk_buff *skb); diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index 78df0866..4b347c0 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -19,6 +19,7 @@ #include <net/flow.h> #include <linux/seq_file.h> #include <net/fib_rules.h> +#include <net/inetpeer.h> struct fib_config { u8 fc_dst_len; @@ -157,11 +158,12 @@ extern __be32 fib_info_update_nh_saddr(struct net *net, struct fib_nh *nh); FIB_RES_SADDR(net, res)) struct fib_table { - struct hlist_node tb_hlist; - u32 tb_id; - int tb_default; - int tb_num_default; - unsigned long tb_data[0]; + struct hlist_node tb_hlist; + u32 tb_id; + int tb_default; + int tb_num_default; + struct inet_peer_base tb_peers; + unsigned long tb_data[0]; }; extern int fib_table_lookup(struct fib_table *tb, const struct flowi4 *flp, diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index aced085..d8f5b9f 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -28,8 +28,8 @@ extern unsigned int nf_conntrack_in(struct net *net, extern int nf_conntrack_init(struct net *net); extern void nf_conntrack_cleanup(struct net *net); -extern int nf_conntrack_proto_init(void); -extern void nf_conntrack_proto_fini(void); +extern int nf_conntrack_proto_init(struct net *net); +extern void nf_conntrack_proto_fini(struct net *net); extern bool nf_ct_get_tuple(const struct sk_buff *skb, diff --git a/include/net/netfilter/nf_conntrack_l3proto.h b/include/net/netfilter/nf_conntrack_l3proto.h index 9699c02..6f7c13f 100644 --- a/include/net/netfilter/nf_conntrack_l3proto.h +++ b/include/net/netfilter/nf_conntrack_l3proto.h @@ -64,11 +64,12 @@ struct nf_conntrack_l3proto { size_t nla_size; #ifdef CONFIG_SYSCTL - struct ctl_table_header *ctl_table_header; const char *ctl_table_path; - struct ctl_table *ctl_table; #endif /* CONFIG_SYSCTL */ + /* Init l3proto pernet data */ + int (*init_net)(struct net *net); + /* Module (if any) which this is connected to. */ struct module *me; }; @@ -76,8 +77,10 @@ struct nf_conntrack_l3proto { extern struct nf_conntrack_l3proto __rcu *nf_ct_l3protos[AF_MAX]; /* Protocol registration. */ -extern int nf_conntrack_l3proto_register(struct nf_conntrack_l3proto *proto); -extern void nf_conntrack_l3proto_unregister(struct nf_conntrack_l3proto *proto); +extern int nf_conntrack_l3proto_register(struct net *net, + struct nf_conntrack_l3proto *proto); +extern void nf_conntrack_l3proto_unregister(struct net *net, + struct nf_conntrack_l3proto *proto); extern struct nf_conntrack_l3proto *nf_ct_l3proto_find_get(u_int16_t l3proto); extern void nf_ct_l3proto_put(struct nf_conntrack_l3proto *p); diff --git a/include/net/netfilter/nf_conntrack_l4proto.h b/include/net/netfilter/nf_conntrack_l4proto.h index 3b572bb..81c52b5 100644 --- a/include/net/netfilter/nf_conntrack_l4proto.h +++ b/include/net/netfilter/nf_conntrack_l4proto.h @@ -12,6 +12,7 @@ #include <linux/netlink.h> #include <net/netlink.h> #include <net/netfilter/nf_conntrack.h> +#include <net/netns/generic.h> struct seq_file; @@ -86,23 +87,18 @@ struct nf_conntrack_l4proto { #if IS_ENABLED(CONFIG_NF_CT_NETLINK_TIMEOUT) struct { size_t obj_size; - int (*nlattr_to_obj)(struct nlattr *tb[], void *data); + int (*nlattr_to_obj)(struct nlattr *tb[], + struct net *net, void *data); int (*obj_to_nlattr)(struct sk_buff *skb, const void *data); unsigned int nlattr_max; const struct nla_policy *nla_policy; } ctnl_timeout; #endif + int *net_id; + /* Init l4proto pernet data */ + int (*init_net)(struct net *net); -#ifdef CONFIG_SYSCTL - struct ctl_table_header **ctl_table_header; - struct ctl_table *ctl_table; - unsigned int *ctl_table_users; -#ifdef CONFIG_NF_CONNTRACK_PROC_COMPAT - struct ctl_table_header *ctl_compat_table_header; - struct ctl_table *ctl_compat_table; -#endif -#endif /* Protocol name */ const char *name; @@ -123,8 +119,10 @@ nf_ct_l4proto_find_get(u_int16_t l3proto, u_int8_t l4proto); extern void nf_ct_l4proto_put(struct nf_conntrack_l4proto *p); /* Protocol registration. */ -extern int nf_conntrack_l4proto_register(struct nf_conntrack_l4proto *proto); -extern void nf_conntrack_l4proto_unregister(struct nf_conntrack_l4proto *proto); +extern int nf_conntrack_l4proto_register(struct net *net, + struct nf_conntrack_l4proto *proto); +extern void nf_conntrack_l4proto_unregister(struct net *net, + struct nf_conntrack_l4proto *proto); /* Generic netlink helpers */ extern int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb, diff --git a/include/net/netns/conntrack.h b/include/net/netns/conntrack.h index a053a19..3aecdc7 100644 --- a/include/net/netns/conntrack.h +++ b/include/net/netns/conntrack.h @@ -4,10 +4,64 @@ #include <linux/list.h> #include <linux/list_nulls.h> #include <linux/atomic.h> +#include <linux/netfilter/nf_conntrack_tcp.h> struct ctl_table_header; struct nf_conntrack_ecache; +struct nf_proto_net { +#ifdef CONFIG_SYSCTL + struct ctl_table_header *ctl_table_header; + struct ctl_table *ctl_table; +#ifdef CONFIG_NF_CONNTRACK_PROC_COMPAT + struct ctl_table_header *ctl_compat_header; + struct ctl_table *ctl_compat_table; +#endif +#endif + unsigned int users; +}; + +struct nf_generic_net { + struct nf_proto_net pn; + unsigned int timeout; +}; + +struct nf_tcp_net { + struct nf_proto_net pn; + unsigned int timeouts[TCP_CONNTRACK_TIMEOUT_MAX]; + unsigned int tcp_loose; + unsigned int tcp_be_liberal; + unsigned int tcp_max_retrans; +}; + +enum udp_conntrack { + UDP_CT_UNREPLIED, + UDP_CT_REPLIED, + UDP_CT_MAX +}; + +struct nf_udp_net { + struct nf_proto_net pn; + unsigned int timeouts[UDP_CT_MAX]; +}; + +struct nf_icmp_net { + struct nf_proto_net pn; + unsigned int timeout; +}; + +struct nf_ip_net { + struct nf_generic_net generic; + struct nf_tcp_net tcp; + struct nf_udp_net udp; + struct nf_icmp_net icmp; + struct nf_icmp_net icmpv6; +#if defined(CONFIG_SYSCTL) && defined(CONFIG_NF_CONNTRACK_PROC_COMPAT) + struct ctl_table_header *ctl_table_header; + struct ctl_table *ctl_table; +#endif +}; + struct netns_ct { atomic_t count; unsigned int expect_count; @@ -28,6 +82,7 @@ struct netns_ct { unsigned int sysctl_log_invalid; /* Log invalid packets */ int sysctl_auto_assign_helper; bool auto_assign_helper_warned; + struct nf_ip_net nf_ct_proto; #ifdef CONFIG_SYSCTL struct ctl_table_header *sysctl_header; struct ctl_table_header *acct_sysctl_header; diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index bbd023a..227f0cd 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -30,7 +30,7 @@ struct netns_ipv4 { struct sock **icmp_sk; struct sock *tcp_sock; - + struct inet_peer_base *peers; struct netns_frags frags; #ifdef CONFIG_NETFILTER struct xt_table *iptable_filter; diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h index b42be53..df0a545 100644 --- a/include/net/netns/ipv6.h +++ b/include/net/netns/ipv6.h @@ -33,6 +33,7 @@ struct netns_ipv6 { struct netns_sysctl_ipv6 sysctl; struct ipv6_devconf *devconf_all; struct ipv6_devconf *devconf_dflt; + struct inet_peer_base *peers; struct netns_frags frags; #ifdef CONFIG_NETFILTER struct xt_table *ip6table_filter; diff --git a/include/net/route.h b/include/net/route.h index ed2b78e..2bfbc93 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -67,10 +67,44 @@ struct rtable { /* Miscellaneous cached information */ __be32 rt_spec_dst; /* RFC1122 specific destination */ u32 rt_peer_genid; - struct inet_peer *peer; /* long-living peer info */ + unsigned long _peer; /* long-living peer info */ struct fib_info *fi; /* for client ref to shared metrics */ }; +static inline struct inet_peer *rt_peer_ptr(struct rtable *rt) +{ + return inetpeer_ptr(rt->_peer); +} + +static inline bool rt_has_peer(struct rtable *rt) +{ + return inetpeer_ptr_is_peer(rt->_peer); +} + +static inline void __rt_set_peer(struct rtable *rt, struct inet_peer *peer) +{ + __inetpeer_ptr_set_peer(&rt->_peer, peer); +} + +static inline bool rt_set_peer(struct rtable *rt, struct inet_peer *peer) +{ + return inetpeer_ptr_set_peer(&rt->_peer, peer); +} + +static inline void rt_init_peer(struct rtable *rt, struct inet_peer_base *base) +{ + inetpeer_init_ptr(&rt->_peer, base); +} + +static inline void rt_transfer_peer(struct rtable *rt, struct rtable *ort) +{ + rt->_peer = ort->_peer; + if (rt_has_peer(ort)) { + struct inet_peer *peer = rt_peer_ptr(ort); + atomic_inc(&peer->refcnt); + } +} + static inline bool rt_is_input_route(const struct rtable *rt) { return rt->rt_route_iif != 0; @@ -181,8 +215,6 @@ static inline int ip_route_input_noref(struct sk_buff *skb, __be32 dst, __be32 s return ip_route_input_common(skb, dst, src, tos, devin, true); } -extern unsigned short ip_rt_frag_needed(struct net *net, const struct iphdr *iph, - unsigned short new_mtu, struct net_device *dev); extern void ip_rt_send_redirect(struct sk_buff *skb); extern unsigned int inet_addr_type(struct net *net, __be32 addr); @@ -296,13 +328,23 @@ static inline struct rtable *ip_route_newports(struct flowi4 *fl4, struct rtable extern void rt_bind_peer(struct rtable *rt, __be32 daddr, int create); +static inline struct inet_peer *__rt_get_peer(struct rtable *rt, __be32 daddr, int create) +{ + if (rt_has_peer(rt)) + return rt_peer_ptr(rt); + + rt_bind_peer(rt, daddr, create); + return (rt_has_peer(rt) ? rt_peer_ptr(rt) : NULL); +} + static inline struct inet_peer *rt_get_peer(struct rtable *rt, __be32 daddr) { - if (rt->peer) - return rt->peer; + return __rt_get_peer(rt, daddr, 0); +} - rt_bind_peer(rt, daddr, 0); - return rt->peer; +static inline struct inet_peer *rt_get_peer_create(struct rtable *rt, __be32 daddr) +{ + return __rt_get_peer(rt, daddr, 1); } static inline int inet_iif(const struct sk_buff *skb) diff --git a/include/net/tcp.h b/include/net/tcp.h index e79aa48..9332f34 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -327,8 +327,7 @@ extern void tcp_shutdown (struct sock *sk, int how); extern int tcp_v4_rcv(struct sk_buff *skb); -extern struct inet_peer *tcp_v4_get_peer(struct sock *sk, bool *release_it); -extern void *tcp_v4_tw_get_peer(struct sock *sk); +extern struct inet_peer *tcp_v4_get_peer(struct sock *sk); extern int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); extern int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t size); diff --git a/include/net/timewait_sock.h b/include/net/timewait_sock.h index 8d6689c..68f0eca 100644 --- a/include/net/timewait_sock.h +++ b/include/net/timewait_sock.h @@ -22,7 +22,6 @@ struct timewait_sock_ops { int (*twsk_unique)(struct sock *sk, struct sock *sktw, void *twp); void (*twsk_destructor)(struct sock *sk); - void *(*twsk_getpeer)(struct sock *sk); }; static inline int twsk_unique(struct sock *sk, struct sock *sktw, void *twp) @@ -41,11 +40,4 @@ static inline void twsk_destructor(struct sock *sk) sk->sk_prot->twsk_prot->twsk_destructor(sk); } -static inline void *twsk_getpeer(struct sock *sk) -{ - if (sk->sk_prot->twsk_prot->twsk_getpeer) - return sk->sk_prot->twsk_prot->twsk_getpeer(sk); - return NULL; -} - #endif /* _TIMEWAIT_SOCK_H */ |
