Commit 8e5eb54d authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: align sk_refcnt on 128 bytes boundary

sk->sk_refcnt is dirtied for every TCP/UDP incoming packet.
This is a performance issue if multiple cpus hit a common socket,
or multiple sockets are chained due to SO_REUSEPORT.

By moving sk_refcnt 8 bytes further, first 128 bytes of sockets
are mostly read. As they contain the lookup keys, this has
a considerable performance impact, as cpus can cache them.

These 8 bytes are not wasted, we use them as a place holder
for various fields, depending on the socket type.

Tested:
 SYN flood hitting a 16 RX queues NIC.
 TCP listener using 16 sockets and SO_REUSEPORT
 and SO_INCOMING_CPU for proper siloing.

 Could process 6.0 Mpps SYN instead of 4.2 Mpps

 Kernel profile looked like :
    11.68%  [kernel]  [k] sha_transform
     6.51%  [kernel]  [k] __inet_lookup_listener
     5.07%  [kernel]  [k] __inet_lookup_established
     4.15%  [kernel]  [k] memcpy_erms
     3.46%  [kernel]  [k] ipt_do_table
     2.74%  [kernel]  [k] fib_table_lookup
     2.54%  [kernel]  [k] tcp_make_synack
     2.34%  [kernel]  [k] tcp_conn_request
     2.05%  [kernel]  [k] __netif_receive_skb_core
     2.03%  [kernel]  [k] kmem_cache_alloc
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 70da268b
...@@ -70,6 +70,7 @@ struct inet_timewait_sock { ...@@ -70,6 +70,7 @@ struct inet_timewait_sock {
#define tw_dport __tw_common.skc_dport #define tw_dport __tw_common.skc_dport
#define tw_num __tw_common.skc_num #define tw_num __tw_common.skc_num
#define tw_cookie __tw_common.skc_cookie #define tw_cookie __tw_common.skc_cookie
#define tw_dr __tw_common.skc_tw_dr
int tw_timeout; int tw_timeout;
volatile unsigned char tw_substate; volatile unsigned char tw_substate;
...@@ -88,7 +89,6 @@ struct inet_timewait_sock { ...@@ -88,7 +89,6 @@ struct inet_timewait_sock {
kmemcheck_bitfield_end(flags); kmemcheck_bitfield_end(flags);
struct timer_list tw_timer; struct timer_list tw_timer;
struct inet_bind_bucket *tw_tb; struct inet_bind_bucket *tw_tb;
struct inet_timewait_death_row *tw_dr;
}; };
#define tw_tclass tw_tos #define tw_tclass tw_tos
......
...@@ -50,9 +50,9 @@ struct request_sock { ...@@ -50,9 +50,9 @@ struct request_sock {
struct sock_common __req_common; struct sock_common __req_common;
#define rsk_refcnt __req_common.skc_refcnt #define rsk_refcnt __req_common.skc_refcnt
#define rsk_hash __req_common.skc_hash #define rsk_hash __req_common.skc_hash
#define rsk_listener __req_common.skc_listener
struct request_sock *dl_next; struct request_sock *dl_next;
struct sock *rsk_listener;
u16 mss; u16 mss;
u8 num_retrans; /* number of retransmits */ u8 num_retrans; /* number of retransmits */
u8 cookie_ts:1; /* syncookie: encode tcpopts in timestamp */ u8 cookie_ts:1; /* syncookie: encode tcpopts in timestamp */
......
...@@ -150,6 +150,9 @@ typedef __u64 __bitwise __addrpair; ...@@ -150,6 +150,9 @@ typedef __u64 __bitwise __addrpair;
* @skc_node: main hash linkage for various protocol lookup tables * @skc_node: main hash linkage for various protocol lookup tables
* @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol * @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
* @skc_tx_queue_mapping: tx queue number for this connection * @skc_tx_queue_mapping: tx queue number for this connection
* @skc_flags: place holder for sk_flags
* %SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
* %SO_OOBINLINE settings, %SO_TIMESTAMPING settings
* @skc_incoming_cpu: record/match cpu processing incoming packets * @skc_incoming_cpu: record/match cpu processing incoming packets
* @skc_refcnt: reference count * @skc_refcnt: reference count
* *
...@@ -201,6 +204,16 @@ struct sock_common { ...@@ -201,6 +204,16 @@ struct sock_common {
atomic64_t skc_cookie; atomic64_t skc_cookie;
/* following fields are padding to force
* offset(struct sock, sk_refcnt) == 128 on 64bit arches
* assuming IPV6 is enabled. We use this padding differently
* for different kind of 'sockets'
*/
union {
unsigned long skc_flags;
struct sock *skc_listener; /* request_sock */
struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
};
/* /*
* fields between dontcopy_begin/dontcopy_end * fields between dontcopy_begin/dontcopy_end
* are not copied in sock_copy() * are not copied in sock_copy()
...@@ -246,8 +259,6 @@ struct cg_proto; ...@@ -246,8 +259,6 @@ struct cg_proto;
* @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler) * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler)
* @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE) * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE)
* @sk_sndbuf: size of send buffer in bytes * @sk_sndbuf: size of send buffer in bytes
* @sk_flags: %SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
* %SO_OOBINLINE settings, %SO_TIMESTAMPING settings
* @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets * @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets
* @sk_no_check_rx: allow zero checksum in RX packets * @sk_no_check_rx: allow zero checksum in RX packets
* @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO) * @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
...@@ -334,6 +345,7 @@ struct sock { ...@@ -334,6 +345,7 @@ struct sock {
#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr #define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
#define sk_cookie __sk_common.skc_cookie #define sk_cookie __sk_common.skc_cookie
#define sk_incoming_cpu __sk_common.skc_incoming_cpu #define sk_incoming_cpu __sk_common.skc_incoming_cpu
#define sk_flags __sk_common.skc_flags
socket_lock_t sk_lock; socket_lock_t sk_lock;
struct sk_buff_head sk_receive_queue; struct sk_buff_head sk_receive_queue;
...@@ -371,7 +383,6 @@ struct sock { ...@@ -371,7 +383,6 @@ struct sock {
#ifdef CONFIG_XFRM #ifdef CONFIG_XFRM
struct xfrm_policy *sk_policy[2]; struct xfrm_policy *sk_policy[2];
#endif #endif
unsigned long sk_flags;
struct dst_entry *sk_rx_dst; struct dst_entry *sk_rx_dst;
struct dst_entry __rcu *sk_dst_cache; struct dst_entry __rcu *sk_dst_cache;
spinlock_t sk_dst_lock; spinlock_t sk_dst_lock;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment