Commit efe4208f authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

ipv6: make lookups simpler and faster

TCP listener refactoring, part 4 :

To speed up inet lookups, we moved IPv4 addresses from inet to struct
sock_common

Now is time to do the same for IPv6, because it permits us to have fast
lookups for all kind of sockets, including upcoming SYN_RECV.

Getting IPv6 addresses in TCP lookups currently requires two extra cache
lines, plus a dereference (and memory stall).

inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6

This patch is way bigger than its IPv4 counter part, because for IPv4,
we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6,
it's not doable easily.

inet6_sk(sk)->daddr becomes sk->sk_v6_daddr
inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr

And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr
at the same offset.

We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic
macro.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 05dbc7b5
......@@ -141,8 +141,6 @@ struct ipv6_fl_socklist;
*/
struct ipv6_pinfo {
struct in6_addr saddr;
struct in6_addr rcv_saddr;
struct in6_addr daddr;
struct in6_pktinfo sticky_pktinfo;
const struct in6_addr *daddr_cache;
#ifdef CONFIG_IPV6_SUBTREES
......@@ -256,22 +254,10 @@ struct tcp6_sock {
extern int inet6_sk_rebuild_header(struct sock *sk);
struct inet6_timewait_sock {
struct in6_addr tw_v6_daddr;
struct in6_addr tw_v6_rcv_saddr;
};
struct tcp6_timewait_sock {
struct tcp_timewait_sock tcp6tw_tcp;
struct inet6_timewait_sock tcp6tw_inet6;
};
static inline struct inet6_timewait_sock *inet6_twsk(const struct sock *sk)
{
return (struct inet6_timewait_sock *)(((u8 *)sk) +
inet_twsk(sk)->tw_ipv6_offset);
}
#if IS_ENABLED(CONFIG_IPV6)
static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk)
{
......@@ -321,21 +307,11 @@ static inline void inet_sk_copy_descendant(struct sock *sk_to,
#define __ipv6_only_sock(sk) (inet6_sk(sk)->ipv6only)
#define ipv6_only_sock(sk) ((sk)->sk_family == PF_INET6 && __ipv6_only_sock(sk))
static inline u16 inet6_tw_offset(const struct proto *prot)
{
return prot->twsk_prot->twsk_obj_size -
sizeof(struct inet6_timewait_sock);
}
static inline struct in6_addr *__inet6_rcv_saddr(const struct sock *sk)
static inline const struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
{
return likely(sk->sk_state != TCP_TIME_WAIT) ?
&inet6_sk(sk)->rcv_saddr : &inet6_twsk(sk)->tw_v6_rcv_saddr;
}
static inline struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
{
return sk->sk_family == AF_INET6 ? __inet6_rcv_saddr(sk) : NULL;
if (sk->sk_family == AF_INET6)
return &sk->sk_v6_rcv_saddr;
return NULL;
}
static inline int inet_v6_ipv6only(const struct sock *sk)
......@@ -363,7 +339,6 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk)
return NULL;
}
#define __inet6_rcv_saddr(__sk) NULL
#define inet6_rcv_saddr(__sk) NULL
#define tcp_twsk_ipv6only(__sk) 0
#define inet_v6_ipv6only(__sk) 0
......@@ -372,19 +347,10 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk)
#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
(((__sk)->sk_portpair == (__ports)) && \
((__sk)->sk_family == AF_INET6) && \
ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr)) && \
ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \
ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr)) && \
ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr)) && \
(!(__sk)->sk_bound_dev_if || \
((__sk)->sk_bound_dev_if == (__dif))) && \
net_eq(sock_net(__sk), (__net)))
#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
(((__sk)->sk_portpair == (__ports)) && \
((__sk)->sk_family == AF_INET6) && \
ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr)) && \
ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \
(!(__sk)->sk_bound_dev_if || \
((__sk)->sk_bound_dev_if == (__dif))) && \
net_eq(sock_net(__sk), (__net)))
#endif /* _IPV6_H */
......@@ -43,9 +43,8 @@ static inline unsigned int inet6_ehashfn(struct net *net,
static inline int inet6_sk_ehashfn(const struct sock *sk)
{
const struct inet_sock *inet = inet_sk(sk);
const struct ipv6_pinfo *np = inet6_sk(sk);
const struct in6_addr *laddr = &np->rcv_saddr;
const struct in6_addr *faddr = &np->daddr;
const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr;
const struct in6_addr *faddr = &sk->sk_v6_daddr;
const __u16 lport = inet->inet_num;
const __be16 fport = inet->inet_dport;
struct net *net = sock_net(sk);
......
......@@ -116,7 +116,9 @@ struct inet_timewait_sock {
#define tw_prot __tw_common.skc_prot
#define tw_net __tw_common.skc_net
#define tw_daddr __tw_common.skc_daddr
#define tw_v6_daddr __tw_common.skc_v6_daddr
#define tw_rcv_saddr __tw_common.skc_rcv_saddr
#define tw_v6_rcv_saddr __tw_common.skc_v6_rcv_saddr
#define tw_dport __tw_common.skc_dport
#define tw_num __tw_common.skc_num
......@@ -133,7 +135,7 @@ struct inet_timewait_sock {
tw_transparent : 1,
tw_pad : 6, /* 6 bits hole */
tw_tos : 8,
tw_ipv6_offset : 16;
tw_pad2 : 16 /* 16 bits hole */
kmemcheck_bitfield_end(flags);
u32 tw_ttd;
struct inet_bind_bucket *tw_tb;
......
......@@ -374,7 +374,7 @@ static __inline__ void inet_reset_saddr(struct sock *sk)
struct ipv6_pinfo *np = inet6_sk(sk);
memset(&np->saddr, 0, sizeof(np->saddr));
memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr));
memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr));
}
#endif
}
......
......@@ -70,7 +70,7 @@ static inline void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
{
struct ipv6_pinfo *np = inet6_sk(sk);
__tcp_v6_send_check(skb, &np->saddr, &np->daddr);
__tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);
}
int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto);
......
......@@ -191,6 +191,12 @@ struct sock_common {
#ifdef CONFIG_NET_NS
struct net *skc_net;
#endif
#if IS_ENABLED(CONFIG_IPV6)
struct in6_addr skc_v6_daddr;
struct in6_addr skc_v6_rcv_saddr;
#endif
/*
* fields between dontcopy_begin/dontcopy_end
* are not copied in sock_copy()
......@@ -314,6 +320,9 @@ struct sock {
#define sk_bind_node __sk_common.skc_bind_node
#define sk_prot __sk_common.skc_prot
#define sk_net __sk_common.skc_net
#define sk_v6_daddr __sk_common.skc_v6_daddr
#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
socket_lock_t sk_lock;
struct sk_buff_head sk_receive_queue;
/*
......
......@@ -67,7 +67,7 @@ static inline void dccp_v6_send_check(struct sock *sk, struct sk_buff *skb)
struct dccp_hdr *dh = dccp_hdr(skb);
dccp_csum_outgoing(skb);
dh->dccph_checksum = dccp_v6_csum_finish(skb, &np->saddr, &np->daddr);
dh->dccph_checksum = dccp_v6_csum_finish(skb, &np->saddr, &sk->sk_v6_daddr);
}
static inline __u64 dccp_v6_init_sequence(struct sk_buff *skb)
......@@ -467,11 +467,11 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
memcpy(newnp, np, sizeof(struct ipv6_pinfo));
ipv6_addr_set_v4mapped(newinet->inet_daddr, &newnp->daddr);
ipv6_addr_set_v4mapped(newinet->inet_daddr, &newsk->sk_v6_daddr);
ipv6_addr_set_v4mapped(newinet->inet_saddr, &newnp->saddr);
newnp->rcv_saddr = newnp->saddr;
newsk->sk_v6_rcv_saddr = newnp->saddr;
inet_csk(newsk)->icsk_af_ops = &dccp_ipv6_mapped;
newsk->sk_backlog_rcv = dccp_v4_do_rcv;
......@@ -538,9 +538,9 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
memcpy(newnp, np, sizeof(struct ipv6_pinfo));
newnp->daddr = ireq6->rmt_addr;
newsk->sk_v6_daddr = ireq6->rmt_addr;
newnp->saddr = ireq6->loc_addr;
newnp->rcv_saddr = ireq6->loc_addr;
newsk->sk_v6_rcv_saddr = ireq6->loc_addr;
newsk->sk_bound_dev_if = ireq6->iif;
/* Now IPv6 options...
......@@ -885,7 +885,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
return -EINVAL;
}
np->daddr = usin->sin6_addr;
sk->sk_v6_daddr = usin->sin6_addr;
np->flow_label = fl6.flowlabel;
/*
......@@ -915,16 +915,16 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
goto failure;
}
ipv6_addr_set_v4mapped(inet->inet_saddr, &np->saddr);
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr, &np->rcv_saddr);
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr, &sk->sk_v6_rcv_saddr);
return err;
}
if (!ipv6_addr_any(&np->rcv_saddr))
saddr = &np->rcv_saddr;
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
saddr = &sk->sk_v6_rcv_saddr;
fl6.flowi6_proto = IPPROTO_DCCP;
fl6.daddr = np->daddr;
fl6.daddr = sk->sk_v6_daddr;
fl6.saddr = saddr ? *saddr : np->saddr;
fl6.flowi6_oif = sk->sk_bound_dev_if;
fl6.fl6_dport = usin->sin6_port;
......@@ -941,7 +941,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
if (saddr == NULL) {
saddr = &fl6.saddr;
np->rcv_saddr = *saddr;
sk->sk_v6_rcv_saddr = *saddr;
}
/* set the source address */
......@@ -963,7 +963,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
goto late_failure;
dp->dccps_iss = secure_dccpv6_sequence_number(np->saddr.s6_addr32,
np->daddr.s6_addr32,
sk->sk_v6_daddr.s6_addr32,
inet->inet_sport,
inet->inet_dport);
err = dccp_connect(sk);
......
......@@ -30,7 +30,6 @@ struct dccp6_request_sock {
struct dccp6_timewait_sock {
struct inet_timewait_sock inet;
struct inet6_timewait_sock tw6;
};
#endif /* _DCCP_IPV6_H */
......@@ -56,12 +56,9 @@ void dccp_time_wait(struct sock *sk, int state, int timeo)
#if IS_ENABLED(CONFIG_IPV6)
if (tw->tw_family == PF_INET6) {
const struct ipv6_pinfo *np = inet6_sk(sk);
struct inet6_timewait_sock *tw6;
tw->tw_ipv6_offset = inet6_tw_offset(sk->sk_prot);
tw6 = inet6_twsk((struct sock *)tw);
tw6->tw_v6_daddr = np->daddr;
tw6->tw_v6_rcv_saddr = np->rcv_saddr;
tw->tw_v6_daddr = sk->sk_v6_daddr;
tw->tw_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
tw->tw_ipv6only = np->ipv6only;
}
#endif
......
......@@ -121,13 +121,13 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
#if IS_ENABLED(CONFIG_IPV6)
if (r->idiag_family == AF_INET6) {
const struct ipv6_pinfo *np = inet6_sk(sk);
*(struct in6_addr *)r->id.idiag_src = np->rcv_saddr;
*(struct in6_addr *)r->id.idiag_dst = np->daddr;
*(struct in6_addr *)r->id.idiag_src = sk->sk_v6_rcv_saddr;
*(struct in6_addr *)r->id.idiag_dst = sk->sk_v6_daddr;
if (ext & (1 << (INET_DIAG_TCLASS - 1)))
if (nla_put_u8(skb, INET_DIAG_TCLASS, np->tclass) < 0)
if (nla_put_u8(skb, INET_DIAG_TCLASS,
inet6_sk(sk)->tclass) < 0)
goto errout;
}
#endif
......@@ -255,11 +255,8 @@ static int inet_twsk_diag_fill(struct inet_timewait_sock *tw,
r->idiag_inode = 0;
#if IS_ENABLED(CONFIG_IPV6)
if (tw->tw_family == AF_INET6) {
const struct inet6_timewait_sock *tw6 =
inet6_twsk((struct sock *)tw);
*(struct in6_addr *)r->id.idiag_src = tw6->tw_v6_rcv_saddr;
*(struct in6_addr *)r->id.idiag_dst = tw6->tw_v6_daddr;
*(struct in6_addr *)r->id.idiag_src = tw->tw_v6_rcv_saddr;
*(struct in6_addr *)r->id.idiag_dst = tw->tw_v6_daddr;
}
#endif
......@@ -273,10 +270,11 @@ static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
const struct nlmsghdr *unlh)
{
if (sk->sk_state == TCP_TIME_WAIT)
return inet_twsk_diag_fill((struct inet_timewait_sock *)sk,
skb, r, portid, seq, nlmsg_flags,
unlh);
return inet_csk_diag_fill(sk, skb, r, user_ns, portid, seq, nlmsg_flags, unlh);
return inet_twsk_diag_fill(inet_twsk(sk), skb, r, portid, seq,
nlmsg_flags, unlh);
return inet_csk_diag_fill(sk, skb, r, user_ns, portid, seq,
nlmsg_flags, unlh);
}
int inet_diag_dump_one_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *in_skb,
......@@ -489,10 +487,9 @@ int inet_diag_bc_sk(const struct nlattr *bc, struct sock *sk)
entry.family = sk->sk_family;
#if IS_ENABLED(CONFIG_IPV6)
if (entry.family == AF_INET6) {
struct ipv6_pinfo *np = inet6_sk(sk);
entry.saddr = np->rcv_saddr.s6_addr32;
entry.daddr = np->daddr.s6_addr32;
entry.saddr = sk->sk_v6_rcv_saddr.s6_addr32;
entry.daddr = sk->sk_v6_daddr.s6_addr32;
} else
#endif
{
......@@ -649,10 +646,8 @@ static int inet_twsk_diag_dump(struct sock *sk,
entry.family = tw->tw_family;
#if IS_ENABLED(CONFIG_IPV6)
if (tw->tw_family == AF_INET6) {
struct inet6_timewait_sock *tw6 =
inet6_twsk((struct sock *)tw);
entry.saddr = tw6->tw_v6_rcv_saddr.s6_addr32;
entry.daddr = tw6->tw_v6_daddr.s6_addr32;
entry.saddr = tw->tw_v6_rcv_saddr.s6_addr32;
entry.daddr = tw->tw_v6_daddr.s6_addr32;
} else
#endif
{
......
......@@ -202,15 +202,14 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident)
#if IS_ENABLED(CONFIG_IPV6)
} else if (skb->protocol == htons(ETH_P_IPV6) &&
sk->sk_family == AF_INET6) {
struct ipv6_pinfo *np = inet6_sk(sk);
pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk,
(int) isk->inet_num,
&inet6_sk(sk)->rcv_saddr,
&sk->sk_v6_rcv_saddr,
sk->sk_bound_dev_if);
if (!ipv6_addr_any(&np->rcv_saddr) &&
!ipv6_addr_equal(&np->rcv_saddr,
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&
!ipv6_addr_equal(&sk->sk_v6_rcv_saddr,
&ipv6_hdr(skb)->daddr))
continue;
#endif
......@@ -362,7 +361,7 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr *saddr)
} else if (saddr->sa_family == AF_INET6) {
struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr;
struct ipv6_pinfo *np = inet6_sk(sk);
np->rcv_saddr = np->saddr = addr->sin6_addr;
sk->sk_v6_rcv_saddr = np->saddr = addr->sin6_addr;
#endif
}
}
......@@ -376,7 +375,7 @@ static void ping_clear_saddr(struct sock *sk, int dif)
#if IS_ENABLED(CONFIG_IPV6)
} else if (sk->sk_family == AF_INET6) {
struct ipv6_pinfo *np = inet6_sk(sk);
memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr));
memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr));
memset(&np->saddr, 0, sizeof(np->saddr));
#endif
}
......@@ -418,7 +417,7 @@ int ping_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
err = 0;
if ((sk->sk_family == AF_INET && isk->inet_rcv_saddr) ||
(sk->sk_family == AF_INET6 &&
!ipv6_addr_any(&inet6_sk(sk)->rcv_saddr)))
!ipv6_addr_any(&sk->sk_v6_rcv_saddr)))
sk->sk_userlocks |= SOCK_BINDADDR_LOCK;
if (snum)
......@@ -429,7 +428,7 @@ int ping_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
#if IS_ENABLED(CONFIG_IPV6)
if (sk->sk_family == AF_INET6)
memset(&inet6_sk(sk)->daddr, 0, sizeof(inet6_sk(sk)->daddr));
memset(&sk->sk_v6_daddr, 0, sizeof(sk->sk_v6_daddr));
#endif
sk_dst_reset(sk);
......
......@@ -240,7 +240,6 @@ static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock *tw)
{
struct inet6_timewait_sock *tw6;
struct tcp_metrics_block *tm;
struct inetpeer_addr addr;
unsigned int hash;
......@@ -253,9 +252,8 @@ static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock
hash = (__force unsigned int) addr.addr.a4;
break;
case AF_INET6:
tw6 = inet6_twsk((struct sock *)tw);
*(struct in6_addr *)addr.addr.a6 = tw6->tw_v6_daddr;
hash = ipv6_addr_hash(&tw6->tw_v6_daddr);
*(struct in6_addr *)addr.addr.a6 = tw->tw_v6_daddr;
hash = ipv6_addr_hash(&tw->tw_v6_daddr);
break;
default:
return NULL;
......@@ -289,8 +287,8 @@ static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
hash = (__force unsigned int) addr.addr.a4;
break;
case AF_INET6:
*(struct in6_addr *)addr.addr.a6 = inet6_sk(sk)->daddr;
hash = ipv6_addr_hash(&inet6_sk(sk)->daddr);
*(struct in6_addr *)addr.addr.a6 = sk->sk_v6_daddr;
hash = ipv6_addr_hash(&sk->sk_v6_daddr);
break;
default:
return NULL;
......
......@@ -293,12 +293,9 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
#if IS_ENABLED(CONFIG_IPV6)
if (tw->tw_family == PF_INET6) {
struct ipv6_pinfo *np = inet6_sk(sk);
struct inet6_timewait_sock *tw6;
tw->tw_ipv6_offset = inet6_tw_offset(sk->sk_prot);
tw6 = inet6_twsk((struct sock *)tw);
tw6->tw_v6_daddr = np->daddr;
tw6->tw_v6_rcv_saddr = np->rcv_saddr;
tw->tw_v6_daddr = sk->sk_v6_daddr;
tw->tw_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
tw->tw_tclass = np->tclass;
tw->tw_ipv6only = np->ipv6only;
}
......
......@@ -101,22 +101,6 @@ static inline int tcp_probe_avail(void)
si4.sin_addr.s_addr = inet->inet_##mem##addr; \
} while (0) \
#if IS_ENABLED(CONFIG_IPV6)
#define tcp_probe_copy_fl_to_si6(inet, si6, mem) \
do { \
struct ipv6_pinfo *pi6 = inet->pinet6; \
si6.sin6_family = AF_INET6; \
si6.sin6_port = inet->inet_##mem##port; \
si6.sin6_addr = pi6->mem##addr; \
si6.sin6_flowinfo = 0; /* No need here. */ \
si6.sin6_scope_id = 0; /* No need here. */ \
} while (0)
#else
#define tcp_probe_copy_fl_to_si6(fl, si6, mem) \
do { \
memset(&si6, 0, sizeof(si6)); \
} while (0)
#endif
/*
* Hook inserted to be called before each receive packet.
......@@ -147,8 +131,17 @@ static void jtcp_rcv_established(struct sock *sk, struct sk_buff *skb,
tcp_probe_copy_fl_to_si4(inet, p->dst.v4, d);
break;
case AF_INET6:
tcp_probe_copy_fl_to_si6(inet, p->src.v6, s);
tcp_probe_copy_fl_to_si6(inet, p->dst.v6, d);
memset(&p->src.v6, 0, sizeof(p->src.v6));
memset(&p->dst.v6, 0, sizeof(p->dst.v6));
#if IS_ENABLED(CONFIG_IPV6)
p->src.v6.sin6_family = AF_INET6;
p->src.v6.sin6_port = inet->inet_sport;
p->src.v6.sin6_addr = inet6_sk(sk)->saddr;
p->dst.v6.sin6_family = AF_INET6;
p->dst.v6.sin6_port = inet->inet_dport;
p->dst.v6.sin6_addr = sk->sk_v6_daddr;
#endif
break;
default:
BUG();
......
......@@ -374,9 +374,8 @@ void tcp_retransmit_timer(struct sock *sk)
}
#if IS_ENABLED(CONFIG_IPV6)
else if (sk->sk_family == AF_INET6) {
struct ipv6_pinfo *np = inet6_sk(sk);
LIMIT_NETDEBUG(KERN_DEBUG pr_fmt("Peer %pI6:%u/%u unexpectedly shrunk window %u:%u (repaired)\n"),
&np->daddr,
&sk->sk_v6_daddr,
ntohs(inet->inet_dport), inet->inet_num,
tp->snd_una, tp->snd_nxt);
}
......
......@@ -364,7 +364,7 @@ int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
inet->inet_rcv_saddr = v4addr;
inet->inet_saddr = v4addr;
np->rcv_saddr = addr->sin6_addr;
sk->sk_v6_rcv_saddr = addr->sin6_addr;
if (!(addr_type & IPV6_ADDR_MULTICAST))
np->saddr = addr->sin6_addr;
......@@ -461,14 +461,14 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr,
peer == 1)
return -ENOTCONN;
sin->sin6_port = inet->inet_dport;
sin->sin6_addr = np->daddr;
sin->sin6_addr = sk->sk_v6_daddr;
if (np->sndflow)
sin->sin6_flowinfo = np->flow_label;
} else {
if (ipv6_addr_any(&np->rcv_saddr))
if (ipv6_addr_any(&sk->sk_v6_rcv_saddr))
sin->sin6_addr = np->saddr;
else
sin->sin6_addr = np->rcv_saddr;
sin->sin6_addr = sk->sk_v6_rcv_saddr;
sin->sin6_port = inet->inet_sport;
}
......@@ -655,7 +655,7 @@ int inet6_sk_rebuild_header(struct sock *sk)
memset(&fl6, 0, sizeof(fl6));
fl6.flowi6_proto = sk->sk_protocol;
fl6.daddr = np->daddr;
fl6.daddr = sk->sk_v6_daddr;
fl6.saddr = np->saddr;
fl6.flowlabel = np->flow_label;
fl6.flowi6_oif = sk->sk_bound_dev_if;
......
......@@ -107,16 +107,16 @@ ipv4_connected:
if (err)
goto out;
ipv6_addr_set_v4mapped(inet->inet_daddr, &np->daddr);
ipv6_addr_set_v4mapped(inet->inet_daddr, &sk->sk_v6_daddr);
if (ipv6_addr_any(&np->saddr) ||
ipv6_mapped_addr_any(&np->saddr))
ipv6_addr_set_v4mapped(inet->inet_saddr, &np->saddr);
if (ipv6_addr_any(&np->rcv_saddr) ||
ipv6_mapped_addr_any(&np->rcv_saddr)) {
if (ipv6_addr_any(&sk->sk_v6_rcv_saddr) ||
ipv6_mapped_addr_any(&sk->sk_v6_rcv_saddr)) {
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr,
&np->rcv_saddr);
&sk->sk_v6_rcv_saddr);
if (sk->sk_prot->rehash)
sk->sk_prot->rehash(sk);
}
......@@ -145,7 +145,7 @@ ipv4_connected:
}
}
np->daddr = *daddr;
sk->sk_v6_daddr = *daddr;
np->flow_label = fl6.flowlabel;
inet->inet_dport = usin->sin6_port;
......@@ -156,7 +156,7 @@ ipv4_connected:
*/
fl6.flowi6_proto = sk->sk_protocol;
fl6.daddr = np->daddr;
fl6.daddr = sk->sk_v6_daddr;
fl6.saddr = np->saddr;
fl6.flowi6_oif = sk->sk_bound_dev_if;
fl6.flowi6_mark = sk->sk_mark;
......@@ -183,16 +183,16 @@ ipv4_connected:
if (ipv6_addr_any(&np->saddr))
np->saddr = fl6.saddr;
if (ipv6_addr_any(&np->rcv_saddr)) {
np->rcv_saddr = fl6.saddr;
if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
sk->sk_v6_rcv_saddr = fl6.saddr;
inet->inet_rcv_saddr = LOOPBACK4_IPV6;
if (sk->sk_prot->rehash)
sk->sk_prot->rehash(sk);
}
ip6_dst_store(sk, dst,
ipv6_addr_equal(&fl6.daddr, &np->daddr) ?
&np->daddr : NULL,
ipv6_addr_equal(&fl6.daddr, &sk->sk_v6_daddr) ?
&sk->sk_v6_daddr : NULL,
#ifdef CONFIG_IPV6_SUBTREES
ipv6_addr_equal(&fl6.saddr, &np->saddr) ?
&np->saddr :
......@@ -883,11 +883,10 @@ EXPORT_SYMBOL_GPL(ip6_datagram_send_ctl);
void ip6_dgram_sock_seq_show(struct seq_file *seq, struct sock *sp,
__u16 srcp, __u16 destp, int bucket)
{
struct ipv6_pinfo *np = inet6_sk(sp);
const struct in6_addr *dest, *src;
dest = &np->daddr;
src = &np->rcv_saddr;
dest = &sp->sk_v6_daddr;
src = &sp->sk_v6_rcv_saddr;
seq_printf(seq,
"%5d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
"%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d\n",
......
......@@ -165,11 +165,10 @@ EXPORT_SYMBOL_GPL(inet6_csk_reqsk_queue_hash_add);
void inet6_csk_addr2sockaddr(struct sock *sk, struct sockaddr * uaddr)
{
struct ipv6_pinfo *np = inet6_sk(sk);
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) uaddr;
sin6->sin6_family = AF_INET6;
sin6->sin6_addr = np->daddr;
sin6->sin6_addr = sk->sk_v6_daddr;
sin6->sin6_port = inet_sk(sk)->inet_dport;
/* We do not store received flowlabel for TCP */
sin6->sin6_flowinfo = 0;
......@@ -203,7 +202,7 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk,
memset(fl6, 0, sizeof(*fl6));
fl6->flowi6_proto = sk->sk_protocol;
fl6->daddr = np->daddr;
fl6->daddr = sk->sk_v6_daddr;
fl6->saddr = np->saddr;
fl6->flowlabel = np->flow_label;
IP6_ECN_flow_xmit(sk, fl6->flowlabel);
......@@ -245,7 +244,7 @@ int inet6_csk_xmit(struct sk_buff *skb, struct flowi *fl_unused)
skb_dst_set_noref(skb, dst);
/* Restore final destination back after routing done */
fl6.daddr = np->daddr;
fl6.daddr = sk->sk_v6_daddr;
res = ip6_xmit(sk, skb, &fl6, np->opt, np->tclass);
rcu_read_unlock();
......
......@@ -89,30 +89,16 @@ begin:
sk_nulls_for_each_rcu(sk, node, &head->chain) {
if (sk->sk_hash != hash)
continue;
if (sk->sk_state == TCP_TIME_WAIT) {
if (!INET6_TW_MATCH(sk, net, saddr, daddr, ports, dif))
continue;
} else {
if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif))
continue;
}
if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif))
continue;
if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
goto out;
if (sk->sk_state == TCP_TIME_WAIT) {
if (unlikely(!INET6_TW_MATCH(sk, net, saddr, daddr,
ports, dif))) {
sock_gen_put(sk);
goto begin;
}
} else {
if (unlikely(!INET6_MATCH(sk, net, saddr, daddr,
ports, dif))) {
sock_put(sk);
goto begin;
}
goto found;
if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
sock_gen_put(sk);
goto begin;
}
goto found;
}
if (get_nulls_value(node) != slot)
goto begin;
......@@ -133,11 +119,10 @@ static inline int compute_score(struct sock *sk, struct net *net,
if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
sk->sk_family == PF_INET6) {
const struct ipv6_pinfo *np = inet6_sk(sk);
score = 1;
if (!ipv6_addr_any(&np->rcv_saddr)) {
if (!ipv6_addr_equal(&np->rcv_saddr, daddr))
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
return -1;
score++;
}
......@@ -229,9 +214,8 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
{
struct inet_hashinfo *hinfo = death_row->hashinfo;
struct inet_sock *inet = inet_sk(sk);
const struct ipv6_pinfo *np = inet6_sk(sk);
const struct in6_addr *daddr = &np->rcv_saddr;
const struct in6_addr *saddr = &np->daddr;
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
const struct in6_addr *saddr = &sk->sk_v6_daddr;
const int dif = sk->sk_bound_dev_if;
const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
struct net *net = sock_net(sk);
......@@ -250,23 +234,19 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
if (sk2->sk_hash != hash)
continue;
if (sk2->sk_state == TCP_TIME_WAIT) {
if (likely(INET6_TW_MATCH(sk2, net, saddr, daddr,
ports, dif))) {
if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif))) {
if (sk2->sk_state == TCP_TIME_WAIT) {
tw = inet_twsk(sk2);
if (twsk_unique(sk, sk2, twp))
goto unique;
else
goto not_unique;
break;
}
}
if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif)))
goto not_unique;