inet_hashtables.c 11.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Generic INET transport hashtables
 *
 * Authors:	Lotsa people, from code originally in tcp
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

16
#include <linux/module.h>
17
#include <linux/random.h>
18
#include <linux/sched.h>
19
#include <linux/slab.h>
20
#include <linux/wait.h>
21

22
#include <net/inet_connection_sock.h>
23
#include <net/inet_hashtables.h>
24
#include <net/ip.h>
25
26
27
28
29

/*
 * Allocate and initialize a new local port bind bucket.
 * The bindhash mutex for snum's hash chain must be held here.
 */
30
struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
31
32
33
						 struct inet_bind_hashbucket *head,
						 const unsigned short snum)
{
34
	struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
35
36
37
38
39
40
41
42
43
44
45
46
47

	if (tb != NULL) {
		tb->port      = snum;
		tb->fastreuse = 0;
		INIT_HLIST_HEAD(&tb->owners);
		hlist_add_head(&tb->node, &head->chain);
	}
	return tb;
}

/*
 * Caller must hold hashbucket lock for this tb with local BH disabled
 */
48
void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
49
50
51
52
53
54
{
	if (hlist_empty(&tb->owners)) {
		__hlist_del(&tb->node);
		kmem_cache_free(cachep, tb);
	}
}
55
56
57
58

void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
		    const unsigned short snum)
{
59
	inet_sk(sk)->num = snum;
60
	sk_add_bind_node(sk, &tb->owners);
61
	inet_csk(sk)->icsk_bind_hash = tb;
62
63
64
65
66
67
68
}

/*
 * Get rid of any references to a local port held by the given sock.
 */
static void __inet_put_port(struct inet_hashinfo *hashinfo, struct sock *sk)
{
69
	const int bhash = inet_bhashfn(inet_sk(sk)->num, hashinfo->bhash_size);
70
71
72
73
	struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
	struct inet_bind_bucket *tb;

	spin_lock(&head->lock);
74
	tb = inet_csk(sk)->icsk_bind_hash;
75
	__sk_del_bind_node(sk);
76
77
	inet_csk(sk)->icsk_bind_hash = NULL;
	inet_sk(sk)->num = 0;
78
79
80
81
82
83
84
85
86
87
88
89
	inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
	spin_unlock(&head->lock);
}

void inet_put_port(struct inet_hashinfo *hashinfo, struct sock *sk)
{
	local_bh_disable();
	__inet_put_port(hashinfo, sk);
	local_bh_enable();
}

EXPORT_SYMBOL(inet_put_port);
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

/*
 * This lock without WQ_FLAG_EXCLUSIVE is good on UP and it can be very bad on SMP.
 * Look, when several writers sleep and reader wakes them up, all but one
 * immediately hit write lock and grab all the cpus. Exclusive sleep solves
 * this, _but_ remember, it adds useless work on UP machines (wake up each
 * exclusive lock release). It should be ifdefed really.
 */
void inet_listen_wlock(struct inet_hashinfo *hashinfo)
{
	write_lock(&hashinfo->lhash_lock);

	if (atomic_read(&hashinfo->lhash_users)) {
		DEFINE_WAIT(wait);

		for (;;) {
			prepare_to_wait_exclusive(&hashinfo->lhash_wait,
						  &wait, TASK_UNINTERRUPTIBLE);
			if (!atomic_read(&hashinfo->lhash_users))
				break;
			write_unlock_bh(&hashinfo->lhash_lock);
			schedule();
			write_lock_bh(&hashinfo->lhash_lock);
		}

		finish_wait(&hashinfo->lhash_wait, &wait);
	}
}

EXPORT_SYMBOL(inet_listen_wlock);
120
121
122
123
124
125
126

/*
 * Don't inline this cruft. Here are some nice properties to exploit here. The
 * BSD API does not allow a listening sock to specify the remote port nor the
 * remote address for the connection. So always assume those are both
 * wildcarded during the search since they can never be otherwise.
 */
127
static struct sock *inet_lookup_listener_slow(const struct hlist_head *head,
128
					      const __be32 daddr,
129
130
					      const unsigned short hnum,
					      const int dif)
131
132
133
134
135
136
137
138
139
{
	struct sock *result = NULL, *sk;
	const struct hlist_node *node;
	int hiscore = -1;

	sk_for_each(sk, node, head) {
		const struct inet_sock *inet = inet_sk(sk);

		if (inet->num == hnum && !ipv6_only_sock(sk)) {
140
			const __be32 rcv_saddr = inet->rcv_saddr;
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
			int score = sk->sk_family == PF_INET ? 1 : 0;

			if (rcv_saddr) {
				if (rcv_saddr != daddr)
					continue;
				score += 2;
			}
			if (sk->sk_bound_dev_if) {
				if (sk->sk_bound_dev_if != dif)
					continue;
				score += 2;
			}
			if (score == 5)
				return sk;
			if (score > hiscore) {
				hiscore	= score;
				result	= sk;
			}
		}
	}
	return result;
}
163

164
/* Optimize the common listener case. */
165
struct sock *__inet_lookup_listener(struct inet_hashinfo *hashinfo,
166
				    const __be32 daddr, const unsigned short hnum,
167
				    const int dif)
168
169
170
171
172
173
174
175
176
177
178
179
180
181
{
	struct sock *sk = NULL;
	const struct hlist_head *head;

	read_lock(&hashinfo->lhash_lock);
	head = &hashinfo->listening_hash[inet_lhashfn(hnum)];
	if (!hlist_empty(head)) {
		const struct inet_sock *inet = inet_sk((sk = __sk_head(head)));

		if (inet->num == hnum && !sk->sk_node.next &&
		    (!inet->rcv_saddr || inet->rcv_saddr == daddr) &&
		    (sk->sk_family == PF_INET || !ipv6_only_sock(sk)) &&
		    !sk->sk_bound_dev_if)
			goto sherry_cache;
182
		sk = inet_lookup_listener_slow(head, daddr, hnum, dif);
183
184
185
186
187
188
189
190
	}
	if (sk) {
sherry_cache:
		sock_hold(sk);
	}
	read_unlock(&hashinfo->lhash_lock);
	return sk;
}
191
EXPORT_SYMBOL_GPL(__inet_lookup_listener);
192

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
struct sock * __inet_lookup_established(struct inet_hashinfo *hashinfo,
				  const __be32 saddr, const __be16 sport,
				  const __be32 daddr, const u16 hnum,
				  const int dif)
{
	INET_ADDR_COOKIE(acookie, saddr, daddr)
	const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
	struct sock *sk;
	const struct hlist_node *node;
	/* Optimize here for direct hit, only listening connections can
	 * have wildcards anyways.
	 */
	unsigned int hash = inet_ehashfn(daddr, hnum, saddr, sport);
	struct inet_ehash_bucket *head = inet_ehash_bucket(hashinfo, hash);
	rwlock_t *lock = inet_ehash_lockp(hashinfo, hash);

	prefetch(head->chain.first);
	read_lock(lock);
	sk_for_each(sk, node, &head->chain) {
		if (INET_MATCH(sk, hash, acookie, saddr, daddr, ports, dif))
			goto hit; /* You sunk my battleship! */
	}

	/* Must check for a TIME_WAIT'er before going to listener hash. */
	sk_for_each(sk, node, &head->twchain) {
		if (INET_TW_MATCH(sk, hash, acookie, saddr, daddr, ports, dif))
			goto hit;
	}
	sk = NULL;
out:
	read_unlock(lock);
	return sk;
hit:
	sock_hold(sk);
	goto out;
}
EXPORT_SYMBOL_GPL(__inet_lookup_established);

231
232
233
234
235
236
237
/* called with local bh disabled */
static int __inet_check_established(struct inet_timewait_death_row *death_row,
				    struct sock *sk, __u16 lport,
				    struct inet_timewait_sock **twp)
{
	struct inet_hashinfo *hinfo = death_row->hashinfo;
	struct inet_sock *inet = inet_sk(sk);
238
239
	__be32 daddr = inet->rcv_saddr;
	__be32 saddr = inet->daddr;
240
241
	int dif = sk->sk_bound_dev_if;
	INET_ADDR_COOKIE(acookie, saddr, daddr)
Al Viro's avatar
Al Viro committed
242
	const __portpair ports = INET_COMBINED_PORTS(inet->dport, lport);
243
244
	unsigned int hash = inet_ehashfn(daddr, lport, saddr, inet->dport);
	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
245
	rwlock_t *lock = inet_ehash_lockp(hinfo, hash);
246
247
248
249
250
	struct sock *sk2;
	const struct hlist_node *node;
	struct inet_timewait_sock *tw;

	prefetch(head->chain.first);
251
	write_lock(lock);
252
253

	/* Check TIME-WAIT sockets first. */
254
	sk_for_each(sk2, node, &head->twchain) {
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
		tw = inet_twsk(sk2);

		if (INET_TW_MATCH(sk2, hash, acookie, saddr, daddr, ports, dif)) {
			if (twsk_unique(sk, sk2, twp))
				goto unique;
			else
				goto not_unique;
		}
	}
	tw = NULL;

	/* And established part... */
	sk_for_each(sk2, node, &head->chain) {
		if (INET_MATCH(sk2, hash, acookie, saddr, daddr, ports, dif))
			goto not_unique;
	}

unique:
	/* Must record num and sport now. Otherwise we will see
	 * in hash table socket with a funny identity. */
	inet->num = lport;
	inet->sport = htons(lport);
	sk->sk_hash = hash;
	BUG_TRAP(sk_unhashed(sk));
	__sk_add_node(sk, &head->chain);
	sock_prot_inc_use(sk->sk_prot);
281
	write_unlock(lock);
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296

	if (twp) {
		*twp = tw;
		NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED);
	} else if (tw) {
		/* Silly. Should hash-dance instead... */
		inet_twsk_deschedule(tw, death_row);
		NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED);

		inet_twsk_put(tw);
	}

	return 0;

not_unique:
297
	write_unlock(lock);
298
299
300
301
302
303
	return -EADDRNOTAVAIL;
}

static inline u32 inet_sk_port_offset(const struct sock *sk)
{
	const struct inet_sock *inet = inet_sk(sk);
304
	return secure_ipv4_port_ephemeral(inet->rcv_saddr, inet->daddr,
305
306
307
					  inet->dport);
}

308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
void __inet_hash_nolisten(struct inet_hashinfo *hashinfo, struct sock *sk)
{
	struct hlist_head *list;
	rwlock_t *lock;
	struct inet_ehash_bucket *head;

	BUG_TRAP(sk_unhashed(sk));

	sk->sk_hash = inet_sk_ehashfn(sk);
	head = inet_ehash_bucket(hashinfo, sk->sk_hash);
	list = &head->chain;
	lock = inet_ehash_lockp(hashinfo, sk->sk_hash);

	write_lock(lock);
	__sk_add_node(sk, list);
	sock_prot_inc_use(sk->sk_prot);
	write_unlock(lock);
}
EXPORT_SYMBOL_GPL(__inet_hash_nolisten);

void __inet_hash(struct inet_hashinfo *hashinfo, struct sock *sk)
{
	struct hlist_head *list;
	rwlock_t *lock;

	if (sk->sk_state != TCP_LISTEN) {
		__inet_hash_nolisten(hashinfo, sk);
		return;
	}

	BUG_TRAP(sk_unhashed(sk));
	list = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
	lock = &hashinfo->lhash_lock;

	inet_listen_wlock(hashinfo);
	__sk_add_node(sk, list);
	sock_prot_inc_use(sk->sk_prot);
	write_unlock(lock);
	wake_up(&hashinfo->lhash_wait);
}
EXPORT_SYMBOL_GPL(__inet_hash);

350
351
352
353
354
355
356
357
/*
 * Bind a port for a connect operation and hash it.
 */
int inet_hash_connect(struct inet_timewait_death_row *death_row,
		      struct sock *sk)
{
	struct inet_hashinfo *hinfo = death_row->hashinfo;
	const unsigned short snum = inet_sk(sk)->num;
358
359
	struct inet_bind_hashbucket *head;
	struct inet_bind_bucket *tb;
360
361
	int ret;

362
	if (!snum) {
363
		int i, remaining, low, high, port;
364
365
366
		static u32 hint;
		u32 offset = hint + inet_sk_port_offset(sk);
		struct hlist_node *node;
367
		struct inet_timewait_sock *tw = NULL;
368

369
		inet_get_local_port_range(&low, &high);
370
		remaining = (high - low) + 1;
371

372
		local_bh_disable();
373
374
		for (i = 1; i <= remaining; i++) {
			port = low + (i + offset) % remaining;
375
376
			head = &hinfo->bhash[inet_bhashfn(port, hinfo->bhash_size)];
			spin_lock(&head->lock);
377

378
379
380
381
			/* Does not bother with rcv_saddr checks,
			 * because the established check is already
			 * unique enough.
			 */
382
			inet_bind_bucket_for_each(tb, node, &head->chain) {
383
384
385
386
387
				if (tb->port == port) {
					BUG_TRAP(!hlist_empty(&tb->owners));
					if (tb->fastreuse >= 0)
						goto next_port;
					if (!__inet_check_established(death_row,
388
389
								      sk, port,
								      &tw))
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
						goto ok;
					goto next_port;
				}
			}

			tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, head, port);
			if (!tb) {
				spin_unlock(&head->lock);
				break;
			}
			tb->fastreuse = -1;
			goto ok;

		next_port:
			spin_unlock(&head->lock);
		}
		local_bh_enable();

		return -EADDRNOTAVAIL;
409
410
411
412

ok:
		hint += i;

413
414
		/* Head lock still held and bh's disabled */
		inet_bind_hash(sk, tb, port);
415
		if (sk_unhashed(sk)) {
416
			inet_sk(sk)->sport = htons(port);
417
			__inet_hash_nolisten(hinfo, sk);
418
419
		}
		spin_unlock(&head->lock);
420

421
422
423
424
		if (tw) {
			inet_twsk_deschedule(tw, death_row);
			inet_twsk_put(tw);
		}
425
426
427

		ret = 0;
		goto out;
428
	}
429

430
431
	head = &hinfo->bhash[inet_bhashfn(snum, hinfo->bhash_size)];
	tb  = inet_csk(sk)->icsk_bind_hash;
432
433
	spin_lock_bh(&head->lock);
	if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
434
		__inet_hash_nolisten(hinfo, sk);
435
436
437
438
439
440
441
442
443
444
445
446
447
		spin_unlock_bh(&head->lock);
		return 0;
	} else {
		spin_unlock(&head->lock);
		/* No definite answer... Walk to established hash table */
		ret = __inet_check_established(death_row, sk, snum, NULL);
out:
		local_bh_enable();
		return ret;
	}
}

EXPORT_SYMBOL_GPL(inet_hash_connect);