xfrm_policy.c 62.1 KB
Newer Older
1
/*
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
5
6
7
8
9
10
11
12
 * xfrm_policy.c
 *
 * Changes:
 *	Mitsuru KANDA @USAGI
 * 	Kazunori MIYAZAWA @USAGI
 * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 * 		IPv6 support
 * 	Kazunori MIYAZAWA @USAGI
 * 	YOSHIFUJI Hideaki
 * 		Split up af-specific portion
 *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
13
 *
Linus Torvalds's avatar
Linus Torvalds committed
14
15
 */

16
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
18
19
20
21
22
23
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <linux/notifier.h>
#include <linux/netdevice.h>
24
#include <linux/netfilter.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/module.h>
26
#include <linux/cache.h>
Paul Moore's avatar
Paul Moore committed
27
#include <linux/audit.h>
28
#include <net/dst.h>
Linus Torvalds's avatar
Linus Torvalds committed
29
30
#include <net/xfrm.h>
#include <net/ip.h>
31
32
33
#ifdef CONFIG_XFRM_STATISTICS
#include <net/snmp.h>
#endif
Linus Torvalds's avatar
Linus Torvalds committed
34

35
36
#include "xfrm_hash.h"

37
int sysctl_xfrm_larval_drop __read_mostly;
38

39
40
41
42
43
#ifdef CONFIG_XFRM_STATISTICS
DEFINE_SNMP_STAT(struct linux_xfrm_mib, xfrm_statistics) __read_mostly;
EXPORT_SYMBOL(xfrm_statistics);
#endif

Arjan van de Ven's avatar
Arjan van de Ven committed
44
45
DEFINE_MUTEX(xfrm_cfg_mutex);
EXPORT_SYMBOL(xfrm_cfg_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
46
47
48

static DEFINE_RWLOCK(xfrm_policy_lock);

49
50
unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
EXPORT_SYMBOL(xfrm_policy_count);
Linus Torvalds's avatar
Linus Torvalds committed
51
52
53
54

static DEFINE_RWLOCK(xfrm_policy_afinfo_lock);
static struct xfrm_policy_afinfo *xfrm_policy_afinfo[NPROTO];

55
static struct kmem_cache *xfrm_dst_cache __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
56
57

static struct work_struct xfrm_policy_gc_work;
58
static HLIST_HEAD(xfrm_policy_gc_list);
Linus Torvalds's avatar
Linus Torvalds committed
59
60
61
62
static DEFINE_SPINLOCK(xfrm_policy_gc_lock);

static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family);
static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo);
63
static void xfrm_init_pmtu(struct dst_entry *dst);
Linus Torvalds's avatar
Linus Torvalds committed
64

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
static inline int
__xfrm4_selector_match(struct xfrm_selector *sel, struct flowi *fl)
{
	return  addr_match(&fl->fl4_dst, &sel->daddr, sel->prefixlen_d) &&
		addr_match(&fl->fl4_src, &sel->saddr, sel->prefixlen_s) &&
		!((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
		(fl->proto == sel->proto || !sel->proto) &&
		(fl->oif == sel->ifindex || !sel->ifindex);
}

static inline int
__xfrm6_selector_match(struct xfrm_selector *sel, struct flowi *fl)
{
	return  addr_match(&fl->fl6_dst, &sel->daddr, sel->prefixlen_d) &&
		addr_match(&fl->fl6_src, &sel->saddr, sel->prefixlen_s) &&
		!((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
		(fl->proto == sel->proto || !sel->proto) &&
		(fl->oif == sel->ifindex || !sel->ifindex);
}

int xfrm_selector_match(struct xfrm_selector *sel, struct flowi *fl,
		    unsigned short family)
{
	switch (family) {
	case AF_INET:
		return __xfrm4_selector_match(sel, fl);
	case AF_INET6:
		return __xfrm6_selector_match(sel, fl);
	}
	return 0;
}

99
100
static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos,
						int family)
Linus Torvalds's avatar
Linus Torvalds committed
101
{
102
103
104
105
106
107
108
109
110
	xfrm_address_t *saddr = &x->props.saddr;
	xfrm_address_t *daddr = &x->id.daddr;
	struct xfrm_policy_afinfo *afinfo;
	struct dst_entry *dst;

	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR)
		saddr = x->coaddr;
	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR)
		daddr = x->coaddr;
Linus Torvalds's avatar
Linus Torvalds committed
111

112
	afinfo = xfrm_policy_get_afinfo(family);
Linus Torvalds's avatar
Linus Torvalds committed
113
	if (unlikely(afinfo == NULL))
114
		return ERR_PTR(-EAFNOSUPPORT);
Linus Torvalds's avatar
Linus Torvalds committed
115

116
	dst = afinfo->dst_lookup(tos, saddr, daddr);
Linus Torvalds's avatar
Linus Torvalds committed
117
	xfrm_policy_put_afinfo(afinfo);
118
	return dst;
Linus Torvalds's avatar
Linus Torvalds committed
119
120
121
122
123
124
125
}

static inline unsigned long make_jiffies(long secs)
{
	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
		return MAX_SCHEDULE_TIMEOUT-1;
	else
126
		return secs*HZ;
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
130
131
}

static void xfrm_policy_timer(unsigned long data)
{
	struct xfrm_policy *xp = (struct xfrm_policy*)data;
132
	unsigned long now = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
133
134
135
136
137
138
139
140
141
	long next = LONG_MAX;
	int warn = 0;
	int dir;

	read_lock(&xp->lock);

	if (xp->dead)
		goto out;

142
	dir = xfrm_policy_id2dir(xp->index);
Linus Torvalds's avatar
Linus Torvalds committed
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181

	if (xp->lft.hard_add_expires_seconds) {
		long tmo = xp->lft.hard_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.hard_use_expires_seconds) {
		long tmo = xp->lft.hard_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_add_expires_seconds) {
		long tmo = xp->lft.soft_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_use_expires_seconds) {
		long tmo = xp->lft.soft_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}

	if (warn)
182
		km_policy_expired(xp, dir, 0, 0);
Linus Torvalds's avatar
Linus Torvalds committed
183
184
185
186
187
188
189
190
191
192
193
	if (next != LONG_MAX &&
	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
		xfrm_pol_hold(xp);

out:
	read_unlock(&xp->lock);
	xfrm_pol_put(xp);
	return;

expired:
	read_unlock(&xp->lock);
194
	if (!xfrm_policy_delete(xp, dir))
195
		km_policy_expired(xp, dir, 1, 0);
Linus Torvalds's avatar
Linus Torvalds committed
196
197
198
199
200
201
202
203
	xfrm_pol_put(xp);
}


/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 * SPD calls.
 */

204
struct xfrm_policy *xfrm_policy_alloc(gfp_t gfp)
Linus Torvalds's avatar
Linus Torvalds committed
205
206
207
{
	struct xfrm_policy *policy;

208
	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
Linus Torvalds's avatar
Linus Torvalds committed
209
210

	if (policy) {
211
212
		INIT_HLIST_NODE(&policy->bydst);
		INIT_HLIST_NODE(&policy->byidx);
Linus Torvalds's avatar
Linus Torvalds committed
213
		rwlock_init(&policy->lock);
214
		atomic_set(&policy->refcnt, 1);
215
216
		setup_timer(&policy->timer, xfrm_policy_timer,
				(unsigned long)policy);
Linus Torvalds's avatar
Linus Torvalds committed
217
218
219
220
221
222
223
	}
	return policy;
}
EXPORT_SYMBOL(xfrm_policy_alloc);

/* Destroy xfrm_policy: descendant resources must be released to this moment. */

224
void xfrm_policy_destroy(struct xfrm_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
225
{
226
	BUG_ON(!policy->dead);
Linus Torvalds's avatar
Linus Torvalds committed
227

228
	BUG_ON(policy->bundles);
Linus Torvalds's avatar
Linus Torvalds committed
229
230
231
232

	if (del_timer(&policy->timer))
		BUG();

233
	security_xfrm_policy_free(policy);
Linus Torvalds's avatar
Linus Torvalds committed
234
235
	kfree(policy);
}
236
EXPORT_SYMBOL(xfrm_policy_destroy);
Linus Torvalds's avatar
Linus Torvalds committed
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255

static void xfrm_policy_gc_kill(struct xfrm_policy *policy)
{
	struct dst_entry *dst;

	while ((dst = policy->bundles) != NULL) {
		policy->bundles = dst->next;
		dst_free(dst);
	}

	if (del_timer(&policy->timer))
		atomic_dec(&policy->refcnt);

	if (atomic_read(&policy->refcnt) > 1)
		flow_cache_flush();

	xfrm_pol_put(policy);
}

David Howells's avatar
David Howells committed
256
static void xfrm_policy_gc_task(struct work_struct *work)
Linus Torvalds's avatar
Linus Torvalds committed
257
258
{
	struct xfrm_policy *policy;
259
260
	struct hlist_node *entry, *tmp;
	struct hlist_head gc_list;
Linus Torvalds's avatar
Linus Torvalds committed
261
262

	spin_lock_bh(&xfrm_policy_gc_lock);
263
264
	gc_list.first = xfrm_policy_gc_list.first;
	INIT_HLIST_HEAD(&xfrm_policy_gc_list);
Linus Torvalds's avatar
Linus Torvalds committed
265
266
	spin_unlock_bh(&xfrm_policy_gc_lock);

267
	hlist_for_each_entry_safe(policy, entry, tmp, &gc_list, bydst)
Linus Torvalds's avatar
Linus Torvalds committed
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
		xfrm_policy_gc_kill(policy);
}

/* Rule must be locked. Release descentant resources, announce
 * entry dead. The rule must be unlinked from lists to the moment.
 */

static void xfrm_policy_kill(struct xfrm_policy *policy)
{
	int dead;

	write_lock_bh(&policy->lock);
	dead = policy->dead;
	policy->dead = 1;
	write_unlock_bh(&policy->lock);

	if (unlikely(dead)) {
		WARN_ON(1);
		return;
	}

	spin_lock(&xfrm_policy_gc_lock);
290
	hlist_add_head(&policy->bydst, &xfrm_policy_gc_list);
Linus Torvalds's avatar
Linus Torvalds committed
291
292
293
294
295
	spin_unlock(&xfrm_policy_gc_lock);

	schedule_work(&xfrm_policy_gc_work);
}

296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
struct xfrm_policy_hash {
	struct hlist_head	*table;
	unsigned int		hmask;
};

static struct hlist_head xfrm_policy_inexact[XFRM_POLICY_MAX*2];
static struct xfrm_policy_hash xfrm_policy_bydst[XFRM_POLICY_MAX*2] __read_mostly;
static struct hlist_head *xfrm_policy_byidx __read_mostly;
static unsigned int xfrm_idx_hmask __read_mostly;
static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;

static inline unsigned int idx_hash(u32 index)
{
	return __idx_hash(index, xfrm_idx_hmask);
}

static struct hlist_head *policy_hash_bysel(struct xfrm_selector *sel, unsigned short family, int dir)
{
	unsigned int hmask = xfrm_policy_bydst[dir].hmask;
	unsigned int hash = __sel_hash(sel, family, hmask);

	return (hash == hmask + 1 ?
		&xfrm_policy_inexact[dir] :
		xfrm_policy_bydst[dir].table + hash);
}

static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
{
	unsigned int hmask = xfrm_policy_bydst[dir].hmask;
	unsigned int hash = __addr_hash(daddr, saddr, family, hmask);

	return xfrm_policy_bydst[dir].table + hash;
}

static void xfrm_dst_hash_transfer(struct hlist_head *list,
				   struct hlist_head *ndsttable,
				   unsigned int nhashmask)
{
334
	struct hlist_node *entry, *tmp, *entry0 = NULL;
335
	struct xfrm_policy *pol;
336
	unsigned int h0 = 0;
337

338
redo:
339
340
341
342
343
	hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {
		unsigned int h;

		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
				pol->family, nhashmask);
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
		if (!entry0) {
			hlist_del(entry);
			hlist_add_head(&pol->bydst, ndsttable+h);
			h0 = h;
		} else {
			if (h != h0)
				continue;
			hlist_del(entry);
			hlist_add_after(entry0, &pol->bydst);
		}
		entry0 = entry;
	}
	if (!hlist_empty(list)) {
		entry0 = NULL;
		goto redo;
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
	}
}

static void xfrm_idx_hash_transfer(struct hlist_head *list,
				   struct hlist_head *nidxtable,
				   unsigned int nhashmask)
{
	struct hlist_node *entry, *tmp;
	struct xfrm_policy *pol;

	hlist_for_each_entry_safe(pol, entry, tmp, list, byidx) {
		unsigned int h;

		h = __idx_hash(pol->index, nhashmask);
		hlist_add_head(&pol->byidx, nidxtable+h);
	}
}

static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
{
	return ((old_hmask + 1) << 1) - 1;
}

static void xfrm_bydst_resize(int dir)
{
	unsigned int hmask = xfrm_policy_bydst[dir].hmask;
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
	struct hlist_head *odst = xfrm_policy_bydst[dir].table;
388
	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
	int i;

	if (!ndst)
		return;

	write_lock_bh(&xfrm_policy_lock);

	for (i = hmask; i >= 0; i--)
		xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);

	xfrm_policy_bydst[dir].table = ndst;
	xfrm_policy_bydst[dir].hmask = nhashmask;

	write_unlock_bh(&xfrm_policy_lock);

404
	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
405
406
407
408
409
410
411
412
}

static void xfrm_byidx_resize(int total)
{
	unsigned int hmask = xfrm_idx_hmask;
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
	struct hlist_head *oidx = xfrm_policy_byidx;
413
	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
	int i;

	if (!nidx)
		return;

	write_lock_bh(&xfrm_policy_lock);

	for (i = hmask; i >= 0; i--)
		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);

	xfrm_policy_byidx = nidx;
	xfrm_idx_hmask = nhashmask;

	write_unlock_bh(&xfrm_policy_lock);

429
	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
}

static inline int xfrm_bydst_should_resize(int dir, int *total)
{
	unsigned int cnt = xfrm_policy_count[dir];
	unsigned int hmask = xfrm_policy_bydst[dir].hmask;

	if (total)
		*total += cnt;

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    cnt > hmask)
		return 1;

	return 0;
}

static inline int xfrm_byidx_should_resize(int total)
{
	unsigned int hmask = xfrm_idx_hmask;

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    total > hmask)
		return 1;

	return 0;
}

458
void xfrm_spd_getinfo(struct xfrmk_spdinfo *si)
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
459
460
461
462
463
464
465
466
467
468
469
470
471
{
	read_lock_bh(&xfrm_policy_lock);
	si->incnt = xfrm_policy_count[XFRM_POLICY_IN];
	si->outcnt = xfrm_policy_count[XFRM_POLICY_OUT];
	si->fwdcnt = xfrm_policy_count[XFRM_POLICY_FWD];
	si->inscnt = xfrm_policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
	si->outscnt = xfrm_policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
	si->fwdscnt = xfrm_policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
	si->spdhcnt = xfrm_idx_hmask;
	si->spdhmcnt = xfrm_policy_hashmax;
	read_unlock_bh(&xfrm_policy_lock);
}
EXPORT_SYMBOL(xfrm_spd_getinfo);
472

Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
473
static DEFINE_MUTEX(hash_resize_mutex);
David Howells's avatar
David Howells committed
474
static void xfrm_hash_resize(struct work_struct *__unused)
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
{
	int dir, total;

	mutex_lock(&hash_resize_mutex);

	total = 0;
	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
		if (xfrm_bydst_should_resize(dir, &total))
			xfrm_bydst_resize(dir);
	}
	if (xfrm_byidx_should_resize(total))
		xfrm_byidx_resize(total);

	mutex_unlock(&hash_resize_mutex);
}

David Howells's avatar
David Howells committed
491
static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize);
492

Linus Torvalds's avatar
Linus Torvalds committed
493
494
/* Generate new index... KAME seems to generate them ordered by cost
 * of an absolute inpredictability of ordering of rules. This will not pass. */
495
static u32 xfrm_gen_index(u8 type, int dir)
Linus Torvalds's avatar
Linus Torvalds committed
496
497
498
499
{
	static u32 idx_generator;

	for (;;) {
500
501
502
503
504
505
		struct hlist_node *entry;
		struct hlist_head *list;
		struct xfrm_policy *p;
		u32 idx;
		int found;

Linus Torvalds's avatar
Linus Torvalds committed
506
507
508
509
		idx = (idx_generator | dir);
		idx_generator += 8;
		if (idx == 0)
			idx = 8;
510
511
512
513
514
		list = xfrm_policy_byidx + idx_hash(idx);
		found = 0;
		hlist_for_each_entry(p, entry, list, byidx) {
			if (p->index == idx) {
				found = 1;
Linus Torvalds's avatar
Linus Torvalds committed
515
				break;
516
			}
Linus Torvalds's avatar
Linus Torvalds committed
517
		}
518
		if (!found)
Linus Torvalds's avatar
Linus Torvalds committed
519
520
521
522
			return idx;
	}
}

523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
{
	u32 *p1 = (u32 *) s1;
	u32 *p2 = (u32 *) s2;
	int len = sizeof(struct xfrm_selector) / sizeof(u32);
	int i;

	for (i = 0; i < len; i++) {
		if (p1[i] != p2[i])
			return 1;
	}

	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
538
539
int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
{
540
541
542
	struct xfrm_policy *pol;
	struct xfrm_policy *delpol;
	struct hlist_head *chain;
Herbert Xu's avatar
Herbert Xu committed
543
	struct hlist_node *entry, *newpos;
544
	struct dst_entry *gc_list;
Linus Torvalds's avatar
Linus Torvalds committed
545
546

	write_lock_bh(&xfrm_policy_lock);
547
548
549
550
	chain = policy_hash_bysel(&policy->selector, policy->family, dir);
	delpol = NULL;
	newpos = NULL;
	hlist_for_each_entry(pol, entry, chain, bydst) {
Herbert Xu's avatar
Herbert Xu committed
551
		if (pol->type == policy->type &&
552
		    !selector_cmp(&pol->selector, &policy->selector) &&
Herbert Xu's avatar
Herbert Xu committed
553
554
		    xfrm_sec_ctx_match(pol->security, policy->security) &&
		    !WARN_ON(delpol)) {
Linus Torvalds's avatar
Linus Torvalds committed
555
556
557
558
559
560
561
562
			if (excl) {
				write_unlock_bh(&xfrm_policy_lock);
				return -EEXIST;
			}
			delpol = pol;
			if (policy->priority > pol->priority)
				continue;
		} else if (policy->priority >= pol->priority) {
Herbert Xu's avatar
Herbert Xu committed
563
			newpos = &pol->bydst;
Linus Torvalds's avatar
Linus Torvalds committed
564
565
566
567
568
569
			continue;
		}
		if (delpol)
			break;
	}
	if (newpos)
570
571
572
		hlist_add_after(newpos, &policy->bydst);
	else
		hlist_add_head(&policy->bydst, chain);
Linus Torvalds's avatar
Linus Torvalds committed
573
	xfrm_pol_hold(policy);
574
	xfrm_policy_count[dir]++;
Linus Torvalds's avatar
Linus Torvalds committed
575
	atomic_inc(&flow_cache_genid);
576
577
578
579
580
	if (delpol) {
		hlist_del(&delpol->bydst);
		hlist_del(&delpol->byidx);
		xfrm_policy_count[dir]--;
	}
581
	policy->index = delpol ? delpol->index : xfrm_gen_index(policy->type, dir);
582
	hlist_add_head(&policy->byidx, xfrm_policy_byidx+idx_hash(policy->index));
583
	policy->curlft.add_time = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
584
585
586
587
588
	policy->curlft.use_time = 0;
	if (!mod_timer(&policy->timer, jiffies + HZ))
		xfrm_pol_hold(policy);
	write_unlock_bh(&xfrm_policy_lock);

589
	if (delpol)
Linus Torvalds's avatar
Linus Torvalds committed
590
		xfrm_policy_kill(delpol);
591
592
	else if (xfrm_bydst_should_resize(dir, NULL))
		schedule_work(&xfrm_hash_work);
593
594
595

	read_lock_bh(&xfrm_policy_lock);
	gc_list = NULL;
596
597
	entry = &policy->bydst;
	hlist_for_each_entry_continue(policy, entry, bydst) {
598
599
600
601
602
603
604
605
606
607
608
609
610
611
		struct dst_entry *dst;

		write_lock(&policy->lock);
		dst = policy->bundles;
		if (dst) {
			struct dst_entry *tail = dst;
			while (tail->next)
				tail = tail->next;
			tail->next = gc_list;
			gc_list = dst;

			policy->bundles = NULL;
		}
		write_unlock(&policy->lock);
Linus Torvalds's avatar
Linus Torvalds committed
612
	}
613
614
615
616
617
618
619
620
621
	read_unlock_bh(&xfrm_policy_lock);

	while (gc_list) {
		struct dst_entry *dst = gc_list;

		gc_list = dst->next;
		dst_free(dst);
	}

Linus Torvalds's avatar
Linus Torvalds committed
622
623
624
625
	return 0;
}
EXPORT_SYMBOL(xfrm_policy_insert);

626
627
struct xfrm_policy *xfrm_policy_bysel_ctx(u8 type, int dir,
					  struct xfrm_selector *sel,
628
629
					  struct xfrm_sec_ctx *ctx, int delete,
					  int *err)
Linus Torvalds's avatar
Linus Torvalds committed
630
{
631
632
633
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
	struct hlist_node *entry;
Linus Torvalds's avatar
Linus Torvalds committed
634

635
	*err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
636
	write_lock_bh(&xfrm_policy_lock);
637
638
639
640
641
642
	chain = policy_hash_bysel(sel, sel->family, dir);
	ret = NULL;
	hlist_for_each_entry(pol, entry, chain, bydst) {
		if (pol->type == type &&
		    !selector_cmp(sel, &pol->selector) &&
		    xfrm_sec_ctx_match(ctx, pol->security)) {
Linus Torvalds's avatar
Linus Torvalds committed
643
			xfrm_pol_hold(pol);
644
			if (delete) {
645
646
647
648
649
				*err = security_xfrm_policy_delete(pol);
				if (*err) {
					write_unlock_bh(&xfrm_policy_lock);
					return pol;
				}
650
651
652
653
654
				hlist_del(&pol->bydst);
				hlist_del(&pol->byidx);
				xfrm_policy_count[dir]--;
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
655
656
657
658
659
			break;
		}
	}
	write_unlock_bh(&xfrm_policy_lock);

660
	if (ret && delete) {
Linus Torvalds's avatar
Linus Torvalds committed
661
		atomic_inc(&flow_cache_genid);
662
		xfrm_policy_kill(ret);
Linus Torvalds's avatar
Linus Torvalds committed
663
	}
664
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
665
}
666
EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
Linus Torvalds's avatar
Linus Torvalds committed
667

668
669
struct xfrm_policy *xfrm_policy_byid(u8 type, int dir, u32 id, int delete,
				     int *err)
Linus Torvalds's avatar
Linus Torvalds committed
670
{
671
672
673
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
	struct hlist_node *entry;
Linus Torvalds's avatar
Linus Torvalds committed
674

675
676
677
678
	*err = -ENOENT;
	if (xfrm_policy_id2dir(id) != dir)
		return NULL;

679
	*err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
680
	write_lock_bh(&xfrm_policy_lock);
681
682
683
684
	chain = xfrm_policy_byidx + idx_hash(id);
	ret = NULL;
	hlist_for_each_entry(pol, entry, chain, byidx) {
		if (pol->type == type && pol->index == id) {
Linus Torvalds's avatar
Linus Torvalds committed
685
			xfrm_pol_hold(pol);
686
			if (delete) {
687
688
689
690
691
				*err = security_xfrm_policy_delete(pol);
				if (*err) {
					write_unlock_bh(&xfrm_policy_lock);
					return pol;
				}
692
693
694
695
696
				hlist_del(&pol->bydst);
				hlist_del(&pol->byidx);
				xfrm_policy_count[dir]--;
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
697
698
699
700
701
			break;
		}
	}
	write_unlock_bh(&xfrm_policy_lock);

702
	if (ret && delete) {
Linus Torvalds's avatar
Linus Torvalds committed
703
		atomic_inc(&flow_cache_genid);
704
		xfrm_policy_kill(ret);
Linus Torvalds's avatar
Linus Torvalds committed
705
	}
706
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
707
708
709
}
EXPORT_SYMBOL(xfrm_policy_byid);

710
711
712
#ifdef CONFIG_SECURITY_NETWORK_XFRM
static inline int
xfrm_policy_flush_secctx_check(u8 type, struct xfrm_audit *audit_info)
Linus Torvalds's avatar
Linus Torvalds committed
713
{
714
715
716
717
718
719
720
721
722
723
724
725
726
	int dir, err = 0;

	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
		struct xfrm_policy *pol;
		struct hlist_node *entry;
		int i;

		hlist_for_each_entry(pol, entry,
				     &xfrm_policy_inexact[dir], bydst) {
			if (pol->type != type)
				continue;
			err = security_xfrm_policy_delete(pol);
			if (err) {
Joy Latten's avatar
Joy Latten committed
727
728
729
				xfrm_audit_policy_delete(pol, 0,
							 audit_info->loginuid,
							 audit_info->secid);
730
731
				return err;
			}
732
		}
733
734
735
736
737
738
739
740
		for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
			hlist_for_each_entry(pol, entry,
					     xfrm_policy_bydst[dir].table + i,
					     bydst) {
				if (pol->type != type)
					continue;
				err = security_xfrm_policy_delete(pol);
				if (err) {
Joy Latten's avatar
Joy Latten committed
741
742
743
					xfrm_audit_policy_delete(pol, 0,
							audit_info->loginuid,
							audit_info->secid);
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
					return err;
				}
			}
		}
	}
	return err;
}
#else
static inline int
xfrm_policy_flush_secctx_check(u8 type, struct xfrm_audit *audit_info)
{
	return 0;
}
#endif

int xfrm_policy_flush(u8 type, struct xfrm_audit *audit_info)
{
	int dir, err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
762
763

	write_lock_bh(&xfrm_policy_lock);
764
765
766
767
768

	err = xfrm_policy_flush_secctx_check(type, audit_info);
	if (err)
		goto out;

Linus Torvalds's avatar
Linus Torvalds committed
769
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
770
771
		struct xfrm_policy *pol;
		struct hlist_node *entry;
772
		int i, killed;
773

774
		killed = 0;
775
776
777
778
779
780
781
	again1:
		hlist_for_each_entry(pol, entry,
				     &xfrm_policy_inexact[dir], bydst) {
			if (pol->type != type)
				continue;
			hlist_del(&pol->bydst);
			hlist_del(&pol->byidx);
Linus Torvalds's avatar
Linus Torvalds committed
782
783
			write_unlock_bh(&xfrm_policy_lock);

Joy Latten's avatar
Joy Latten committed
784
785
			xfrm_audit_policy_delete(pol, 1, audit_info->loginuid,
						 audit_info->secid);
Joy Latten's avatar
Joy Latten committed
786

787
			xfrm_policy_kill(pol);
788
			killed++;
Linus Torvalds's avatar
Linus Torvalds committed
789
790

			write_lock_bh(&xfrm_policy_lock);
791
792
793
794
795
796
797
798
799
800
801
802
803
804
			goto again1;
		}

		for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
	again2:
			hlist_for_each_entry(pol, entry,
					     xfrm_policy_bydst[dir].table + i,
					     bydst) {
				if (pol->type != type)
					continue;
				hlist_del(&pol->bydst);
				hlist_del(&pol->byidx);
				write_unlock_bh(&xfrm_policy_lock);

Joy Latten's avatar
Joy Latten committed
805
806
807
				xfrm_audit_policy_delete(pol, 1,
							 audit_info->loginuid,
							 audit_info->secid);
808
				xfrm_policy_kill(pol);
809
				killed++;
810
811
812
813

				write_lock_bh(&xfrm_policy_lock);
				goto again2;
			}
Linus Torvalds's avatar
Linus Torvalds committed
814
		}
815

816
		xfrm_policy_count[dir] -= killed;
Linus Torvalds's avatar
Linus Torvalds committed
817
818
	}
	atomic_inc(&flow_cache_genid);
819
out:
Linus Torvalds's avatar
Linus Torvalds committed
820
	write_unlock_bh(&xfrm_policy_lock);
821
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
822
823
824
}
EXPORT_SYMBOL(xfrm_policy_flush);

825
int xfrm_policy_walk(u8 type, int (*func)(struct xfrm_policy *, int, int, void*),
Linus Torvalds's avatar
Linus Torvalds committed
826
827
		     void *data)
{
828
	struct xfrm_policy *pol, *last = NULL;
829
	struct hlist_node *entry;
830
	int dir, last_dir = 0, count, error;
Linus Torvalds's avatar
Linus Torvalds committed
831
832

	read_lock_bh(&xfrm_policy_lock);
833
	count = 0;
Linus Torvalds's avatar
Linus Torvalds committed
834
835

	for (dir = 0; dir < 2*XFRM_POLICY_MAX; dir++) {
836
837
838
839
840
841
842
		struct hlist_head *table = xfrm_policy_bydst[dir].table;
		int i;

		hlist_for_each_entry(pol, entry,
				     &xfrm_policy_inexact[dir], bydst) {
			if (pol->type != type)
				continue;
843
844
845
846
847
848
849
850
851
			if (last) {
				error = func(last, last_dir % XFRM_POLICY_MAX,
					     count, data);
				if (error)
					goto out;
			}
			last = pol;
			last_dir = dir;
			count++;
Linus Torvalds's avatar
Linus Torvalds committed
852
		}
853
854
855
856
		for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
			hlist_for_each_entry(pol, entry, table + i, bydst) {
				if (pol->type != type)
					continue;
857
858
859
860
861
862
863
864
865
				if (last) {
					error = func(last, last_dir % XFRM_POLICY_MAX,
						     count, data);
					if (error)
						goto out;
				}
				last = pol;
				last_dir = dir;
				count++;
866
867
			}
		}
Linus Torvalds's avatar
Linus Torvalds committed
868
	}
869
870
871
872
873
	if (count == 0) {
		error = -ENOENT;
		goto out;
	}
	error = func(last, last_dir % XFRM_POLICY_MAX, 0, data);
Linus Torvalds's avatar
Linus Torvalds committed
874
875
876
877
878
879
out:
	read_unlock_bh(&xfrm_policy_lock);
	return error;
}
EXPORT_SYMBOL(xfrm_policy_walk);

880
881
882
883
884
/*
 * Find policy to apply to this flow.
 *
 * Returns 0 if policy found, else an -errno.
 */
885
886
static int xfrm_policy_match(struct xfrm_policy *pol, struct flowi *fl,
			     u8 type, u16 family, int dir)
Linus Torvalds's avatar
Linus Torvalds committed
887
{
888
	struct xfrm_selector *sel = &pol->selector;
889
	int match, ret = -ESRCH;
Linus Torvalds's avatar
Linus Torvalds committed
890

891
892
	if (pol->family != family ||
	    pol->type != type)
893
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
894

895
	match = xfrm_selector_match(sel, fl, family);
896
897
	if (match)
		ret = security_xfrm_policy_lookup(pol, fl->secid, dir);
898

899
	return ret;
900
}
Linus Torvalds's avatar
Linus Torvalds committed
901

902
903
904
static struct xfrm_policy *xfrm_policy_lookup_bytype(u8 type, struct flowi *fl,
						     u16 family, u8 dir)
{
905
	int err;
906
907
908
909
	struct xfrm_policy *pol, *ret;
	xfrm_address_t *daddr, *saddr;
	struct hlist_node *entry;
	struct hlist_head *chain;
910
	u32 priority = ~0U;
911

912
913
914
915
916
917
918
919
920
	daddr = xfrm_flowi_daddr(fl, family);
	saddr = xfrm_flowi_saddr(fl, family);
	if (unlikely(!daddr || !saddr))
		return NULL;

	read_lock_bh(&xfrm_policy_lock);
	chain = policy_hash_direct(daddr, saddr, family, dir);
	ret = NULL;
	hlist_for_each_entry(pol, entry, chain, bydst) {
921
922
923
924
925
926
927
928
929
		err = xfrm_policy_match(pol, fl, type, family, dir);
		if (err) {
			if (err == -ESRCH)
				continue;
			else {
				ret = ERR_PTR(err);
				goto fail;
			}
		} else {
930
			ret = pol;
931
			priority = ret->priority;
932
933
934
			break;
		}
	}
935
936
	chain = &xfrm_policy_inexact[dir];
	hlist_for_each_entry(pol, entry, chain, bydst) {
937
938
939
940
941
942
943
944
945
		err = xfrm_policy_match(pol, fl, type, family, dir);
		if (err) {
			if (err == -ESRCH)
				continue;
			else {
				ret = ERR_PTR(err);
				goto fail;
			}
		} else if (pol->priority < priority) {
946
947
			ret = pol;
			break;
Linus Torvalds's avatar
Linus Torvalds committed
948
949
		}
	}
950
951
	if (ret)
		xfrm_pol_hold(ret);
952
fail:
Linus Torvalds's avatar
Linus Torvalds committed
953
	read_unlock_bh(&xfrm_policy_lock);
954

955
	return ret;
956
957
}

958
static int xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
959
960
961
			       void **objp, atomic_t **obj_refp)
{
	struct xfrm_policy *pol;
962
	int err = 0;
963
964
965

#ifdef CONFIG_XFRM_SUB_POLICY
	pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_SUB, fl, family, dir);
966
967
968
969
970
	if (IS_ERR(pol)) {
		err = PTR_ERR(pol);
		pol = NULL;
	}
	if (pol || err)
971
972
973
		goto end;
#endif
	pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN, fl, family, dir);
974
975
976
977
	if (IS_ERR(pol)) {
		err = PTR_ERR(pol);
		pol = NULL;
	}
978
#ifdef CONFIG_XFRM_SUB_POLICY
979
end:
980
#endif
Linus Torvalds's avatar
Linus Torvalds committed
981
982
	if ((*objp = (void *) pol) != NULL)
		*obj_refp = &pol->refcnt;
983
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
984
985
}

986
987
988
static inline int policy_to_flow_dir(int dir)
{
	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
989
990
991
992
993
994
995
996
997
998
999
	    XFRM_POLICY_OUT == FLOW_DIR_OUT &&
	    XFRM_POLICY_FWD == FLOW_DIR_FWD)
		return dir;
	switch (dir) {
	default:
	case XFRM_POLICY_IN:
		return FLOW_DIR_IN;
	case XFRM_POLICY_OUT:
		return FLOW_DIR_OUT;
	case XFRM_POLICY_FWD:
		return FLOW_DIR_FWD;
1000
	}
1001
1002
}

1003
static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir, struct flowi *fl)
Linus Torvalds's avatar
Linus Torvalds committed
1004
1005
1006
1007
1008
{
	struct xfrm_policy *pol;

	read_lock_bh(&xfrm_policy_lock);
	if ((pol = sk->sk_policy[dir]) != NULL) {
1009
		int match = xfrm_selector_match(&pol->selector, fl,
Linus Torvalds's avatar
Linus Torvalds committed
1010
						sk->sk_family);
1011
		int err = 0;
1012

1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
		if (match) {
			err = security_xfrm_policy_lookup(pol, fl->secid,
					policy_to_flow_dir(dir));
			if (!err)
				xfrm_pol_hold(pol);
			else if (err == -ESRCH)
				pol = NULL;
			else
				pol = ERR_PTR(err);
		} else
Linus Torvalds's avatar
Linus Torvalds committed
1023
1024
1025
1026
1027
1028
1029
1030
			pol = NULL;
	}
	read_unlock_bh(&xfrm_policy_lock);
	return pol;
}

static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
{
1031
1032
	struct hlist_head *chain = policy_hash_bysel(&pol->selector,
						     pol->family, dir);
1033

1034
1035
1036
	hlist_add_head(&pol->bydst, chain);
	hlist_add_head(&pol->byidx, xfrm_policy_byidx+idx_hash(pol->index));
	xfrm_policy_count[dir]++;
Linus Torvalds's avatar
Linus Torvalds committed
1037
	xfrm_pol_hold(pol);
1038
1039
1040

	if (xfrm_bydst_should_resize(dir, NULL))
		schedule_work(&xfrm_hash_work);
Linus Torvalds's avatar
Linus Torvalds committed
1041
1042
1043
1044
1045
}

static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
						int dir)
{
1046
1047
	if (hlist_unhashed(&pol->bydst))
		return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1048

1049
1050
1051
1052
1053
	hlist_del(&pol->bydst);
	hlist_del(&pol->byidx);
	xfrm_policy_count[dir]--;

	return pol;
Linus Torvalds's avatar
Linus Torvalds committed
1054
1055
}

1056
int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
Linus Torvalds's avatar
Linus Torvalds committed
1057
1058
1059
1060
1061
1062
1063
1064
{
	write_lock_bh(&xfrm_policy_lock);
	pol = __xfrm_policy_unlink(pol, dir);
	write_unlock_bh(&xfrm_policy_lock);
	if (pol) {
		if (dir < XFRM_POLICY_MAX)
			atomic_inc(&flow_cache_genid);
		xfrm_policy_kill(pol);
1065
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1066
	}
1067
	return -ENOENT;
Linus Torvalds's avatar
Linus Torvalds committed
1068
}
1069
EXPORT_SYMBOL(xfrm_policy_delete);
Linus Torvalds's avatar
Linus Torvalds committed
1070
1071
1072
1073
1074

int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
{
	struct xfrm_policy *old_pol;

1075
1076
1077
1078
1079
#ifdef CONFIG_XFRM_SUB_POLICY
	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
		return -EINVAL;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1080
1081
1082
1083
	write_lock_bh(&xfrm_policy_lock);
	old_pol = sk->sk_policy[dir];
	sk->sk_policy[dir] = pol;
	if (pol) {
1084
		pol->curlft.add_time = get_seconds();
1085
		pol->index = xfrm_gen_index(pol->type, XFRM_POLICY_MAX+dir);
Linus Torvalds's avatar
Linus Torvalds committed
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
		__xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
	}
	if (old_pol)
		__xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
	write_unlock_bh(&xfrm_policy_lock);

	if (old_pol) {
		xfrm_policy_kill(old_pol);
	}
	return 0;
}

static struct xfrm_policy *clone_policy(struct xfrm_policy *old, int dir)
{
	struct xfrm_policy *newp = xfrm_policy_alloc(GFP_ATOMIC);

	if (newp) {
		newp->selector = old->selector;
1104
1105
1106
1107
		if (security_xfrm_policy_clone(old, newp)) {
			kfree(newp);
			return NULL;  /* ENOMEM */
		}
Linus Torvalds's avatar
Linus Torvalds committed
1108
1109
1110
1111
1112
1113
		newp->lft = old->lft;
		newp->curlft = old->curlft;
		newp->action = old->action;
		newp->flags = old->flags;
		newp->xfrm_nr = old->xfrm_nr;
		newp->index = old->index;
1114
		newp->type = old->type;
Linus Torvalds's avatar
Linus Torvalds committed
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
		memcpy(newp->xfrm_vec, old->xfrm_vec,
		       newp->xfrm_nr*sizeof(struct xfrm_tmpl));
		write_lock_bh(&xfrm_policy_lock);
		__xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
		write_unlock_bh(&xfrm_policy_lock);
		xfrm_pol_put(newp);
	}
	return newp;
}

int __xfrm_sk_clone_policy(struct sock *sk)
{
	struct xfrm_policy *p0 = sk->sk_policy[0],
			   *p1 = sk->sk_policy[1];

	sk->sk_policy[0] = sk->sk_policy[1] = NULL;
	if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
		return -ENOMEM;
	if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
		return -ENOMEM;
	return 0;
}

1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
static int
xfrm_get_saddr(xfrm_address_t *local, xfrm_address_t *remote,
	       unsigned short family)
{
	int err;
	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);

	if (unlikely(afinfo == NULL))
		return -EINVAL;
	err = afinfo->get_saddr(local, remote);
	xfrm_policy_put_afinfo(afinfo);
	return err;
}

Linus Torvalds's avatar
Linus Torvalds committed
1152
1153
1154
/* Resolve list of templates for the flow, given policy. */

static int
1155
1156
1157
xfrm_tmpl_resolve_one(struct xfrm_policy *policy, struct flowi *fl,
		      struct xfrm_state **xfrm,
		      unsigned short family)
Linus Torvalds's avatar
Linus Torvalds committed
1158
1159
1160
1161
1162
{
	int nx;
	int i, error;
	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1163
	xfrm_address_t tmp;
Linus Torvalds's avatar
Linus Torvalds committed
1164
1165
1166
1167
1168
1169
1170

	for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
		struct xfrm_state *x;
		xfrm_address_t *remote = daddr;
		xfrm_address_t *local  = saddr;
		struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];

1171
1172
		if (tmpl->mode == XFRM_MODE_TUNNEL ||
		    tmpl->mode == XFRM_MODE_BEET) {
Linus Torvalds's avatar
Linus Torvalds committed
1173
1174
			remote = &tmpl->id.daddr;
			local = &tmpl->saddr;
1175
			family = tmpl->encap_family;
1176
1177
1178
1179
1180
1181
			if (xfrm_addr_any(local, family)) {
				error = xfrm_get_saddr(&tmp, remote, family);
				if (error)
					goto fail;
				local = &tmp;
			}
Linus Torvalds's avatar
Linus Torvalds committed
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
		}

		x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);

		if (x && x->km.state == XFRM_STATE_VALID) {
			xfrm[nx++] = x;
			daddr = remote;
			saddr = local;
			continue;
		}
		if (x) {
			error = (x->km.state == XFRM_STATE_ERROR ?
				 -EINVAL : -EAGAIN);
			xfrm_state_put(x);
		}

		if (!tmpl->optional)
			goto fail;
	}
	return nx;

fail:
	for (nx--; nx>=0; nx--)
		xfrm_state_put(xfrm[nx]);
	return error;
}

1209
1210
1211
1212
1213
static int
xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, struct flowi *fl,
		  struct xfrm_state **xfrm,
		  unsigned short family)
{
1214
1215
	struct xfrm_state *tp[XFRM_MAX_DEPTH];
	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
	int cnx = 0;
	int error;
	int ret;
	int i;

	for (i = 0; i < npols; i++) {
		if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
			error = -ENOBUFS;
			goto fail;
		}
1226
1227

		ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1228
1229
1230
1231
1232
1233
1234
		if (ret < 0) {
			error = ret;
			goto fail;
		} else
			cnx += ret;
	}

1235
1236
1237
1238
	/* found states are sorted for outbound processing */
	if (npols > 1)
		xfrm_state_sort(xfrm, tpp, cnx, family);

1239
1240
1241
1242
	return cnx;

 fail:
	for (cnx--; cnx>=0; cnx--)
1243
		xfrm_state_put(tpp[cnx]);
1244
1245
1246
1247
	return error;

}

Linus Torvalds's avatar
Linus Torvalds committed
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
/* Check that the bundle accepts the flow and its components are
 * still valid.
 */

static struct dst_entry *
xfrm_find_bundle(struct flowi *fl, struct xfrm_policy *policy, unsigned short family)
{
	struct dst_entry *x;
	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
	if (unlikely(afinfo == NULL))
		return ERR_PTR(-EINVAL);
	x = afinfo->find_bundle(fl, policy);
	xfrm_policy_put_afinfo(afinfo);
	return x;
}

1264
1265
1266
1267
static inline int xfrm_get_tos(struct flowi *fl, int family)
{
	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
	int tos;
Linus Torvalds's avatar
Linus Torvalds committed
1268

1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
	if (!afinfo)
		return -EINVAL;

	tos = afinfo->get_tos(fl);

	xfrm_policy_put_afinfo(afinfo);

	return tos;
}

static inline struct xfrm_dst *xfrm_alloc_dst(int family)
Linus Torvalds's avatar
Linus Torvalds committed
1280
1281
{
	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
	struct xfrm_dst *xdst;

	if (!afinfo)
		return ERR_PTR(-EINVAL);

	xdst = dst_alloc(afinfo->dst_ops) ?: ERR_PTR(-ENOBUFS);

	xfrm_policy_put_afinfo(afinfo);

	return xdst;
}

1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
				 int nfheader_len)
{
	struct xfrm_policy_afinfo *afinfo =
		xfrm_policy_get_afinfo(dst->ops->family);
	int err;

	if (!afinfo)
		return -EINVAL;

	err = afinfo->init_path(path, dst, nfheader_len);

	xfrm_policy_put_afinfo(afinfo);

	return err;
}

1311
1312
1313
1314
1315
1316
1317
static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev)
{
	struct xfrm_policy_afinfo *afinfo =
		xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
	int err;

	if (!afinfo)
Linus Torvalds's avatar
Linus Torvalds committed
1318
		return -EINVAL;
1319
1320
1321

	err = afinfo->fill_dst(xdst, dev);

Linus Torvalds's avatar
Linus Torvalds committed
1322
	xfrm_policy_put_afinfo(afinfo);
1323

Linus Torvalds's avatar
Linus Torvalds committed
1324
1325
1326
	return err;
}

1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
/* Allocate chain of dst_entry's, attach known xfrm's, calculate
 * all the metrics... Shortly, bundle a bundle.
 */

static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
					    struct xfrm_state **xfrm, int nx,
					    struct flowi *fl,
					    struct dst_entry *dst)
{
	unsigned long now = jiffies;
	struct net_device *dev;
	struct dst_entry *dst_prev = NULL;
	struct dst_entry *dst0 = NULL;
	int i = 0;
	int err;
	int header_len = 0;
1343
	int nfheader_len = 0;
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
	int trailer_len = 0;
	int tos;
	int family = policy->selector.family;

	tos = xfrm_get_tos(fl, family);
	err = tos;
	if (tos < 0)
		goto put_states;

	dst_hold(dst);

	for (; i < nx; i++) {
		struct xfrm_dst *xdst = xfrm_alloc_dst(family);