skbuff.h 108 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 *	Definitions for the 'struct sk_buff' memory handlers.
 *
 *	Authors:
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Florian La Roche, <rzsfl@rz.uni-sb.de>
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 */

#ifndef _LINUX_SKBUFF_H
#define _LINUX_SKBUFF_H

#include <linux/kernel.h>
18
#include <linux/kmemcheck.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
20
#include <linux/compiler.h>
#include <linux/time.h>
21
#include <linux/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
22
#include <linux/cache.h>
23
#include <linux/rbtree.h>
24
#include <linux/socket.h>
Linus Torvalds's avatar
Linus Torvalds committed
25

Arun Sharma's avatar
Arun Sharma committed
26
#include <linux/atomic.h>
Linus Torvalds's avatar
Linus Torvalds committed
27
28
29
#include <asm/types.h>
#include <linux/spinlock.h>
#include <linux/net.h>
30
#include <linux/textsearch.h>
Linus Torvalds's avatar
Linus Torvalds committed
31
#include <net/checksum.h>
32
#include <linux/rcupdate.h>
33
#include <linux/hrtimer.h>
34
#include <linux/dma-mapping.h>
35
#include <linux/netdev_features.h>
36
#include <linux/sched.h>
37
#include <net/flow_dissector.h>
38
#include <linux/splice.h>
39
#include <linux/in6.h>
40
#include <net/flow.h>
Linus Torvalds's avatar
Linus Torvalds committed
41

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
/* The interface for checksum offload between the stack and networking drivers
 * is as follows...
 *
 * A. IP checksum related features
 *
 * Drivers advertise checksum offload capabilities in the features of a device.
 * From the stack's point of view these are capabilities offered by the driver,
 * a driver typically only advertises features that it is capable of offloading
 * to its device.
 *
 * The checksum related features are:
 *
 *	NETIF_F_HW_CSUM	- The driver (or its device) is able to compute one
 *			  IP (one's complement) checksum for any combination
 *			  of protocols or protocol layering. The checksum is
 *			  computed and set in a packet per the CHECKSUM_PARTIAL
 *			  interface (see below).
 *
 *	NETIF_F_IP_CSUM - Driver (device) is only able to checksum plain
 *			  TCP or UDP packets over IPv4. These are specifically
 *			  unencapsulated packets of the form IPv4|TCP or
 *			  IPv4|UDP where the Protocol field in the IPv4 header
 *			  is TCP or UDP. The IPv4 header may contain IP options
 *			  This feature cannot be set in features for a device
 *			  with NETIF_F_HW_CSUM also set. This feature is being
 *			  DEPRECATED (see below).
 *
 *	NETIF_F_IPV6_CSUM - Driver (device) is only able to checksum plain
 *			  TCP or UDP packets over IPv6. These are specifically
 *			  unencapsulated packets of the form IPv6|TCP or
 *			  IPv4|UDP where the Next Header field in the IPv6
 *			  header is either TCP or UDP. IPv6 extension headers
 *			  are not supported with this feature. This feature
 *			  cannot be set in features for a device with
 *			  NETIF_F_HW_CSUM also set. This feature is being
 *			  DEPRECATED (see below).
 *
 *	NETIF_F_RXCSUM - Driver (device) performs receive checksum offload.
 *			 This flag is used only used to disable the RX checksum
 *			 feature for a device. The stack will accept receive
 *			 checksum indication in packets received on a device
 *			 regardless of whether NETIF_F_RXCSUM is set.
 *
 * B. Checksumming of received packets by device. Indication of checksum
 *    verification is in set skb->ip_summed. Possible values are:
87
88
89
 *
 * CHECKSUM_NONE:
 *
90
 *   Device did not checksum this packet e.g. due to lack of capabilities.
91
92
93
94
95
96
97
 *   The packet contains full (though not verified) checksum in packet but
 *   not in skb->csum. Thus, skb->csum is undefined in this case.
 *
 * CHECKSUM_UNNECESSARY:
 *
 *   The hardware you're dealing with doesn't calculate the full checksum
 *   (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
98
99
 *   for specific protocols. For such packets it will set CHECKSUM_UNNECESSARY
 *   if their checksums are okay. skb->csum is still undefined in this case
100
101
 *   though. A driver or device must never modify the checksum field in the
 *   packet even if checksum is verified.
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 *
 *   CHECKSUM_UNNECESSARY is applicable to following protocols:
 *     TCP: IPv6 and IPv4.
 *     UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
 *       zero UDP checksum for either IPv4 or IPv6, the networking stack
 *       may perform further validation in this case.
 *     GRE: only if the checksum is present in the header.
 *     SCTP: indicates the CRC in SCTP header has been validated.
 *
 *   skb->csum_level indicates the number of consecutive checksums found in
 *   the packet minus one that have been verified as CHECKSUM_UNNECESSARY.
 *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
 *   and a device is able to verify the checksums for UDP (possibly zero),
 *   GRE (checksum flag is set), and TCP-- skb->csum_level would be set to
 *   two. If the device were only able to verify the UDP checksum and not
 *   GRE, either because it doesn't support GRE checksum of because GRE
 *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
 *   not considered in this case).
120
121
122
123
124
125
126
127
128
129
130
131
 *
 * CHECKSUM_COMPLETE:
 *
 *   This is the most generic way. The device supplied checksum of the _whole_
 *   packet as seen by netif_rx() and fills out in skb->csum. Meaning, the
 *   hardware doesn't need to parse L3/L4 headers to implement this.
 *
 *   Note: Even if device supports only some protocols, but is able to produce
 *   skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
 *
 * CHECKSUM_PARTIAL:
 *
132
133
 *   A checksum is set up to be offloaded to a device as described in the
 *   output description for CHECKSUM_PARTIAL. This may occur on a packet
134
 *   received directly from another Linux OS, e.g., a virtualized Linux kernel
135
136
137
138
139
140
 *   on the same host, or it may be set in the input path in GRO or remote
 *   checksum offload. For the purposes of checksum verification, the checksum
 *   referred to by skb->csum_start + skb->csum_offset and any preceding
 *   checksums in the packet are considered verified. Any checksums in the
 *   packet that are after the checksum being offloaded are not considered to
 *   be verified.
141
 *
142
143
 * C. Checksumming on transmit for non-GSO. The stack requests checksum offload
 *    in the skb->ip_summed for a packet. Values are:
144
145
146
 *
 * CHECKSUM_PARTIAL:
 *
147
 *   The driver is required to checksum the packet as seen by hard_start_xmit()
148
 *   from skb->csum_start up to the end, and to record/write the checksum at
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 *   offset skb->csum_start + skb->csum_offset. A driver may verify that the
 *   csum_start and csum_offset values are valid values given the length and
 *   offset of the packet, however they should not attempt to validate that the
 *   checksum refers to a legitimate transport layer checksum-- it is the
 *   purview of the stack to validate that csum_start and csum_offset are set
 *   correctly.
 *
 *   When the stack requests checksum offload for a packet, the driver MUST
 *   ensure that the checksum is set correctly. A driver can either offload the
 *   checksum calculation to the device, or call skb_checksum_help (in the case
 *   that the device does not support offload for a particular checksum).
 *
 *   NETIF_F_IP_CSUM and NETIF_F_IPV6_CSUM are being deprecated in favor of
 *   NETIF_F_HW_CSUM. New devices should use NETIF_F_HW_CSUM to indicate
 *   checksum offload capability. If a	device has limited checksum capabilities
 *   (for instance can only perform NETIF_F_IP_CSUM or NETIF_F_IPV6_CSUM as
 *   described above) a helper function can be called to resolve
 *   CHECKSUM_PARTIAL. The helper functions are skb_csum_off_chk*. The helper
 *   function takes a spec argument that describes the protocol layer that is
 *   supported for checksum offload and can be called for each packet. If a
 *   packet does not match the specification for offload, skb_checksum_help
 *   is called to resolve the checksum.
171
 *
172
 * CHECKSUM_NONE:
173
 *
174
175
 *   The skb was already checksummed by the protocol, or a checksum is not
 *   required.
176
177
178
 *
 * CHECKSUM_UNNECESSARY:
 *
179
180
 *   This has the same meaning on as CHECKSUM_NONE for checksum offload on
 *   output.
181
 *
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
 * CHECKSUM_COMPLETE:
 *   Not used in checksum output. If a driver observes a packet with this value
 *   set in skbuff, if should treat as CHECKSUM_NONE being set.
 *
 * D. Non-IP checksum (CRC) offloads
 *
 *   NETIF_F_SCTP_CRC - This feature indicates that a device is capable of
 *     offloading the SCTP CRC in a packet. To perform this offload the stack
 *     will set ip_summed to CHECKSUM_PARTIAL and set csum_start and csum_offset
 *     accordingly. Note the there is no indication in the skbuff that the
 *     CHECKSUM_PARTIAL refers to an SCTP checksum, a driver that supports
 *     both IP checksum offload and SCTP CRC offload must verify which offload
 *     is configured for a packet presumably by inspecting packet headers.
 *
 *   NETIF_F_FCOE_CRC - This feature indicates that a device is capable of
 *     offloading the FCOE CRC in a packet. To perform this offload the stack
 *     will set ip_summed to CHECKSUM_PARTIAL and set csum_start and csum_offset
 *     accordingly. Note the there is no indication in the skbuff that the
 *     CHECKSUM_PARTIAL refers to an FCOE checksum, a driver that supports
 *     both IP checksum offload and FCOE CRC offload must verify which offload
 *     is configured for a packet presumably by inspecting packet headers.
 *
 * E. Checksumming on output with GSO.
 *
 * In the case of a GSO packet (skb_is_gso(skb) is true), checksum offload
 * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
 * gso_type is SKB_GSO_TCPV4 or SKB_GSO_TCPV6, TCP checksum offload as
 * part of the GSO operation is implied. If a checksum is being offloaded
 * with GSO then ip_summed is CHECKSUM_PARTIAL, csum_start and csum_offset
 * are set to refer to the outermost checksum being offload (two offloaded
 * checksums are possible with UDP encapsulation).
213
214
 */

215
/* Don't change this without changing skb_csum_unnecessary! */
216
217
218
219
#define CHECKSUM_NONE		0
#define CHECKSUM_UNNECESSARY	1
#define CHECKSUM_COMPLETE	2
#define CHECKSUM_PARTIAL	3
Linus Torvalds's avatar
Linus Torvalds committed
220

221
222
223
/* Maximum value in skb->csum_level */
#define SKB_MAX_CSUM_LEVEL	3

224
#define SKB_DATA_ALIGN(X)	ALIGN(X, SMP_CACHE_BYTES)
225
#define SKB_WITH_OVERHEAD(X)	\
226
	((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
227
228
#define SKB_MAX_ORDER(X, ORDER) \
	SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
Linus Torvalds's avatar
Linus Torvalds committed
229
230
231
#define SKB_MAX_HEAD(X)		(SKB_MAX_ORDER((X), 0))
#define SKB_MAX_ALLOC		(SKB_MAX_ORDER(0, 2))

Eric Dumazet's avatar
Eric Dumazet committed
232
233
234
235
236
/* return minimum truesize of one skb containing X bytes of data */
#define SKB_TRUESIZE(X) ((X) +						\
			 SKB_DATA_ALIGN(sizeof(struct sk_buff)) +	\
			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

Linus Torvalds's avatar
Linus Torvalds committed
237
struct net_device;
238
struct scatterlist;
Jens Axboe's avatar
Jens Axboe committed
239
struct pipe_inode_info;
Herbert Xu's avatar
Herbert Xu committed
240
struct iov_iter;
241
struct napi_struct;
Linus Torvalds's avatar
Linus Torvalds committed
242

243
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
struct nf_conntrack {
	atomic_t use;
};
247
#endif
Linus Torvalds's avatar
Linus Torvalds committed
248

249
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
Linus Torvalds's avatar
Linus Torvalds committed
250
struct nf_bridge_info {
251
	atomic_t		use;
252
253
254
255
	enum {
		BRNF_PROTO_UNCHANGED,
		BRNF_PROTO_8021Q,
		BRNF_PROTO_PPPOE
256
	} orig_proto:8;
257
258
259
	u8			pkt_otherhost:1;
	u8			in_prerouting:1;
	u8			bridged_dnat:1;
260
	__u16			frag_max_size;
261
	struct net_device	*physindev;
262
263
264

	/* always valid & non-NULL from FORWARD on, for physdev match */
	struct net_device	*physoutdev;
265
	union {
266
		/* prerouting: detect dnat in orig/reply direction */
267
268
		__be32          ipv4_daddr;
		struct in6_addr ipv6_daddr;
269
270
271
272
273
274

		/* after prerouting + nat detected: store original source
		 * mac since neigh resolution overwrites it, only used while
		 * skb is out in neigh layer.
		 */
		char neigh_header[8];
275
	};
Linus Torvalds's avatar
Linus Torvalds committed
276
277
278
279
280
281
282
283
284
285
286
287
288
289
};
#endif

struct sk_buff_head {
	/* These two members must be first. */
	struct sk_buff	*next;
	struct sk_buff	*prev;

	__u32		qlen;
	spinlock_t	lock;
};

struct sk_buff;

290
291
292
293
294
295
/* To allow 64K frame to be packed as single skb without frag_list we
 * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
 * buffers which do not start on a page boundary.
 *
 * Since GRO uses frags we allocate at least 16 regardless of page
 * size.
296
 */
297
#if (65536/PAGE_SIZE + 1) < 16
298
#define MAX_SKB_FRAGS 16UL
299
#else
300
#define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
301
#endif
302
extern int sysctl_max_skb_frags;
Linus Torvalds's avatar
Linus Torvalds committed
303
304
305
306

typedef struct skb_frag_struct skb_frag_t;

struct skb_frag_struct {
307
308
309
	struct {
		struct page *p;
	} page;
310
#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
311
312
	__u32 page_offset;
	__u32 size;
313
314
315
316
#else
	__u16 page_offset;
	__u16 size;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
317
318
};

319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
static inline unsigned int skb_frag_size(const skb_frag_t *frag)
{
	return frag->size;
}

static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
{
	frag->size = size;
}

static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
{
	frag->size += delta;
}

static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
{
	frag->size -= delta;
}

339
340
341
#define HAVE_HW_TIME_STAMP

/**
342
 * struct skb_shared_hwtstamps - hardware time stamps
343
344
345
346
 * @hwtstamp:	hardware time stamp transformed into duration
 *		since arbitrary point in time
 *
 * Software time stamps generated by ktime_get_real() are stored in
347
 * skb->tstamp.
348
349
350
351
352
353
354
355
356
357
358
 *
 * hwtstamps can only be compared against other hwtstamps from
 * the same device.
 *
 * This structure is attached to packets as part of the
 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
 */
struct skb_shared_hwtstamps {
	ktime_t	hwtstamp;
};

359
360
361
362
363
/* Definitions for tx_flags in struct skb_shared_info */
enum {
	/* generate hardware time stamp */
	SKBTX_HW_TSTAMP = 1 << 0,

364
	/* generate software time stamp when queueing packet to NIC */
365
366
367
368
369
	SKBTX_SW_TSTAMP = 1 << 1,

	/* device driver is going to provide hardware time stamp */
	SKBTX_IN_PROGRESS = 1 << 2,

370
	/* device driver supports TX zero-copy buffers */
Eric Dumazet's avatar
Eric Dumazet committed
371
	SKBTX_DEV_ZEROCOPY = 1 << 3,
372
373

	/* generate wifi status information (where possible) */
Eric Dumazet's avatar
Eric Dumazet committed
374
	SKBTX_WIFI_STATUS = 1 << 4,
375
376
377
378
379
380
381

	/* This indicates at least one fragment might be overwritten
	 * (as in vmsplice(), sendfile() ...)
	 * If we need to compute a TX checksum, we'll need to copy
	 * all frags to avoid possible bad checksum
	 */
	SKBTX_SHARED_FRAG = 1 << 5,
382
383
384

	/* generate software time stamp when entering packet scheduling */
	SKBTX_SCHED_TSTAMP = 1 << 6,
385
386
};

387
#define SKBTX_ANY_SW_TSTAMP	(SKBTX_SW_TSTAMP    | \
388
				 SKBTX_SCHED_TSTAMP)
389
390
#define SKBTX_ANY_TSTAMP	(SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)

391
392
393
/*
 * The callback notifies userspace to release buffers when skb DMA is done in
 * lower device, the skb last reference should be 0 when calling this.
394
395
 * The zerocopy_success argument is true if zero copy transmit occurred,
 * false on data copy or out of memory error caused by data copy attempt.
396
397
 * The ctx field is used to track device context.
 * The desc field is used to track userspace buffer index.
398
399
 */
struct ubuf_info {
400
	void (*callback)(struct ubuf_info *, bool zerocopy_success);
401
	void *ctx;
402
	unsigned long desc;
403
404
};

Linus Torvalds's avatar
Linus Torvalds committed
405
406
407
408
/* This data is invariant across clones and lives at
 * the end of the header data, ie. at skb->end.
 */
struct skb_shared_info {
409
410
	unsigned char	nr_frags;
	__u8		tx_flags;
411
412
413
414
	unsigned short	gso_size;
	/* Warning: this field is not always filled in (UFO)! */
	unsigned short	gso_segs;
	unsigned short  gso_type;
Linus Torvalds's avatar
Linus Torvalds committed
415
	struct sk_buff	*frag_list;
416
	struct skb_shared_hwtstamps hwtstamps;
417
	u32		tskey;
418
	__be32          ip6_frag_id;
Eric Dumazet's avatar
Eric Dumazet committed
419
420
421
422
423
424

	/*
	 * Warning : all fields before dataref are cleared in __alloc_skb()
	 */
	atomic_t	dataref;

Johann Baudy's avatar
Johann Baudy committed
425
426
427
	/* Intermediate layers must ensure that destructor_arg
	 * remains valid until skb destructor */
	void *		destructor_arg;
428

429
430
	/* must be last field, see pskb_expand_head() */
	skb_frag_t	frags[MAX_SKB_FRAGS];
Linus Torvalds's avatar
Linus Torvalds committed
431
432
433
434
};

/* We divide dataref into two halves.  The higher 16 bits hold references
 * to the payload part of skb->data.  The lower 16 bits hold references to
435
436
 * the entire skb->data.  A clone of a headerless skb holds the length of
 * the header in skb->hdr_len.
Linus Torvalds's avatar
Linus Torvalds committed
437
438
439
440
441
442
443
444
445
446
 *
 * All users must obey the rule that the skb->data reference count must be
 * greater than or equal to the payload reference count.
 *
 * Holding a reference to the payload part means that the user does not
 * care about modifications to the header part of skb->data.
 */
#define SKB_DATAREF_SHIFT 16
#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)

447
448

enum {
449
450
451
	SKB_FCLONE_UNAVAILABLE,	/* skb has no fclone (from head_cache) */
	SKB_FCLONE_ORIG,	/* orig skb (from fclone_cache) */
	SKB_FCLONE_CLONE,	/* companion fclone skb (from fclone_cache) */
452
453
};

454
455
enum {
	SKB_GSO_TCPV4 = 1 << 0,
456
	SKB_GSO_UDP = 1 << 1,
457
458
459

	/* This indicates the skb is from an untrusted source. */
	SKB_GSO_DODGY = 1 << 2,
Michael Chan's avatar
Michael Chan committed
460
461

	/* This indicates the tcp segment has CWR set. */
462
463
	SKB_GSO_TCP_ECN = 1 << 3,

464
	SKB_GSO_TCP_FIXEDID = 1 << 4,
465

466
	SKB_GSO_TCPV6 = 1 << 5,
467

468
	SKB_GSO_FCOE = 1 << 6,
469

470
	SKB_GSO_GRE = 1 << 7,
Simon Horman's avatar
Simon Horman committed
471

472
	SKB_GSO_GRE_CSUM = 1 << 8,
Eric Dumazet's avatar
Eric Dumazet committed
473

474
	SKB_GSO_IPXIP4 = 1 << 9,
Eric Dumazet's avatar
Eric Dumazet committed
475

476
	SKB_GSO_IPXIP6 = 1 << 10,
477

478
	SKB_GSO_UDP_TUNNEL = 1 << 11,
Tom Herbert's avatar
Tom Herbert committed
479

480
481
	SKB_GSO_UDP_TUNNEL_CSUM = 1 << 12,

482
483
484
	SKB_GSO_PARTIAL = 1 << 13,

	SKB_GSO_TUNNEL_REMCSUM = 1 << 14,
485
486
};

487
488
489
490
491
492
493
494
495
496
#if BITS_PER_LONG > 32
#define NET_SKBUFF_DATA_USES_OFFSET 1
#endif

#ifdef NET_SKBUFF_DATA_USES_OFFSET
typedef unsigned int sk_buff_data_t;
#else
typedef unsigned char *sk_buff_data_t;
#endif

497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
/**
 * struct skb_mstamp - multi resolution time stamps
 * @stamp_us: timestamp in us resolution
 * @stamp_jiffies: timestamp in jiffies
 */
struct skb_mstamp {
	union {
		u64		v64;
		struct {
			u32	stamp_us;
			u32	stamp_jiffies;
		};
	};
};

/**
 * skb_mstamp_get - get current timestamp
 * @cl: place to store timestamps
 */
static inline void skb_mstamp_get(struct skb_mstamp *cl)
{
	u64 val = local_clock();

	do_div(val, NSEC_PER_USEC);
	cl->stamp_us = (u32)val;
	cl->stamp_jiffies = (u32)jiffies;
}

/**
 * skb_mstamp_delta - compute the difference in usec between two skb_mstamp
 * @t1: pointer to newest sample
 * @t0: pointer to oldest sample
 */
static inline u32 skb_mstamp_us_delta(const struct skb_mstamp *t1,
				      const struct skb_mstamp *t0)
{
	s32 delta_us = t1->stamp_us - t0->stamp_us;
	u32 delta_jiffies = t1->stamp_jiffies - t0->stamp_jiffies;

	/* If delta_us is negative, this might be because interval is too big,
	 * or local_clock() drift is too big : fallback using jiffies.
	 */
	if (delta_us <= 0 ||
	    delta_jiffies >= (INT_MAX / (USEC_PER_SEC / HZ)))

		delta_us = jiffies_to_usecs(delta_jiffies);

	return delta_us;
}

Yuchung Cheng's avatar
Yuchung Cheng committed
547
548
549
550
551
552
553
554
555
static inline bool skb_mstamp_after(const struct skb_mstamp *t1,
				    const struct skb_mstamp *t0)
{
	s32 diff = t1->stamp_jiffies - t0->stamp_jiffies;

	if (!diff)
		diff = t1->stamp_us - t0->stamp_us;
	return diff > 0;
}
556

Linus Torvalds's avatar
Linus Torvalds committed
557
558
559
560
/** 
 *	struct sk_buff - socket buffer
 *	@next: Next buffer in list
 *	@prev: Previous buffer in list
561
 *	@tstamp: Time we arrived/left
562
 *	@rbnode: RB tree node, alternative to next/prev for netem/tcp
563
 *	@sk: Socket we are owned by
Linus Torvalds's avatar
Linus Torvalds committed
564
 *	@dev: Device we arrived on/are leaving by
565
 *	@cb: Control buffer. Free for use by every layer. Put private vars here
Eric Dumazet's avatar
Eric Dumazet committed
566
 *	@_skb_refdst: destination entry (with norefcount bit)
567
 *	@sp: the security path, used for xfrm
Linus Torvalds's avatar
Linus Torvalds committed
568
569
570
 *	@len: Length of actual data
 *	@data_len: Data length
 *	@mac_len: Length of link layer header
571
 *	@hdr_len: writable header length of cloned skb
572
573
574
 *	@csum: Checksum (must include start/offset pair)
 *	@csum_start: Offset from skb->head where checksumming should start
 *	@csum_offset: Offset from csum_start where checksum should be stored
575
 *	@priority: Packet queueing priority
WANG Cong's avatar
WANG Cong committed
576
 *	@ignore_df: allow local fragmentation
Linus Torvalds's avatar
Linus Torvalds committed
577
 *	@cloned: Head may be cloned (check refcnt to be sure)
578
 *	@ip_summed: Driver fed us an IP checksum
Linus Torvalds's avatar
Linus Torvalds committed
579
 *	@nohdr: Payload reference only, must not modify header
580
 *	@nfctinfo: Relationship of this skb to the connection
Linus Torvalds's avatar
Linus Torvalds committed
581
 *	@pkt_type: Packet class
582
583
 *	@fclone: skbuff clone status
 *	@ipvs_property: skbuff is owned by ipvs
584
585
 *	@peeked: this packet has been seen already, so stats have been
 *		done for it, don't do them again
586
 *	@nf_trace: netfilter packet trace flag
587
588
589
 *	@protocol: Packet protocol from driver
 *	@destructor: Destruct function
 *	@nfct: Associated connection, if any
Linus Torvalds's avatar
Linus Torvalds committed
590
 *	@nf_bridge: Saved data about a bridged frame - see br_netfilter.c
591
 *	@skb_iif: ifindex of device we arrived on
Linus Torvalds's avatar
Linus Torvalds committed
592
593
 *	@tc_index: Traffic control index
 *	@tc_verd: traffic control verdict
594
 *	@hash: the packet hash
595
 *	@queue_mapping: Queue mapping for multiqueue devices
596
 *	@xmit_more: More SKBs are pending for this queue
597
 *	@ndisc_nodetype: router type (from link layer)
598
 *	@ooo_okay: allow the mapping of a socket to a queue to be changed
599
 *	@l4_hash: indicate hash is a canonical 4-tuple hash over transport
600
 *		ports.
601
 *	@sw_hash: indicates hash was computed in software stack
602
603
 *	@wifi_acked_valid: wifi_acked was set
 *	@wifi_acked: whether frame was acked on wifi or not
604
 *	@no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
605
  *	@napi_id: id of the NAPI struct this skb came from
606
 *	@secmark: security marking
607
 *	@offload_fwd_mark: fwding offload mark
608
 *	@mark: Generic packet mark
609
 *	@vlan_proto: vlan encapsulation protocol
610
 *	@vlan_tci: vlan tag control information
Simon Horman's avatar
Simon Horman committed
611
 *	@inner_protocol: Protocol (encapsulation)
612
613
 *	@inner_transport_header: Inner transport layer header (encapsulation)
 *	@inner_network_header: Network layer header (encapsulation)
614
 *	@inner_mac_header: Link layer header (encapsulation)
615
616
617
618
619
620
621
622
623
 *	@transport_header: Transport layer header
 *	@network_header: Network layer header
 *	@mac_header: Link layer header
 *	@tail: Tail pointer
 *	@end: End pointer
 *	@head: Head of buffer
 *	@data: Data head pointer
 *	@truesize: Buffer size
 *	@users: User count - see {datagram,tcp}.c
Linus Torvalds's avatar
Linus Torvalds committed
624
625
626
 */

struct sk_buff {
627
	union {
628
629
630
631
632
633
634
635
636
637
638
		struct {
			/* These two members must be first. */
			struct sk_buff		*next;
			struct sk_buff		*prev;

			union {
				ktime_t		tstamp;
				struct skb_mstamp skb_mstamp;
			};
		};
		struct rb_node	rbnode; /* used in netem & tcp stack */
639
	};
640
	struct sock		*sk;
Linus Torvalds's avatar
Linus Torvalds committed
641
642
643
644
645
646
647
648
	struct net_device	*dev;

	/*
	 * This is the control buffer. It is free to use for every
	 * layer. Please put your private variables there. If you
	 * want to keep them across layers you have to do a skb_clone()
	 * first. This is owned by whoever has the skb queued ATM.
	 */
649
	char			cb[48] __aligned(8);
Linus Torvalds's avatar
Linus Torvalds committed
650

Eric Dumazet's avatar
Eric Dumazet committed
651
	unsigned long		_skb_refdst;
652
	void			(*destructor)(struct sk_buff *skb);
653
654
#ifdef CONFIG_XFRM
	struct	sec_path	*sp;
655
656
657
658
#endif
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
	struct nf_conntrack	*nfct;
#endif
659
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
660
	struct nf_bridge_info	*nf_bridge;
661
#endif
Linus Torvalds's avatar
Linus Torvalds committed
662
	unsigned int		len,
663
664
665
				data_len;
	__u16			mac_len,
				hdr_len;
666
667
668
669

	/* Following fields are _not_ copied in __copy_skb_header()
	 * Note that queue_mapping is here mostly to fill a hole.
	 */
670
	kmemcheck_bitfield_begin(flags1);
671
672
	__u16			queue_mapping;
	__u8			cloned:1,
673
				nohdr:1,
674
				fclone:2,
675
				peeked:1,
676
677
678
				head_frag:1,
				xmit_more:1;
	/* one bit hole */
679
	kmemcheck_bitfield_end(flags1);
680

681
682
683
	/* fields enclosed in headers_start/headers_end are copied
	 * using a single memcpy() in __copy_skb_header()
	 */
684
	/* private: */
685
	__u32			headers_start[0];
686
	/* public: */
687

688
689
690
691
692
/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX	(7 << 5)
#else
#define PKT_TYPE_MAX	7
Linus Torvalds's avatar
Linus Torvalds committed
693
#endif
694
#define PKT_TYPE_OFFSET()	offsetof(struct sk_buff, __pkt_type_offset)
695

696
	__u8			__pkt_type_offset[0];
697
	__u8			pkt_type:3;
698
	__u8			pfmemalloc:1;
699
700
701
702
703
	__u8			ignore_df:1;
	__u8			nfctinfo:3;

	__u8			nf_trace:1;
	__u8			ip_summed:2;
704
	__u8			ooo_okay:1;
705
	__u8			l4_hash:1;
706
	__u8			sw_hash:1;
707
708
	__u8			wifi_acked_valid:1;
	__u8			wifi_acked:1;
709

710
	__u8			no_fcs:1;
711
	/* Indicates the inner headers are valid in the skbuff. */
712
	__u8			encapsulation:1;
713
	__u8			encap_hdr_csum:1;
714
	__u8			csum_valid:1;
715
	__u8			csum_complete_sw:1;
716
717
	__u8			csum_level:2;
	__u8			csum_bad:1;
718

719
720
721
722
#ifdef CONFIG_IPV6_NDISC_NODETYPE
	__u8			ndisc_nodetype:2;
#endif
	__u8			ipvs_property:1;
Tom Herbert's avatar
Tom Herbert committed
723
	__u8			inner_protocol_type:1;
724
725
	__u8			remcsum_offload:1;
	/* 3 or 5 bit hole */
726
727
728
729
730
731
732

#ifdef CONFIG_NET_SCHED
	__u16			tc_index;	/* traffic control index */
#ifdef CONFIG_NET_CLS_ACT
	__u16			tc_verd;	/* traffic control verdict */
#endif
#endif
733

734
735
736
737
738
739
740
741
742
743
744
745
	union {
		__wsum		csum;
		struct {
			__u16	csum_start;
			__u16	csum_offset;
		};
	};
	__u32			priority;
	int			skb_iif;
	__u32			hash;
	__be16			vlan_proto;
	__u16			vlan_tci;
746
747
748
749
750
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
	union {
		unsigned int	napi_id;
		unsigned int	sender_cpu;
	};
751
#endif
752
	union {
753
#ifdef CONFIG_NETWORK_SECMARK
754
755
756
757
		__u32		secmark;
#endif
#ifdef CONFIG_NET_SWITCHDEV
		__u32		offload_fwd_mark;
758
#endif
759
760
	};

761
762
	union {
		__u32		mark;
Eric Dumazet's avatar
Eric Dumazet committed
763
		__u32		reserved_tailroom;
764
	};
Linus Torvalds's avatar
Linus Torvalds committed
765

Tom Herbert's avatar
Tom Herbert committed
766
767
768
769
770
	union {
		__be16		inner_protocol;
		__u8		inner_ipproto;
	};

771
772
773
	__u16			inner_transport_header;
	__u16			inner_network_header;
	__u16			inner_mac_header;
774
775

	__be16			protocol;
776
777
778
	__u16			transport_header;
	__u16			network_header;
	__u16			mac_header;
779

780
	/* private: */
781
	__u32			headers_end[0];
782
	/* public: */
783

Linus Torvalds's avatar
Linus Torvalds committed
784
	/* These elements must be at the end, see alloc_skb() for details.  */
785
	sk_buff_data_t		tail;
786
	sk_buff_data_t		end;
Linus Torvalds's avatar
Linus Torvalds committed
787
	unsigned char		*head,
788
				*data;
789
790
	unsigned int		truesize;
	atomic_t		users;
Linus Torvalds's avatar
Linus Torvalds committed
791
792
793
794
795
796
797
798
799
};

#ifdef __KERNEL__
/*
 *	Handling routines are only of interest to the kernel
 */
#include <linux/slab.h>


800
801
#define SKB_ALLOC_FCLONE	0x01
#define SKB_ALLOC_RX		0x02
802
#define SKB_ALLOC_NAPI		0x04
803
804
805
806
807
808
809

/* Returns true if the skb was allocated from PFMEMALLOC reserves */
static inline bool skb_pfmemalloc(const struct sk_buff *skb)
{
	return unlikely(skb->pfmemalloc);
}

Eric Dumazet's avatar
Eric Dumazet committed
810
811
812
813
814
815
816
817
818
819
820
821
822
/*
 * skb might have a dst pointer attached, refcounted or not.
 * _skb_refdst low order bit is set if refcount was _not_ taken
 */
#define SKB_DST_NOREF	1UL
#define SKB_DST_PTRMASK	~(SKB_DST_NOREF)

/**
 * skb_dst - returns skb dst_entry
 * @skb: buffer
 *
 * Returns skb dst_entry, regardless of reference taken or not.
 */
Eric Dumazet's avatar
Eric Dumazet committed
823
824
static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
{
Eric Dumazet's avatar
Eric Dumazet committed
825
826
827
828
829
830
831
	/* If refdst was not refcounted, check we still are in a 
	 * rcu_read_lock section
	 */
	WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
		!rcu_read_lock_held() &&
		!rcu_read_lock_bh_held());
	return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
Eric Dumazet's avatar
Eric Dumazet committed
832
833
}

Eric Dumazet's avatar
Eric Dumazet committed
834
835
836
837
838
839
840
841
/**
 * skb_dst_set - sets skb dst
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was taken on dst and should
 * be released by skb_dst_drop()
 */
Eric Dumazet's avatar
Eric Dumazet committed
842
843
static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
{
Eric Dumazet's avatar
Eric Dumazet committed
844
845
846
	skb->_skb_refdst = (unsigned long)dst;
}

847
848
849
850
851
852
853
854
855
856
857
858
/**
 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was not taken on dst.
 * If dst entry is cached, we do not take reference and dst_release
 * will be avoided by refdst_drop. If dst entry is not cached, we take
 * reference, so that last dst_release can destroy the dst immediately.
 */
static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
{
859
860
	WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
	skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
861
}
Eric Dumazet's avatar
Eric Dumazet committed
862
863

/**
Lucas De Marchi's avatar
Lucas De Marchi committed
864
 * skb_dst_is_noref - Test if skb dst isn't refcounted
Eric Dumazet's avatar
Eric Dumazet committed
865
866
867
868
869
 * @skb: buffer
 */
static inline bool skb_dst_is_noref(const struct sk_buff *skb)
{
	return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
Eric Dumazet's avatar
Eric Dumazet committed
870
871
}

Eric Dumazet's avatar
Eric Dumazet committed
872
873
static inline struct rtable *skb_rtable(const struct sk_buff *skb)
{
Eric Dumazet's avatar
Eric Dumazet committed
874
	return (struct rtable *)skb_dst(skb);
Eric Dumazet's avatar
Eric Dumazet committed
875
876
}

877
878
879
880
881
void kfree_skb(struct sk_buff *skb);
void kfree_skb_list(struct sk_buff *segs);
void skb_tx_error(struct sk_buff *skb);
void consume_skb(struct sk_buff *skb);
void  __kfree_skb(struct sk_buff *skb);
882
extern struct kmem_cache *skbuff_head_cache;
883

884
885
886
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
		      bool *fragstolen, int *delta_truesize);
887

888
889
struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
			    int node);
Eric Dumazet's avatar
Eric Dumazet committed
890
struct sk_buff *__build_skb(void *data, unsigned int frag_size);
891
struct sk_buff *build_skb(void *data, unsigned int frag_size);
892
static inline struct sk_buff *alloc_skb(unsigned int size,
893
					gfp_t priority)
894
{
895
	return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
896
897
}

898
899
900
901
902
903
struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
				     unsigned long data_len,
				     int max_page_order,
				     int *errcode,
				     gfp_t gfp_mask);

904
905
906
907
908
909
910
911
912
913
914
915
916
/* Layout of fast clones : [skb1][skb2][fclone_ref] */
struct sk_buff_fclones {
	struct sk_buff	skb1;

	struct sk_buff	skb2;

	atomic_t	fclone_ref;
};

/**
 *	skb_fclone_busy - check if fclone is busy
 *	@skb: buffer
 *
917
 * Returns true if skb is a fast clone, and its clone is not freed.
918
919
 * Some drivers call skb_orphan() in their ndo_start_xmit(),
 * so we also check that this didnt happen.
920
 */
921
922
static inline bool skb_fclone_busy(const struct sock *sk,
				   const struct sk_buff *skb)
923
924
925
926
927
928
{
	const struct sk_buff_fclones *fclones;

	fclones = container_of(skb, struct sk_buff_fclones, skb1);

	return skb->fclone == SKB_FCLONE_ORIG &&
929
	       atomic_read(&fclones->fclone_ref) > 1 &&
930
	       fclones->skb2.sk == sk;
931
932
}

933
static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
934
					       gfp_t priority)
935
{
936
	return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
937
938
}

939
struct sk_buff *__alloc_skb_head(gfp_t priority, int node);
940
941
942
943
944
static inline struct sk_buff *alloc_skb_head(gfp_t priority)
{
	return __alloc_skb_head(priority, -1);
}

945
946
947
948
struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
949
950
951
952
953
954
955
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
				   gfp_t gfp_mask, bool fclone);
static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
					  gfp_t gfp_mask)
{
	return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
}
956
957
958
959
960
961

int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
				     unsigned int headroom);
struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
				int newtailroom, gfp_t priority);
962
963
int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
			int offset, int len);
964
965
966
967
int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset,
		 int len);
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
int skb_pad(struct sk_buff *skb, int pad);
968
#define dev_kfree_skb(a)	consume_skb(a)
Linus Torvalds's avatar
Linus Torvalds committed
969

970
971
972
973
int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length);
974

975
976
977
int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
			 int offset, size_t size);

Eric Dumazet's avatar
Eric Dumazet committed
978
struct skb_seq_state {
979
980
981
982
983
984
985
986
987
	__u32		lower_offset;
	__u32		upper_offset;
	__u32		frag_idx;
	__u32		stepped_offset;
	struct sk_buff	*root_skb;
	struct sk_buff	*cur_skb;
	__u8		*frag_data;
};

988
989
990
991
992
void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
			  unsigned int to, struct skb_seq_state *st);
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
			  struct skb_seq_state *st);
void skb_abort_seq_read(struct skb_seq_state *st);
993

994
unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
995
			   unsigned int to, struct ts_config *config);
996

997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
/*
 * Packet hash types specify the type of hash in skb_set_hash.
 *
 * Hash types refer to the protocol layer addresses which are used to
 * construct a packet's hash. The hashes are used to differentiate or identify
 * flows of the protocol layer for the hash type. Hash types are either
 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
 *
 * Properties of hashes:
 *
 * 1) Two packets in different flows have different hash values
 * 2) Two packets in the same flow should have the same hash value
 *
 * A hash at a higher layer is considered to be more specific. A driver should
 * set the most specific hash possible.
 *
 * A driver cannot indicate a more specific hash than the layer at which a hash
 * was computed. For instance an L3 hash cannot be set as an L4 hash.
 *
 * A driver may indicate a hash level which is less specific than the
 * actual layer the hash was computed on. For instance, a hash computed
 * at L4 may be considered an L3 hash. This should only be done if the
 * driver can't unambiguously determine that the HW computed the hash at
 * the higher layer. Note that the "should" in the second property above
 * permits this.
 */
enum pkt_hash_types {
	PKT_HASH_TYPE_NONE,	/* Undefined type */
	PKT_HASH_TYPE_L2,	/* Input: src_MAC, dest_MAC */
	PKT_HASH_TYPE_L3,	/* Input: src_IP, dst_IP */
	PKT_HASH_TYPE_L4,	/* Input: src_IP, dst_IP, src_port, dst_port */
};

1030
static inline void skb_clear_hash(struct sk_buff *skb)
1031
{
1032
	skb->hash = 0;
1033
	skb->sw_hash = 0;
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
	skb->l4_hash = 0;
}

static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
{
	if (!skb->l4_hash)
		skb_clear_hash(skb);
}

static inline void
__skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
{
	skb->l4_hash = is_l4;
	skb->sw_hash = is_sw;
1048
	skb->hash = hash;
1049
1050
}

1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
static inline void
skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
{
	/* Used by drivers to set hash from HW */
	__skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
}

static inline void
__skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
{
	__skb_set_hash(skb, hash, true, is_l4);
}

1064
void __skb_get_hash(struct sk_buff *skb);
1065
u32 __skb_get_hash_symmetric(struct sk_buff *skb);
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
u32 skb_get_poff(const struct sk_buff *skb);
u32 __skb_get_poff(const struct sk_buff *skb, void *data,
		   const struct flow_keys *keys, int hlen);
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
			    void *data, int hlen_proto);

static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
					int thoff, u8 ip_proto)
{
	return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
}

void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
			     const struct flow_dissector_key *key,
			     unsigned int key_count);

bool __skb_flow_dissect(const struct sk_buff *skb,
			struct flow_dissector *flow_dissector,
			void *target_container,
1085
1086
			void *data, __be16 proto, int nhoff, int hlen,
			unsigned int flags);
1087
1088
1089

static inline bool skb_flow_dissect(const struct sk_buff *skb,
				    struct flow_dissector *flow_dissector,
1090
				    void *target_container, unsigned int flags)
1091
1092
{
	return __skb_flow_dissect(skb, flow_dissector, target_container,
1093
				  NULL, 0, 0, 0, flags);
1094
1095
1096
}

static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1097
1098
					      struct flow_keys *flow,
					      unsigned int flags)
1099
1100
1101
{
	memset(flow, 0, sizeof(*flow));
	return __skb_flow_dissect(skb, &flow_keys_dissector, flow,
1102
				  NULL, 0, 0, 0, flags);
1103
1104
1105
1106
}

static inline bool skb_flow_dissect_flow_keys_buf(struct flow_keys *flow,
						  void *data, __be16 proto,
1107
1108
						  int nhoff, int hlen,
						  unsigned int flags)
1109
1110
1111
{
	memset(flow, 0, sizeof(*flow));
	return __skb_flow_dissect(NULL, &flow_keys_buf_dissector, flow,
1112
				  data, proto, nhoff, hlen, flags);
1113
1114
}

1115
static inline __u32 skb_get_hash(struct sk_buff *skb)
1116
{
1117
	if (!skb->l4_hash && !skb->sw_hash)
1118
		__skb_get_hash(skb);
1119

1120
	return skb->hash;
1121
1122
}

1123
__u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6);
1124

1125
static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1126
{
1127
1128
	if (!skb->l4_hash && !skb->sw_hash) {
		struct flow_keys keys;
1129
		__u32 hash = __get_hash_from_flowi6(fl6, &keys);
1130

1131
		__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1132
	}
1133
1134
1135
1136

	return skb->hash;
}

1137
__u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl);
1138

1139
static inline __u32 skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4)
1140
{
1141
1142
	if (!skb->l4_hash && !skb->sw_hash) {
		struct flow_keys keys;
1143
		__u32 hash = __get_hash_from_flowi4(fl4, &keys);
1144

1145
		__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1146
	}
1147
1148
1149
1150

	return skb->hash;
}

Tom Herbert's avatar
Tom Herbert committed
1151
1152
__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb);

Tom Herbert's avatar
Tom Herbert committed
1153
1154
static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
{
1155
	return skb->hash;
Tom Herbert's avatar
Tom Herbert committed
1156
1157
}

1158
1159
static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
{
1160
	to->hash = from->hash;
1161
	to->sw_hash = from->sw_hash;
1162
	to->l4_hash = from->l4_hash;
1163
1164
};

1165
1166
1167
1168
1169
#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
	return skb->head + skb->end;
}
1170
1171
1172
1173
1174

static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
	return skb->end;
}
1175
1176
1177
1178
1179
#else
static inline