diff --git a/include/net/ip.h b/include/net/ip.h
index 3b524df7dddb04490af6cf01826ccfd3d900fb0b..890f9725d68156b8d4df25adea50880ae3bf46a0 100644
--- a/include/net/ip.h
+++ b/include/net/ip.h
@@ -165,12 +165,12 @@ struct ipv4_config {
};
extern struct ipv4_config ipv4_config;
-#define IP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.ip_statistics, field)
-#define IP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.ip_statistics, field)
-#define IP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.ip_statistics, field, val)
-#define IP_ADD_STATS_BH(net, field, val) SNMP_ADD_STATS_BH((net)->mib.ip_statistics, field, val)
-#define IP_UPD_PO_STATS(net, field, val) SNMP_UPD_PO_STATS((net)->mib.ip_statistics, field, val)
-#define IP_UPD_PO_STATS_BH(net, field, val) SNMP_UPD_PO_STATS_BH((net)->mib.ip_statistics, field, val)
+#define IP_INC_STATS(net, field) SNMP_INC_STATS64((net)->mib.ip_statistics, field)
+#define IP_INC_STATS_BH(net, field) SNMP_INC_STATS64_BH((net)->mib.ip_statistics, field)
+#define IP_ADD_STATS(net, field, val) SNMP_ADD_STATS64((net)->mib.ip_statistics, field, val)
+#define IP_ADD_STATS_BH(net, field, val) SNMP_ADD_STATS64_BH((net)->mib.ip_statistics, field, val)
+#define IP_UPD_PO_STATS(net, field, val) SNMP_UPD_PO_STATS64((net)->mib.ip_statistics, field, val)
+#define IP_UPD_PO_STATS_BH(net, field, val) SNMP_UPD_PO_STATS64_BH((net)->mib.ip_statistics, field, val)
#define NET_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.net_statistics, field)
#define NET_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.net_statistics, field)
#define NET_INC_STATS_USER(net, field) SNMP_INC_STATS_USER((net)->mib.net_statistics, field)
@@ -178,6 +178,14 @@ extern struct ipv4_config ipv4_config;
#define NET_ADD_STATS_USER(net, field, adnd) SNMP_ADD_STATS_USER((net)->mib.net_statistics, field, adnd)
extern unsigned long snmp_fold_field(void __percpu *mib[], int offt);
+#if BITS_PER_LONG==32
+extern u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t sync_off);
+#else
+static inline u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t syncp_off)
+{
+ return snmp_fold_field(mib, offt);
+}
+#endif
extern int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align);
extern void snmp_mib_free(void __percpu *ptr[2]);
diff --git a/include/net/ipv6.h b/include/net/ipv6.h
index f5808d596aab60000b2441a3fe53d28a2a03b0f3..1f841241099802145a8e1cff634e7e26c2f7c16c 100644
--- a/include/net/ipv6.h
+++ b/include/net/ipv6.h
@@ -136,17 +136,17 @@ extern struct ctl_path net_ipv6_ctl_path[];
/* MIBs */
#define IP6_INC_STATS(net, idev,field) \
- _DEVINC(net, ipv6, , idev, field)
+ _DEVINC(net, ipv6, 64, idev, field)
#define IP6_INC_STATS_BH(net, idev,field) \
- _DEVINC(net, ipv6, _BH, idev, field)
+ _DEVINC(net, ipv6, 64_BH, idev, field)
#define IP6_ADD_STATS(net, idev,field,val) \
- _DEVADD(net, ipv6, , idev, field, val)
+ _DEVADD(net, ipv6, 64, idev, field, val)
#define IP6_ADD_STATS_BH(net, idev,field,val) \
- _DEVADD(net, ipv6, _BH, idev, field, val)
+ _DEVADD(net, ipv6, 64_BH, idev, field, val)
#define IP6_UPD_PO_STATS(net, idev,field,val) \
- _DEVUPD(net, ipv6, , idev, field, val)
+ _DEVUPD(net, ipv6, 64, idev, field, val)
#define IP6_UPD_PO_STATS_BH(net, idev,field,val) \
- _DEVUPD(net, ipv6, _BH, idev, field, val)
+ _DEVUPD(net, ipv6, 64_BH, idev, field, val)
#define ICMP6_INC_STATS(net, idev, field) \
_DEVINC(net, icmpv6, , idev, field)
#define ICMP6_INC_STATS_BH(net, idev, field) \
diff --git a/include/net/snmp.h b/include/net/snmp.h
index 899003d18db9e57ab8813176f4ea896a27376c49..a0e61806d480bed5d13198b9852c10d482c04556 100644
--- a/include/net/snmp.h
+++ b/include/net/snmp.h
@@ -47,15 +47,16 @@ struct snmp_mib {
}
/*
- * We use all unsigned longs. Linux will soon be so reliable that even
- * these will rapidly get too small 8-). Seriously consider the IpInReceives
- * count on the 20Gb/s + networks people expect in a few years time!
+ * We use unsigned longs for most mibs but u64 for ipstats.
*/
+#include <linux/u64_stats_sync.h>
/* IPstats */
#define IPSTATS_MIB_MAX __IPSTATS_MIB_MAX
struct ipstats_mib {
- unsigned long mibs[IPSTATS_MIB_MAX];
+ /* mibs[] must be first field of struct ipstats_mib */
+ u64 mibs[IPSTATS_MIB_MAX];
+ struct u64_stats_sync syncp;
};
/* ICMP */
@@ -155,4 +156,70 @@ struct linux_xfrm_mib {
ptr->mibs[basefield##PKTS]++; \
ptr->mibs[basefield##OCTETS] += addend;\
} while (0)
+
+
+#if BITS_PER_LONG==32
+
+#define SNMP_ADD_STATS64_BH(mib, field, addend) \
+ do { \
+ __typeof__(*mib[0]) *ptr = __this_cpu_ptr((mib)[0]); \
+ u64_stats_update_begin(&ptr->syncp); \
+ ptr->mibs[field] += addend; \
+ u64_stats_update_end(&ptr->syncp); \
+ } while (0)
+#define SNMP_ADD_STATS64_USER(mib, field, addend) \
+ do { \
+ __typeof__(*mib[0]) *ptr; \
+ preempt_disable(); \
+ ptr = __this_cpu_ptr((mib)[1]); \
+ u64_stats_update_begin(&ptr->syncp); \
+ ptr->mibs[field] += addend; \
+ u64_stats_update_end(&ptr->syncp); \
+ preempt_enable(); \
+ } while (0)
+#define SNMP_ADD_STATS64(mib, field, addend) \
+ do { \
+ __typeof__(*mib[0]) *ptr; \
+ preempt_disable(); \
+ ptr = __this_cpu_ptr((mib)[!in_softirq()]); \
+ u64_stats_update_begin(&ptr->syncp); \
+ ptr->mibs[field] += addend; \
+ u64_stats_update_end(&ptr->syncp); \
+ preempt_enable(); \
+ } while (0)
+#define SNMP_INC_STATS64_BH(mib, field) SNMP_ADD_STATS64_BH(mib, field, 1)
+#define SNMP_INC_STATS64_USER(mib, field) SNMP_ADD_STATS64_USER(mib, field, 1)
+#define SNMP_INC_STATS64(mib, field) SNMP_ADD_STATS64(mib, field, 1)
+#define SNMP_UPD_PO_STATS64(mib, basefield, addend) \
+ do { \
+ __typeof__(*mib[0]) *ptr; \
+ preempt_disable(); \
+ ptr = __this_cpu_ptr((mib)[!in_softirq()]); \
+ u64_stats_update_begin(&ptr->syncp); \
+ ptr->mibs[basefield##PKTS]++; \
+ ptr->mibs[basefield##OCTETS] += addend; \
+ u64_stats_update_end(&ptr->syncp); \
+ preempt_enable(); \
+ } while (0)
+#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) \
+ do { \
+ __typeof__(*mib[0]) *ptr; \
+ ptr = __this_cpu_ptr((mib)[!in_softirq()]); \
+ u64_stats_update_begin(&ptr->syncp); \
+ ptr->mibs[basefield##PKTS]++; \
+ ptr->mibs[basefield##OCTETS] += addend; \
+ u64_stats_update_end(&ptr->syncp); \
+ } while (0)
+#else
+#define SNMP_INC_STATS64_BH(mib, field) SNMP_INC_STATS_BH(mib, field)
+#define SNMP_INC_STATS64_USER(mib, field) SNMP_INC_STATS_USER(mib, field)
+#define SNMP_INC_STATS64(mib, field) SNMP_INC_STATS(mib, field)
+#define SNMP_DEC_STATS64(mib, field) SNMP_DEC_STATS(mib, field)
+#define SNMP_ADD_STATS64_BH(mib, field, addend) SNMP_ADD_STATS_BH(mib, field, addend)
+#define SNMP_ADD_STATS64_USER(mib, field, addend) SNMP_ADD_STATS_USER(mib, field, addend)
+#define SNMP_ADD_STATS64(mib, field, addend) SNMP_ADD_STATS(mib, field, addend)
+#define SNMP_UPD_PO_STATS64(mib, basefield, addend) SNMP_UPD_PO_STATS(mib, basefield, addend)
+#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) SNMP_UPD_PO_STATS_BH(mib, basefield, addend)
+#endif
+
#endif
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 640db9b903306842e4f469fa7a92f330c283e2d1..3ceb025b16f29db6cf735b58839c61fa42223657 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1427,6 +1427,42 @@ unsigned long snmp_fold_field(void __percpu *mib[], int offt)
}
EXPORT_SYMBOL_GPL(snmp_fold_field);
+#if BITS_PER_LONG==32
+
+u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t syncp_offset)
+{
+ u64 res = 0;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ void *bhptr, *userptr;
+ struct u64_stats_sync *syncp;
+ u64 v_bh, v_user;
+ unsigned int start;
+
+ /* first mib used by softirq context, we must use _bh() accessors */
+ bhptr = per_cpu_ptr(SNMP_STAT_BHPTR(mib), cpu);
+ syncp = (struct u64_stats_sync *)(bhptr + syncp_offset);
+ do {
+ start = u64_stats_fetch_begin_bh(syncp);
+ v_bh = *(((u64 *) bhptr) + offt);
+ } while (u64_stats_fetch_retry_bh(syncp, start));
+
+ /* second mib used in USER context */
+ userptr = per_cpu_ptr(SNMP_STAT_USRPTR(mib), cpu);
+ syncp = (struct u64_stats_sync *)(userptr + syncp_offset);
+ do {
+ start = u64_stats_fetch_begin(syncp);
+ v_user = *(((u64 *) userptr) + offt);
+ } while (u64_stats_fetch_retry(syncp, start));
+
+ res += v_bh + v_user;
+ }
+ return res;
+}
+EXPORT_SYMBOL_GPL(snmp_fold_field64);
+#endif
+
int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align)
{
BUG_ON(ptr == NULL);
diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
index e320ca6b3ef3c16d93df3d47de0192173972bf92..4ae1f203f7cbb5daf7c07bea4a062c352c83700d 100644
--- a/net/ipv4/proc.c
+++ b/net/ipv4/proc.c
@@ -343,10 +343,12 @@ static int snmp_seq_show(struct seq_file *seq, void *v)
IPV4_DEVCONF_ALL(net, FORWARDING) ? 1 : 2,
sysctl_ip_default_ttl);
+ BUILD_BUG_ON(offsetof(struct ipstats_mib, mibs) != 0);
for (i = 0; snmp4_ipstats_list[i].name != NULL; i++)
- seq_printf(seq, " %lu",
- snmp_fold_field((void __percpu **)net->mib.ip_statistics,
- snmp4_ipstats_list[i].entry));
+ seq_printf(seq, " %llu",
+ snmp_fold_field64((void __percpu **)net->mib.ip_statistics,
+ snmp4_ipstats_list[i].entry,
+ offsetof(struct ipstats_mib, syncp)));
icmp_put(seq); /* RFC 2011 compatibility */
icmpmsg_put(seq);
@@ -432,9 +434,10 @@ static int netstat_seq_show(struct seq_file *seq, void *v)
seq_puts(seq, "\nIpExt:");
for (i = 0; snmp4_ipextstats_list[i].name != NULL; i++)
- seq_printf(seq, " %lu",
- snmp_fold_field((void __percpu **)net->mib.ip_statistics,
- snmp4_ipextstats_list[i].entry));
+ seq_printf(seq, " %llu",
+ snmp_fold_field64((void __percpu **)net->mib.ip_statistics,
+ snmp4_ipextstats_list[i].entry,
+ offsetof(struct ipstats_mib, syncp)));
seq_putc(seq, '\n');
return 0;
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 2514adf5251e94005db3a58935eb6981b6454fb0..e81155d2f251c94295c514660552469c25dff384 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -3862,12 +3862,28 @@ static inline void __snmp6_fill_stats(u64 *stats, void __percpu **mib,
memset(&stats[items], 0, pad);
}
+static inline void __snmp6_fill_stats64(u64 *stats, void __percpu **mib,
+ int items, int bytes, size_t syncpoff)
+{
+ int i;
+ int pad = bytes - sizeof(u64) * items;
+ BUG_ON(pad < 0);
+
+ /* Use put_unaligned() because stats may not be aligned for u64. */
+ put_unaligned(items, &stats[0]);
+ for (i = 1; i < items; i++)
+ put_unaligned(snmp_fold_field64(mib, i, syncpoff), &stats[i]);
+
+ memset(&stats[items], 0, pad);
+}
+
static void snmp6_fill_stats(u64 *stats, struct inet6_dev *idev, int attrtype,
int bytes)
{
switch (attrtype) {
case IFLA_INET6_STATS:
- __snmp6_fill_stats(stats, (void __percpu **)idev->stats.ipv6, IPSTATS_MIB_MAX, bytes);
+ __snmp6_fill_stats64(stats, (void __percpu **)idev->stats.ipv6,
+ IPSTATS_MIB_MAX, bytes, offsetof(struct ipstats_mib, syncp));
break;
case IFLA_INET6_ICMP6STATS:
__snmp6_fill_stats(stats, (void __percpu **)idev->stats.icmpv6, ICMP6_MIB_MAX, bytes);
diff --git a/net/ipv6/proc.c b/net/ipv6/proc.c
index 566798d69f37d223016cfc31a398d0649dad68bf..d082eaeefa25a039b5af47f4715e05f08e4ce813 100644
--- a/net/ipv6/proc.c
+++ b/net/ipv6/proc.c
@@ -174,17 +174,28 @@ static void snmp6_seq_show_item(struct seq_file *seq, void __percpu **mib,
const struct snmp_mib *itemlist)
{
int i;
- for (i=0; itemlist[i].name; i++)
+
+ for (i = 0; itemlist[i].name; i++)
seq_printf(seq, "%-32s\t%lu\n", itemlist[i].name,
snmp_fold_field(mib, itemlist[i].entry));
}
+static void snmp6_seq_show_item64(struct seq_file *seq, void __percpu **mib,
+ const struct snmp_mib *itemlist, size_t syncpoff)
+{
+ int i;
+
+ for (i = 0; itemlist[i].name; i++)
+ seq_printf(seq, "%-32s\t%llu\n", itemlist[i].name,
+ snmp_fold_field64(mib, itemlist[i].entry, syncpoff));
+}
+
static int snmp6_seq_show(struct seq_file *seq, void *v)
{
struct net *net = (struct net *)seq->private;
- snmp6_seq_show_item(seq, (void __percpu **)net->mib.ipv6_statistics,
- snmp6_ipstats_list);
+ snmp6_seq_show_item64(seq, (void __percpu **)net->mib.ipv6_statistics,
+ snmp6_ipstats_list, offsetof(struct ipstats_mib, syncp));
snmp6_seq_show_item(seq, (void __percpu **)net->mib.icmpv6_statistics,
snmp6_icmp6_list);
snmp6_seq_show_icmpv6msg(seq,