diff --git a/drivers/net/e1000/e1000.h b/drivers/net/e1000/e1000.h
index 896ea8a723867500d9bf775b31870ca35b49f6b4..f091042b146eaed8d4de0641010cd7a3a3b77e73 100644
--- a/drivers/net/e1000/e1000.h
+++ b/drivers/net/e1000/e1000.h
@@ -257,6 +257,17 @@ struct e1000_adapter {
spinlock_t tx_queue_lock;
#endif
atomic_t irq_sem;
+ unsigned int detect_link;
+ unsigned int total_tx_bytes;
+ unsigned int total_tx_packets;
+ unsigned int total_rx_bytes;
+ unsigned int total_rx_packets;
+ /* Interrupt Throttle Rate */
+ uint32_t itr;
+ uint32_t itr_setting;
+ uint16_t tx_itr;
+ uint16_t rx_itr;
+
struct work_struct reset_task;
uint8_t fc_autoneg;
@@ -314,8 +325,6 @@ struct e1000_adapter {
uint64_t gorcl_old;
uint16_t rx_ps_bsize0;
- /* Interrupt Throttle Rate */
- uint32_t itr;
/* OS defined structs */
struct net_device *netdev;
diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c
index 35e4e32c7702b3c029e0a83c564473b28ba8f41c..56be5c89363c99e9d8e0e47d8655725532ebbcc4 100644
--- a/drivers/net/e1000/e1000_main.c
+++ b/drivers/net/e1000/e1000_main.c
@@ -1897,7 +1897,7 @@ e1000_configure_rx(struct e1000_adapter *adapter)
if (hw->mac_type >= e1000_82540) {
E1000_WRITE_REG(hw, RADV, adapter->rx_abs_int_delay);
- if (adapter->itr > 1)
+ if (adapter->itr_setting != 0)
E1000_WRITE_REG(hw, ITR,
1000000000 / (adapter->itr * 256));
}
@@ -1907,11 +1907,11 @@ e1000_configure_rx(struct e1000_adapter *adapter)
/* Reset delay timers after every interrupt */
ctrl_ext |= E1000_CTRL_EXT_INT_TIMER_CLR;
#ifdef CONFIG_E1000_NAPI
- /* Auto-Mask interrupts upon ICR read. */
+ /* Auto-Mask interrupts upon ICR access */
ctrl_ext |= E1000_CTRL_EXT_IAME;
+ E1000_WRITE_REG(hw, IAM, 0xffffffff);
#endif
E1000_WRITE_REG(hw, CTRL_EXT, ctrl_ext);
- E1000_WRITE_REG(hw, IAM, ~0);
E1000_WRITE_FLUSH(hw);
}
@@ -2576,19 +2576,6 @@ e1000_watchdog(unsigned long data)
}
}
- /* Dynamic mode for Interrupt Throttle Rate (ITR) */
- if (adapter->hw.mac_type >= e1000_82540 && adapter->itr == 1) {
- /* Symmetric Tx/Rx gets a reduced ITR=2000; Total
- * asymmetrical Tx or Rx gets ITR=8000; everyone
- * else is between 2000-8000. */
- uint32_t goc = (adapter->gotcl + adapter->gorcl) / 10000;
- uint32_t dif = (adapter->gotcl > adapter->gorcl ?
- adapter->gotcl - adapter->gorcl :
- adapter->gorcl - adapter->gotcl) / 10000;
- uint32_t itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
- E1000_WRITE_REG(&adapter->hw, ITR, 1000000000 / (itr * 256));
- }
-
/* Cause software interrupt to ensure rx ring is cleaned */
E1000_WRITE_REG(&adapter->hw, ICS, E1000_ICS_RXDMT0);
@@ -2604,6 +2591,135 @@ e1000_watchdog(unsigned long data)
mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ);
}
+enum latency_range {
+ lowest_latency = 0,
+ low_latency = 1,
+ bulk_latency = 2,
+ latency_invalid = 255
+};
+
+/**
+ * e1000_update_itr - update the dynamic ITR value based on statistics
+ * Stores a new ITR value based on packets and byte
+ * counts during the last interrupt. The advantage of per interrupt
+ * computation is faster updates and more accurate ITR for the current
+ * traffic pattern. Constants in this function were computed
+ * based on theoretical maximum wire speed and thresholds were set based
+ * on testing data as well as attempting to minimize response time
+ * while increasing bulk throughput.
+ * this functionality is controlled by the InterruptThrottleRate module
+ * parameter (see e1000_param.c)
+ * @adapter: pointer to adapter
+ * @itr_setting: current adapter->itr
+ * @packets: the number of packets during this measurement interval
+ * @bytes: the number of bytes during this measurement interval
+ **/
+static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
+ uint16_t itr_setting,
+ int packets,
+ int bytes)
+{
+ unsigned int retval = itr_setting;
+ struct e1000_hw *hw = &adapter->hw;
+
+ if (unlikely(hw->mac_type < e1000_82540))
+ goto update_itr_done;
+
+ if (packets == 0)
+ goto update_itr_done;
+
+
+ switch (itr_setting) {
+ case lowest_latency:
+ if ((packets < 5) && (bytes > 512))
+ retval = low_latency;
+ break;
+ case low_latency: /* 50 usec aka 20000 ints/s */
+ if (bytes > 10000) {
+ if ((packets < 10) ||
+ ((bytes/packets) > 1200))
+ retval = bulk_latency;
+ else if ((packets > 35))
+ retval = lowest_latency;
+ } else if (packets <= 2 && bytes < 512)
+ retval = lowest_latency;
+ break;
+ case bulk_latency: /* 250 usec aka 4000 ints/s */
+ if (bytes > 25000) {
+ if (packets > 35)
+ retval = low_latency;
+ } else {
+ if (bytes < 6000)
+ retval = low_latency;
+ }
+ break;
+ }
+
+update_itr_done:
+ return retval;
+}
+
+static void e1000_set_itr(struct e1000_adapter *adapter)
+{
+ struct e1000_hw *hw = &adapter->hw;
+ uint16_t current_itr;
+ uint32_t new_itr = adapter->itr;
+
+ if (unlikely(hw->mac_type < e1000_82540))
+ return;
+
+ /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
+ if (unlikely(adapter->link_speed != SPEED_1000)) {
+ current_itr = 0;
+ new_itr = 4000;
+ goto set_itr_now;
+ }
+
+ adapter->tx_itr = e1000_update_itr(adapter,
+ adapter->tx_itr,
+ adapter->total_tx_packets,
+ adapter->total_tx_bytes);
+ adapter->rx_itr = e1000_update_itr(adapter,
+ adapter->rx_itr,
+ adapter->total_rx_packets,
+ adapter->total_rx_bytes);
+
+ current_itr = max(adapter->rx_itr, adapter->tx_itr);
+
+ /* conservative mode eliminates the lowest_latency setting */
+ if (current_itr == lowest_latency && (adapter->itr_setting == 3))
+ current_itr = low_latency;
+
+ switch (current_itr) {
+ /* counts and packets in update_itr are dependent on these numbers */
+ case lowest_latency:
+ new_itr = 70000;
+ break;
+ case low_latency:
+ new_itr = 20000; /* aka hwitr = ~200 */
+ break;
+ case bulk_latency:
+ new_itr = 4000;
+ break;
+ default:
+ break;
+ }
+
+set_itr_now:
+ if (new_itr != adapter->itr) {
+ /* this attempts to bias the interrupt rate towards Bulk
+ * by adding intermediate steps when interrupt rate is
+ * increasing */
+ new_itr = new_itr > adapter->itr ?
+ min(adapter->itr + (new_itr >> 2), new_itr) :
+ new_itr;
+ adapter->itr = new_itr;
+ E1000_WRITE_REG(hw, ITR, 1000000000 / (new_itr * 256));
+ }
+
+ return;
+}
+
#define E1000_TX_FLAGS_CSUM 0x00000001
#define E1000_TX_FLAGS_VLAN 0x00000002
#define E1000_TX_FLAGS_TSO 0x00000004
@@ -3538,15 +3654,27 @@ irqreturn_t e1000_intr_msi(int irq, void *data)
}
#ifdef CONFIG_E1000_NAPI
- if (likely(netif_rx_schedule_prep(netdev)))
+ if (likely(netif_rx_schedule_prep(netdev))) {
+ adapter->total_tx_bytes = 0;
+ adapter->total_tx_packets = 0;
+ adapter->total_rx_bytes = 0;
+ adapter->total_rx_packets = 0;
__netif_rx_schedule(netdev);
- else
+ } else
e1000_irq_enable(adapter);
#else
+ adapter->total_tx_bytes = 0;
+ adapter->total_rx_bytes = 0;
+ adapter->total_tx_packets = 0;
+ adapter->total_rx_packets = 0;
+
for (i = 0; i < E1000_MAX_INTR; i++)
if (unlikely(!adapter->clean_rx(adapter, adapter->rx_ring) &
!e1000_clean_tx_irq(adapter, adapter->tx_ring)))
break;
+
+ if (likely(adapter->itr_setting & 3))
+ e1000_set_itr(adapter);
#endif
return IRQ_HANDLED;
@@ -3568,7 +3696,17 @@ e1000_intr(int irq, void *data)
uint32_t rctl, icr = E1000_READ_REG(hw, ICR);
#ifndef CONFIG_E1000_NAPI
int i;
-#else
+#endif
+ if (unlikely(!icr))
+ return IRQ_NONE; /* Not our interrupt */
+
+#ifdef CONFIG_E1000_NAPI
+ /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
+ * not set, then the adapter didn't send an interrupt */
+ if (unlikely(hw->mac_type >= e1000_82571 &&
+ !(icr & E1000_ICR_INT_ASSERTED)))
+ return IRQ_NONE;
+
/* Interrupt Auto-Mask...upon reading ICR,
* interrupts are masked. No need for the
* IMC write, but it does mean we should
@@ -3577,14 +3715,6 @@ e1000_intr(int irq, void *data)
atomic_inc(&adapter->irq_sem);
#endif
- if (unlikely(!icr)) {
-#ifdef CONFIG_E1000_NAPI
- if (hw->mac_type >= e1000_82571)
- e1000_irq_enable(adapter);
-#endif
- return IRQ_NONE; /* Not our interrupt */
- }
-
if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
hw->get_link_status = 1;
/* 80003ES2LAN workaround--
@@ -3605,13 +3735,18 @@ e1000_intr(int irq, void *data)
#ifdef CONFIG_E1000_NAPI
if (unlikely(hw->mac_type < e1000_82571)) {
+ /* disable interrupts, without the synchronize_irq bit */
atomic_inc(&adapter->irq_sem);
E1000_WRITE_REG(hw, IMC, ~0);
E1000_WRITE_FLUSH(hw);
}
- if (likely(netif_rx_schedule_prep(netdev)))
+ if (likely(netif_rx_schedule_prep(netdev))) {
+ adapter->total_tx_bytes = 0;
+ adapter->total_tx_packets = 0;
+ adapter->total_rx_bytes = 0;
+ adapter->total_rx_packets = 0;
__netif_rx_schedule(netdev);
- else
+ } else
/* this really should not happen! if it does it is basically a
* bug, but not a hard error, so enable ints and continue */
e1000_irq_enable(adapter);
@@ -3631,11 +3766,19 @@ e1000_intr(int irq, void *data)
E1000_WRITE_REG(hw, IMC, ~0);
}
+ adapter->total_tx_bytes = 0;
+ adapter->total_rx_bytes = 0;
+ adapter->total_tx_packets = 0;
+ adapter->total_rx_packets = 0;
+
for (i = 0; i < E1000_MAX_INTR; i++)
if (unlikely(!adapter->clean_rx(adapter, adapter->rx_ring) &
!e1000_clean_tx_irq(adapter, adapter->tx_ring)))
break;
+ if (likely(adapter->itr_setting & 3))
+ e1000_set_itr(adapter);
+
if (hw->mac_type == e1000_82547 || hw->mac_type == e1000_82547_rev_2)
e1000_irq_enable(adapter);
@@ -3683,6 +3826,8 @@ e1000_clean(struct net_device *poll_dev, int *budget)
if ((!tx_cleaned && (work_done == 0)) ||
!netif_running(poll_dev)) {
quit_polling:
+ if (likely(adapter->itr_setting & 3))
+ e1000_set_itr(adapter);
netif_rx_complete(poll_dev);
e1000_irq_enable(adapter);
return 0;
@@ -3709,6 +3854,7 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter,
unsigned int count = 0;
#endif
boolean_t cleaned = FALSE;
+ unsigned int total_tx_bytes=0, total_tx_packets=0;
i = tx_ring->next_to_clean;
eop = tx_ring->buffer_info[i].next_to_watch;
@@ -3720,6 +3866,13 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter,
buffer_info = &tx_ring->buffer_info[i];
cleaned = (i == eop);
+ if (cleaned) {
+ /* this packet count is wrong for TSO but has a
+ * tendency to make dynamic ITR change more
+ * towards bulk */
+ total_tx_packets++;
+ total_tx_bytes += buffer_info->skb->len;
+ }
e1000_unmap_and_free_tx_resource(adapter, buffer_info);
tx_desc->upper.data = 0;
@@ -3785,6 +3938,8 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter,
netif_stop_queue(netdev);
}
}
+ adapter->total_tx_bytes += total_tx_bytes;
+ adapter->total_tx_packets += total_tx_packets;
return cleaned;
}
@@ -3864,6 +4019,7 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
unsigned int i;
int cleaned_count = 0;
boolean_t cleaned = FALSE;
+ unsigned int total_rx_bytes=0, total_rx_packets=0;
i = rx_ring->next_to_clean;
rx_desc = E1000_RX_DESC(*rx_ring, i);
@@ -3930,6 +4086,10 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter,
* done after the TBI_ACCEPT workaround above */
length -= 4;
+ /* probably a little skewed due to removing CRC */
+ total_rx_bytes += length;
+ total_rx_packets++;
+
/* code added for copybreak, this should improve
* performance for small packets with large amounts
* of reassembly being done in the stack */
@@ -3998,6 +4158,8 @@ next_desc:
if (cleaned_count)
adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
+ adapter->total_rx_packets += total_rx_packets;
+ adapter->total_rx_bytes += total_rx_bytes;
return cleaned;
}
@@ -4027,6 +4189,7 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
uint32_t length, staterr;
int cleaned_count = 0;
boolean_t cleaned = FALSE;
+ unsigned int total_rx_bytes=0, total_rx_packets=0;
i = rx_ring->next_to_clean;
rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
@@ -4131,6 +4294,9 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
pskb_trim(skb, skb->len - 4);
copydone:
+ total_rx_bytes += skb->len;
+ total_rx_packets++;
+
e1000_rx_checksum(adapter, staterr,
le16_to_cpu(rx_desc->wb.lower.hi_dword.csum_ip.csum), skb);
skb->protocol = eth_type_trans(skb, netdev);
@@ -4179,6 +4345,8 @@ next_desc:
if (cleaned_count)
adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
+ adapter->total_rx_packets += total_rx_packets;
+ adapter->total_rx_bytes += total_rx_bytes;
return cleaned;
}
diff --git a/drivers/net/e1000/e1000_param.c b/drivers/net/e1000/e1000_param.c
index e4f88920b46b05b3dc909183f3ea00f59e2ddfc1..cbfcd7f2889f6fea29232bf6fd2b720efe0fdcda 100644
--- a/drivers/net/e1000/e1000_param.c
+++ b/drivers/net/e1000/e1000_param.c
@@ -139,7 +139,7 @@ E1000_PARAM(XsumRX, "Disable or enable Receive Checksum offload");
* Valid Range: 0-65535
*/
E1000_PARAM(TxIntDelay, "Transmit Interrupt Delay");
-#define DEFAULT_TIDV 64
+#define DEFAULT_TIDV 8
#define MAX_TXDELAY 0xFFFF
#define MIN_TXDELAY 0
@@ -148,7 +148,7 @@ E1000_PARAM(TxIntDelay, "Transmit Interrupt Delay");
* Valid Range: 0-65535
*/
E1000_PARAM(TxAbsIntDelay, "Transmit Absolute Interrupt Delay");
-#define DEFAULT_TADV 64
+#define DEFAULT_TADV 32
#define MAX_TXABSDELAY 0xFFFF
#define MIN_TXABSDELAY 0
@@ -167,16 +167,16 @@ E1000_PARAM(RxIntDelay, "Receive Interrupt Delay");
* Valid Range: 0-65535
*/
E1000_PARAM(RxAbsIntDelay, "Receive Absolute Interrupt Delay");
-#define DEFAULT_RADV 128
+#define DEFAULT_RADV 8
#define MAX_RXABSDELAY 0xFFFF
#define MIN_RXABSDELAY 0
/* Interrupt Throttle Rate (interrupts/sec)
*
- * Valid Range: 100-100000 (0=off, 1=dynamic)
+ * Valid Range: 100-100000 (0=off, 1=dynamic, 3=dynamic conservative)
*/
E1000_PARAM(InterruptThrottleRate, "Interrupt Throttling Rate");
-#define DEFAULT_ITR 8000
+#define DEFAULT_ITR 3
#define MAX_ITR 100000
#define MIN_ITR 100
@@ -472,15 +472,27 @@ e1000_check_options(struct e1000_adapter *adapter)
break;
case 1:
DPRINTK(PROBE, INFO, "%s set to dynamic mode\n",
- opt.name);
+ opt.name);
+ adapter->itr_setting = adapter->itr;
+ adapter->itr = 20000;
+ break;
+ case 3:
+ DPRINTK(PROBE, INFO,
+ "%s set to dynamic conservative mode\n",
+ opt.name);
+ adapter->itr_setting = adapter->itr;
+ adapter->itr = 20000;
break;
default:
e1000_validate_option(&adapter->itr, &opt,
- adapter);
+ adapter);
+ /* save the setting, because the dynamic bits change itr */
+ adapter->itr_setting = adapter->itr;
break;
}
} else {
- adapter->itr = opt.def;
+ adapter->itr_setting = opt.def;
+ adapter->itr = 20000;
}
}
{ /* Smart Power Down */