lcd/ixgbe: Use a private channel for xmit

Instead of sharing the same async channel with dispatch loop, create a separate
channel for xmit. This channel is exclusively used only for xmit
Signed-off-by: Vikram Narayanan <vikram186@gmail.com>

lcd-domains/test_mods/ixgbe/ixgbe_lcd/glue/ixgbe_caller.c

@@ -19,6 +19,7 @@ extern struct glue_cspace *ixgbe_cspace;
 extern struct thc_channel *ixgbe_async;
 extern cptr_t ixgbe_sync_endpoint;
 extern cptr_t ixgbe_register_channel;
+extern struct thc_channel_group ch_grp;
 
 static struct net_device *g_net_device;
 
@@ -249,7 +250,12 @@ int create_async_channel(void)
 {
 	int ret;
 	cptr_t tx, rx;
+	cptr_t tx_xmit, rx_xmit;
 	struct thc_channel *chnl;
+	struct thc_channel *xmit_chnl;
+	struct thc_channel_group_item *ch_item;
+	struct thc_channel_group_item *xmit_ch_item;
+
 	/*
 	 * Set up async and sync channels
 	 */
@@ -263,9 +269,30 @@ int create_async_channel(void)
 		LIBLCD_ERR("async chnl setup failed");
 		goto fail2;
 	}
-        lcd_set_cr0(ixgbe_sync_endpoint);
+
+	ch_item = kzalloc(sizeof(*ch_item), GFP_KERNEL);
+
+	thc_channel_group_item_init(ch_item, chnl, NULL);
+
+	thc_channel_group_item_add(&ch_grp, ch_item);
+
+	ret = setup_async_channel(&tx_xmit, &rx_xmit, &xmit_chnl);
+	if (ret) {
+		LIBLCD_ERR("async xmit chnl setup failed");
+		goto fail2;
+	}
+
+	xmit_ch_item = kzalloc(sizeof(*xmit_ch_item), GFP_KERNEL);
+
+	thc_channel_group_item_init(xmit_ch_item, xmit_chnl, NULL);
+
+	thc_channel_group_item_add(&ch_grp, xmit_ch_item);
+
+	lcd_set_cr0(ixgbe_sync_endpoint);
         lcd_set_cr1(rx);
         lcd_set_cr2(tx);
+        lcd_set_cr3(rx_xmit);
+        lcd_set_cr4(tx_xmit);
 
 	LIBLCD_MSG("sync call %s", __func__);
 	ret = lcd_sync_call(ixgbe_register_channel);
@@ -278,6 +305,9 @@ int create_async_channel(void)
         lcd_set_cr0(CAP_CPTR_NULL);
         lcd_set_cr1(CAP_CPTR_NULL);
         lcd_set_cr2(CAP_CPTR_NULL);
+        lcd_set_cr3(CAP_CPTR_NULL);
+        lcd_set_cr4(CAP_CPTR_NULL);
+
         if (ret) {
                 LIBLCD_ERR("lcd_call");
                 goto fail3;

lcd-domains/test_mods/ixgbe/ixgbe_lcd/main.c

@@ -17,6 +17,7 @@ struct thc_channel *ixgbe_async;
 struct glue_cspace *ixgbe_cspace;
 cptr_t ixgbe_sync_endpoint;
 int ixgbe_done = 0;
+struct thc_channel_group ch_grp;
 
 int ixgbe_init_module(void);
 void ixgbe_exit_module(void);
@@ -59,10 +60,12 @@ static void main_and_loop(void)
 		 * will not yield until it tries to use the async
 		 * channel). */
 		while (!stop && !ixgbe_done) {
+			struct thc_channel_group_item* curr_item;
 			/*
 			 * Do one async receive
 			 */
-			ret = thc_ipc_poll_recv(ixgbe_async, &msg);
+			ret = thc_poll_recv_group(&ch_grp,
+						&curr_item, &msg);
 			if (ret) {
 				if (ret == -EWOULDBLOCK) {
 					continue;
@@ -76,7 +79,7 @@ static void main_and_loop(void)
 			 */
 			ASYNC(
 
-				ret = dispatch_async_loop(ixgbe_async, msg,
+				ret = dispatch_async_loop(curr_item->channel, msg,
 							ixgbe_cspace,
 							ixgbe_sync_endpoint);
 				if (ret) {
@@ -124,6 +127,7 @@ static int __noreturn ixgbe_lcd_init(void)
 		goto fail2;
 	}
 
+	thc_channel_group_init(&ch_grp);
 	/* RUN CODE / LOOP ---------------------------------------- */
 
 	main_and_loop();

lcd-domains/test_mods/ixgbe/net_klcd/glue/ixgbe_callee.c

@@ -23,7 +23,7 @@ struct glue_cspace *c_cspace = NULL;
 struct thc_channel *ixgbe_async;
 struct cptr sync_ep;
 extern struct cspace *klcd_cspace;
-
+extern struct thc_channel *xmit_chnl;
 struct timer_list service_timer;
 struct napi_struct *napi_q0;
 
@@ -1359,6 +1359,14 @@ int ndo_start_xmit(struct sk_buff *skb,
 		sync_end.cptr = PTS()->sync_ep;
 	}
 
+	if (xmit_type == VOLUNTEER_XMIT) {
+		printk("%s, comm %s | pid %d | skblen %d "
+			"| skb->proto %02X\n", __func__,
+			current->comm, current->pid, skb->len,
+			ntohs(skb->protocol));
+		return NETDEV_TX_OK;
+	}
+
 	dev_container = container_of(dev,
 		struct net_device_container,
 		net_device);
@@ -1399,11 +1407,6 @@ int ndo_start_xmit(struct sk_buff *skb,
 
 	switch (xmit_type) {
 	case VOLUNTEER_XMIT:
-		printk("%s, comm %s | pid %d | skblen %d "
-			"| skb->proto %02X\n", __func__,
-			current->comm, current->pid, skb->len,
-			ntohs(skb->protocol));
-
 		ret = thc_ipc_send_request(hidden_args->async_chnl,
 				_request, &request_cookie);
 
@@ -2459,7 +2462,7 @@ void setup_netdev_ops(struct net_device_container *dev_c,
 	dev_netdev_ops_ndo_start_xmit_hidden_args->struct_container = netdev_ops_container;
 	dev_netdev_ops_ndo_start_xmit_hidden_args->cspace = c_cspace;
 	dev_netdev_ops_ndo_start_xmit_hidden_args->sync_ep = sync_ep;
-	dev_netdev_ops_ndo_start_xmit_hidden_args->async_chnl = _channel;
+	dev_netdev_ops_ndo_start_xmit_hidden_args->async_chnl = xmit_chnl;
 	netdev_ops_container->net_device_ops.ndo_start_xmit = LCD_HANDLE_TO_TRAMPOLINE(dev_netdev_ops_ndo_start_xmit_hidden_args->t_handle);
 	ret = set_memory_x(( ( unsigned  long   )dev_netdev_ops_ndo_start_xmit_hidden_args->t_handle ) & ( PAGE_MASK ),
 		( ALIGN(LCD_TRAMPOLINE_SIZE(ndo_start_xmit_trampoline),

lcd-domains/test_mods/ixgbe/net_klcd/main.c

@@ -21,6 +21,7 @@ struct net_info {
 	struct list_head list;
 };
 static LIST_HEAD(net_infos);
+struct thc_channel *xmit_chnl;
 
 extern int setup_async_net_ring_channel(cptr_t tx, cptr_t rx, 
 				struct thc_channel **chnl_out);
@@ -150,6 +151,7 @@ static int do_one_register(cptr_t register_chnl)
 {
 	int ret;
 	cptr_t sync_endpoint, tx, rx;
+	cptr_t tx_xmit, rx_xmit;
 	struct thc_channel *chnl;
 	struct net_info *net_info;
 
@@ -171,12 +173,25 @@ static int do_one_register(cptr_t register_chnl)
 		LIBLCD_ERR("cptr alloc failed");
 		goto fail3;
 	}
+	ret = lcd_cptr_alloc(&tx_xmit);
+	if (ret) {
+		LIBLCD_ERR("cptr alloc failed");
+		goto fail2;
+	}
+	ret = lcd_cptr_alloc(&rx_xmit);
+	if (ret) {
+		LIBLCD_ERR("cptr alloc failed");
+		goto fail3;
+	}
+
 	/*
 	 * Set up regs and poll
 	 */
 	lcd_set_cr0(sync_endpoint);
 	lcd_set_cr1(tx);
 	lcd_set_cr2(rx);
+	lcd_set_cr3(tx_xmit);
+	lcd_set_cr4(rx_xmit);
 	ret = lcd_sync_poll_recv(register_chnl);
 	if (ret) {
 		if (ret == -EWOULDBLOCK)
@@ -200,6 +215,18 @@ static int do_one_register(cptr_t register_chnl)
 		LIBLCD_ERR("error setting up ring channel");
 		goto fail6;
 	}
+
+	LIBLCD_MSG("settingup xmit channel");
+	/*
+	 * Set up async ring channel
+	 */
+	ret = setup_async_net_ring_channel(tx_xmit, rx_xmit,
+					&xmit_chnl);
+	if (ret) {
+		LIBLCD_ERR("error setting up ring channel");
+		goto fail6;
+	}
+
 	/*
 	 * Add to dispatch loop
 	 */
@@ -211,10 +238,19 @@ static int do_one_register(cptr_t register_chnl)
 		goto fail7;
 	}
 
+	net_info = add_net(xmit_chnl, c_cspace, sync_endpoint);
+	if (!net_info) {
+		LIBLCD_ERR("error adding to dispatch loop");
+		goto fail7;
+	}
+
 	LIBLCD_MSG("Returning from %s", __func__);
 	lcd_set_cr0(CAP_CPTR_NULL);
 	lcd_set_cr1(CAP_CPTR_NULL);
 	lcd_set_cr2(CAP_CPTR_NULL);
+	lcd_set_cr3(CAP_CPTR_NULL);
+	lcd_set_cr4(CAP_CPTR_NULL);
+
 	if (lcd_sync_reply())
 		LIBLCD_ERR("Error reply");
 	return 0;
@@ -226,11 +262,15 @@ free_cptrs:
 	lcd_set_cr0(CAP_CPTR_NULL);
 	lcd_set_cr1(CAP_CPTR_NULL);
 	lcd_set_cr2(CAP_CPTR_NULL);
+	lcd_set_cr3(CAP_CPTR_NULL);
+	lcd_set_cr4(CAP_CPTR_NULL);
 	lcd_cptr_free(sync_endpoint);
 fail3:
 	lcd_cptr_free(tx);
+	lcd_cptr_free(tx_xmit);
 fail2:
 	lcd_cptr_free(rx);
+	lcd_cptr_free(rx_xmit);
 fail1:
 	return ret;
 }