Merge in Michael's changes with mine.

Looks like we were both headed in the same direction ...

src/common/awe_mapper.c

@@ -35,6 +35,11 @@
  * storage size requirements.
  */
 
+/*
+ * This value is used to determine if a slot is allocated but not yet set in the awe table.
+ */
+static unsigned long initialized_marker = 0xDeadBeef;
+
 
 /*
  * Initilaizes awe mapper.
@@ -92,7 +97,7 @@ awe_mapper_create_id(void)
     } 
     while( is_slot_allocated(awe_map->next_id) );
 
-    (awe_map->awe_list)[awe_map->next_id] = (void*)0xdeadbeef;
+    (awe_map->awe_list)[awe_map->next_id] = (void*)initialized_marker;
 
     awe_map->used_slots++;
 
@@ -101,6 +106,7 @@ awe_mapper_create_id(void)
 EXPORT_SYMBOL(awe_mapper_create_id);
 
 
+
 /*
  * Marks provided id as available
  */
@@ -121,6 +127,7 @@ awe_mapper_remove_id(uint32_t id)
 EXPORT_SYMBOL(awe_mapper_remove_id);
 
 
+
 /*
  * Links awe_ptr with id.
  */
@@ -147,22 +154,3 @@ awe_mapper_get_awe_ptr(uint32_t id)
 
     return (awe_map->awe_list)[id];
 }
-
-
-void 
-LIBASYNC_FUNC_ATTR 
-awe_mapper_print_list(void)
-{
-    int i,j;
-    awe_table_t *awe_map =  get_awe_map();
-    for(i = 0; i < AWE_TABLE_COUNT; i+= 8)
-    {
-        printk(KERN_ERR "\n");
-        for(j = i; j < i + 8; j++)
-        {
-            printk(KERN_ERR "0x%p ", (awe_map->awe_list)[j]);
-        }
-    }
-    printk(KERN_ERR "\n");
-}
-EXPORT_SYMBOL(awe_mapper_print_list);

src/common/thc.c

@@ -1072,6 +1072,7 @@ void
 LIBASYNC_FUNC_ATTR 
 THCYieldToIdAndSave(uint32_t id_to, uint32_t id_from) {
   awe_t *awe_ptr = (awe_t *)awe_mapper_get_awe_ptr(id_to);
+
   if (PTS() == awe_ptr->pts) {
     CALL_CONT_LAZY_AND_SAVE((void*)&thc_yieldto_with_cont, id_from, (void*)awe_ptr);
   }

src/common/thc_ipc.c

@@ -205,6 +205,18 @@ thc_ipc_reply(struct fipc_ring_channel *chnl,
 }
 EXPORT_SYMBOL(thc_ipc_reply);
 
+int
+LIBASYNC_FUNC_ATTR
+thc_ipc_reply_with_id(struct fipc_ring_channel *chnl,
+	uint32_t msg_id,
+	struct fipc_message *response)
+{
+    thc_set_msg_type(response, msg_type_response);
+    thc_set_msg_id(response, msg_id);
+    return fipc_send_msg_end(chnl, response);
+}
+EXPORT_SYMBOL(thc_ipc_reply_with_id);
+
 int 
 LIBASYNC_FUNC_ATTR 
 thc_channel_group_init(struct thc_channel_group* channel_group)

src/include/awe_mapper.h

@@ -63,13 +63,15 @@ static inline awe_table_t* get_awe_map(void)
     return PTS()->awe_map;
 }
 
+/*
+ * Called in awe_mapper_init.
+ * Sets awe_map struct of the current PTS to a specific awe_table.
+ */
 static inline void set_awe_map(awe_table_t * map_ptr)
 {
     PTS()->awe_map = map_ptr;
 }
 
-void awe_mapper_print_list(void);
-
 #endif /* LINUX_KERNEL */
 
 #endif

src/include/thc_ipc.h

@@ -43,34 +43,124 @@ static inline uint32_t thc_get_request_cookie(struct fipc_message *request)
 {
 	return thc_get_msg_id(request);
 }
-
+/*
+ * thc_ipc_recv
+ * 
+ * Performs an async receive of a message.
+ * msg_id is used to receive a message with a particular id and must always
+ * be valid. If a message is received but its id does not match the id provided
+ * to this function, then this function will check if the message corresponds
+ * to another pending AWE, and if so, it will yield to that AWE. If there is 
+ * not a message that has been received yet, it will yield to dispatch any
+ * pending work. If a message is received that corresponds to the provided
+ * msg_id, this function will put the message in the *out_msg parameter and return.
+ * This last case is the only time the function will return assuming there are
+ * no error conditions. In the other two cases involving yields, execution will
+ * eventually come back to this function until it receives the message
+ * corresponding to its msg_id. 
+ *  It is the caller's responsibility to mark the message as having completed
+ * the receive. (ex: fipc_recv_msg_end(chnl, msg))
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_ipc_recv(struct fipc_ring_channel *chnl, 
                  unsigned long msg_id, 
                  struct fipc_message** out_msg);
 
+/*
+ * thc_ipc_call
+ * 
+ * Performs both a send and an async receive. 
+ * request is the message to send.
+ * (*response) will have the received message.
+ *
+ * It is the caller's responsibility to mark the message as having completed
+ * the receive. (ex: fipc_recv_msg_end(chnl, msg))
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_ipc_call(struct fipc_ring_channel *chnl, 
 		struct fipc_message *request, 
 		struct fipc_message **response);
 
+/*
+ * thc_ipc_reply
+ *
+ * Sets the msg_id of response to the msg_id of request
+ * and marks the response message as having a response message type.
+ * Sends response message on the provided ring channel. 
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_ipc_reply(struct fipc_ring_channel *chnl,
-		struct fipc_message *request,
+		uint32_t request_cookie,
 		struct fipc_message *response);
 
-int thc_poll_recv_group(struct thc_channel_group* chan_group, 
-                        struct thc_channel_group_item** chan_group_item, 
-                        struct fipc_message** out_msg);
-
+/* thc_poll_recv
+ *
+ * Same as thc_ipc_recv except that if no message is present, it
+ * returns -EWOULDBLOCK instead of yielding.
+ * Additionally, the channel inside the thc_channel_group_item is what
+ * is used instead of a channel directly.
+ * There is also no msg_id passed in because poll_recv should not expect a
+ * specific message.
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_poll_recv(struct thc_channel_group_item* item,
                  struct fipc_message** out_msg);
 
+/*
+ * thc_poll_recv_group
+ *
+ * Calls thc_poll_recv on a list of thc_channel_group_items represented
+ * by thc_channel_group. If there is a message available that does not
+ * correspond to a pending AWE, the thc_channel_group_item that corresponds
+ * to the received message is set to the out param (*chan_group_item),
+ * and the received message is set to the out param (*out_msg).
+ * If there is no message available in any of the thc_channel_group_items,
+ * then this function returns -EWOULDBLOCK.
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
+int thc_poll_recv_group(struct thc_channel_group* chan_group, 
+                        struct thc_channel_group_item** chan_group_item, 
+                        struct fipc_message** out_msg);
+/*
+ * thc_channel_group_init
+ *
+ * Initializes thc_channel_group structure after it has been allocated.
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_channel_group_init(struct thc_channel_group* channel_group);
 
+/*
+ * thc_channel_group_item_add
+ *
+ * Adds a thc_channel_group_item to a thc_channel_group.
+ *
+ * Returns 0 on success, non-zero otherwise.
+ */
 int thc_channel_group_item_add(struct thc_channel_group* channel_group,
                           struct thc_channel_group_item* item);
 
+/*
+ * thc_channel_group_item_add
+ *
+ * Removes a thc_channel_group_item from a thc_channel_group.
+ */
 void thc_channel_group_item_remove(struct thc_channel_group* channel_group, 
 				struct thc_channel_group_item* item);
 
+/*
+ * thc_channel_group_item_get
+ *
+ * Gets a thc_channel_group_item at a particular index in a channel_group.
+ * Puts the thc_channel_group_item in the out parameter (*out_item) on success.
+ *
+ * Returns 0 on success. Returns 1 if index is out of range.
+ */
 int thc_channel_group_item_get(struct thc_channel_group* channel_group, 
                                int index, 
                                struct thc_channel_group_item **out_item);

src/include/thc_ipc_types.h

@@ -19,6 +19,12 @@ struct predicate_payload
     msg_type_t msg_type;
 };
 
+/*
+ * struct thc_channel_group_item
+ *
+ * Contains a channel and a function that should get called when a message
+ * is received on the channel.
+ */
 struct thc_channel_group_item
 {
     struct list_head list;
@@ -26,6 +32,11 @@ struct thc_channel_group_item
     int (*dispatch_fn)(struct fipc_ring_channel*, struct fipc_message*);
 };
 
+/*
+ * struct thc_channel_group
+ *
+ * Represents a linked list of thc_channel_group_items.
+ */
 struct thc_channel_group
 {
     struct list_head head;

src/tests/Kbuild.in

 
 obj-m += simple/
-#obj-m += dispatch/
-#obj-m += ctx-switch/
-#obj-m += async-msg-benchmarks/
+obj-m += dispatch/
+obj-m += ctx-switch/
+obj-m += async-msg-benchmarks/
 

src/tests/async-msg-benchmarks/callee.c

@@ -69,34 +69,34 @@ static inline int send_response(struct fipc_ring_channel *chnl,
 {
 	int ret;
 	struct fipc_message *response;
-	/*
-	 * Mark recvd msg slot as available
-	 */
-	ret = fipc_recv_msg_end(chnl, recvd_msg);
-	if (ret) {
-		pr_err("Error marking msg as recvd");
-		return ret;
-	}
+
 	/*
 	 * Response
 	 */
 	ret = test_fipc_blocking_send_start(chnl, &response);
-	if (ret) {
+	if ( ret ) {
 		pr_err("Error getting send slot");
 		return ret;
 	}
-    
-    THC_MSG_TYPE(response) = msg_type_response;
-    THC_MSG_ID(response)   = THC_MSG_ID(recvd_msg);
+
 	set_fn_type(response, type);
 	response->regs[0] = val;
 
-	ret = fipc_send_msg_end(chnl, response);
-	if (ret) {
+	ret = thc_ipc_reply(chnl, recvd_msg, response);
+	if ( ret ) {
 		pr_err("Error marking message as sent");
 		return ret;
 	}
 
+    	/*
+	 * Mark recvd msg slot as available
+	 */
+	ret = fipc_recv_msg_end(chnl, recvd_msg);
+	if ( ret ) {
+		pr_err("Error marking msg as recvd");
+		return ret;
+	}
+
 	return 0;
 }
 

src/tests/async-msg-benchmarks/caller.c

@@ -16,12 +16,10 @@
 
 static unsigned long msg_times[TRANSACTIONS];
 
-static inline int send_and_get_response(
+static inline int send_and_get_response_sync(
 	struct fipc_ring_channel *chan,
 	struct fipc_message *request,
-	struct fipc_message **response,
-    uint32_t msg_id,
-    bool is_async)
+	struct fipc_message **response)
 {
 	int ret;
 	struct fipc_message *resp;
@@ -37,15 +35,10 @@ static inline int send_and_get_response(
 	/*
 	 * Try to get the response
 	 */
-    if( is_async )
-    {
-        ret = thc_ipc_recv(chan, msg_id, &resp);
-    }
-    else
-    {
-        ret = test_fipc_blocking_recv_start(chan, &resp);
-    }
-	if (ret) {
+        
+    ret = test_fipc_blocking_recv_start(chan, &resp);
+	
+    if (ret) {
 		pr_err("failed to get a response, ret = %d\n", ret);
 		goto fail2;
 	}
@@ -93,8 +86,13 @@ static inline int finish_response_check_fn_type_and_reg0(
 	enum fn_type actual_type = get_fn_type(response);
 	unsigned long actual_reg0 = fipc_get_reg0(response);
 
+    ret = fipc_recv_msg_end(chnl, response);
 
-	if (actual_type != expected_type) {
+	if (ret) {
+		pr_err("Error finishing receipt of response, ret = %d\n", ret);
+		return ret;
+	} 
+    else if (actual_type != expected_type) {
 		pr_err("Unexpected fn type: actual = %d, expected = %d\n",
 			actual_type, expected_type);
 		return -EINVAL;
@@ -116,7 +114,6 @@ static int noinline __used add_nums(struct fipc_ring_channel *chan,
 	struct fipc_message *request;
 	struct fipc_message *response;
     unsigned long start_time, stop_time;
-    uint32_t msg_id;
 	int ret;
 	/*
 	 * Set up request
@@ -128,20 +125,22 @@ static int noinline __used add_nums(struct fipc_ring_channel *chan,
 		pr_err("Error getting send message, ret = %d\n", ret);
 		goto fail;
 	}
-    if( is_async )
-    {
-        msg_id                = awe_mapper_create_id();
-    }
 
-    THC_MSG_TYPE(request) = msg_type_request;
-    THC_MSG_ID(request)   = msg_id;
 	set_fn_type(request, ADD_NUMS);
 	fipc_set_reg0(request, trans);
 	fipc_set_reg1(request, res1);
+
+    if( is_async )
+    {
+        ret = thc_ipc_call(chan, request, &response);
+    }
+    else
+    {
+	    ret = send_and_get_response_sync(chan, request, &response);
+    }
 	/*
 	 * Send request, and get response
 	 */
-	ret = send_and_get_response(chan, request, &response, msg_id, is_async);
     stop_time = test_fipc_stop_stopwatch();
 
     msg_times[trans] = stop_time - start_time;
@@ -151,7 +150,6 @@ static int noinline __used add_nums(struct fipc_ring_channel *chan,
 		goto fail;
 	}
 
-
 	/*
 	 * Maybe check message
 	 */

src/tests/async-msg-benchmarks/rpc.h

@@ -9,7 +9,7 @@
 #define LIBFIPC_RPC_TEST_H
 
 #include <libfipc.h>
-
+#include <thc_ipc.h>
 enum fn_type {
 	NULL_INVOCATION, 
 	ADD_CONSTANT, 
@@ -21,24 +21,10 @@ enum fn_type {
 };
 
 /* must be divisible by 6... because I call 6 functions in the callee.c */
-#define TRANSACTIONS 60000
+#define TRANSACTIONS 600
 
 /* thread main functions */
 int callee(void *_callee_channel_header);
 int caller(void *_caller_channel_header);
 
-static inline
-int
-get_fn_type(struct fipc_message *msg)
-{
-	return fipc_get_flags(msg);
-}
-
-static inline
-void
-set_fn_type(struct fipc_message *msg, enum fn_type type)
-{
-	fipc_set_flags(msg, type);
-}
-
 #endif /* LIBFIPC_RPC_TEST_H */

src/tests/dispatch/rpc.h

-/*
- * rpc.h
- *
- * Internal defs
- *
- * Copyright: University of Utah
- */
-#ifndef LIBFIPC_RPC_TEST_H
-#define LIBFIPC_RPC_TEST_H
-
-#include <libfipc.h>
-
-static inline
-int
-get_fn_type(struct fipc_message *msg)
-{
-	return fipc_get_flags(msg);
-}
-
-static inline
-void
-set_fn_type(struct fipc_message *msg, uint32_t type)
-{
-	fipc_set_flags(msg, type);
-}
-
-#endif /* LIBFIPC_RPC_TEST_H */

src/tests/dispatch/thread1_fn.c

@@ -5,7 +5,6 @@
 #include "thc_dispatch_test.h"
 #include "thread_fn_util.h"
 #include "../test_helpers.h"
-#include "rpc.h"
 #include <awe_mapper.h>
 #include <linux/delay.h>
 
@@ -42,11 +41,61 @@ static void check_response(unsigned long lhs,
     }
 }
 
-static unsigned long add_nums_async(unsigned long lhs, unsigned long rhs, unsigned long msg_id, int fn_type)
+static inline int finish_response_check_fn_type_and_reg0(
+	struct fipc_ring_channel *chnl,
+	struct fipc_message *response,
+	uint32_t expected_type,
+	unsigned long expected_lhs,
+	unsigned long expected_rhs)
+{
+	int ret;
+	uint32_t actual_type      = get_fn_type(response);
+	unsigned long actual_reg0 = fipc_get_reg0(response);
+    unsigned long expected_reg0;
+
+	ret = fipc_recv_msg_end(chnl, response);
+
+    switch( expected_type )
+    {
+        case ADD_2_FN:
+            expected_reg0 = expected_lhs + expected_rhs;
+            break;
+        case ADD_10_FN:
+            expected_reg0 = expected_lhs + expected_rhs + 10;
+            break;
+        default:
+            printk(KERN_ERR "invalid fn_type %d\n", expected_type);
+            return -EINVAL;
+    }
+
+	if (ret) {
+		pr_err("Error finishing receipt of response, ret = %d\n", ret);
+		return ret;
+	} else if (actual_type != expected_type) {
+		pr_err("Unexpected fn type: actual = %u, expected = %u\n",
+			actual_type, expected_type);
+		return -EINVAL;
+	} else if (actual_reg0 != expected_reg0) {
+		pr_err("Unexpected return value (reg0): actual = 0x%lx, expected = 0x%lx\n",
+			actual_reg0, expected_reg0);
+		return -EINVAL;
+
+	} else {
+		return 0;
+	}
+}
+
+
+
+static int add_nums_async(struct fipc_ring_channel* channel,
+        unsigned long lhs, 
+        unsigned long rhs, 
+        int fn_type)
 {
 	struct fipc_message *msg;
     struct fipc_message *response;
-	unsigned long result;
+	int ret;
+
 	if( test_fipc_blocking_send_start(channel, &msg) )
     {
         printk(KERN_ERR "Error getting send message for add_nums_async.\n");
@@ -54,22 +103,31 @@ static unsigned long add_nums_async(unsigned long lhs, unsigned long rhs, unsign
 	set_fn_type(msg, fn_type);
 	fipc_set_reg0(msg, lhs);
 	fipc_set_reg1(msg, rhs);
-	THC_MSG_ID(msg)   = msg_id;
-    THC_MSG_TYPE(msg) = msg_type_request;
-    send_and_get_response(channel, msg, &response, msg_id);
-    fipc_recv_msg_end(channel, response);
-	result = fipc_get_reg0(response);
-	
-	return result;
+
+    ret = thc_ipc_call(channel, msg, &response);
+
+  	if ( ret ) {
+		printk(KERN_ERR "Error getting response, ret = %d\n", ret);
+		goto fail;
+	}
+
+	return finish_response_check_fn_type_and_reg0(
+		channel,
+		response, 
+		fn_type,
+        lhs,
+        rhs);
+
+fail:
+    return ret;
 }
 
 
 static int run_thread1(void* chan)
 {
-    volatile int num_transactions = 0;
+    volatile int num_transactions    = 0;
     int print_transactions_threshold = 0;
-    unsigned long msg_response    = 0;
-	uint32_t id_num;
+    unsigned long msg_response       = 0;
     num_responses = 0;
     channel       = chan;
 
@@ -83,30 +141,21 @@ static int run_thread1(void* chan)
         }
 
 		ASYNC(
-			id_num = awe_mapper_create_id();
 		    num_transactions++;
-            volatile int old_trans = num_transactions;
-			msg_response = add_nums_async(old_trans, 1,(unsigned long) id_num, ADD_2_FN);
-            check_response(old_trans, 1, msg_response, ADD_2_FN);
+			msg_response = add_nums_async(channel, num_transactions, 1, ADD_2_FN);
             num_responses++;
 		     );
         if( (num_transactions) % THD3_INTERVAL == 0 )
         {
             ASYNC(
-                id_num = awe_mapper_create_id();
 		        num_transactions++;
-                volatile int old_trans = num_transactions;
-                msg_response = add_nums_async(old_trans, 2,(unsigned long) id_num, ADD_10_FN);
-                check_response(old_trans, 2, msg_response, ADD_10_FN);
+                msg_response = add_nums_async(channel, num_transactions, 2, ADD_10_FN);
                 num_responses++;
                  );
         }
 		ASYNC(
-			id_num = awe_mapper_create_id();
 		    num_transactions++;
-            volatile int old_trans = num_transactions;
-			msg_response = add_nums_async(old_trans, 3,(unsigned long) id_num, ADD_2_FN);
-            check_response(old_trans, 3, msg_response, ADD_2_FN);
+			msg_response = add_nums_async(channel, num_transactions, 3, ADD_2_FN);
             num_responses++;
 		     );
             //msleep(1);

src/tests/dispatch/thread2_fn.c

@@ -3,7 +3,6 @@
 #include <thcinternal.h>