Skip to content
GitLab
Commits (56)
Expand all Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 02123f67
*.sw*
*.o
*.ko
*.cmd
......@@ -6,6 +7,7 @@
*.symvers
*.mod.c
*Makefile.in
*.patch
aclocal.m4
autom4te.cache
config.h.in
......
.gitmodules
View file @ 02123f67
[submodule "fast-ipc-module"]
path = fast-ipc-module
url = git@gitlab.flux.utah.edu:ramstore/fast-ipc-module.git
branch = one_slot
configure.ac
View file @ 02123f67
......@@ -26,6 +26,23 @@ AM_COND_IF([ENABLE_KERNEL_TESTS_BUILD],
[AC_MSG_NOTICE([Building kernel tests.])],
[AC_MSG_NOTICE([Not building kernel tests.])])
# Lazy or Eager?
AC_ARG_ENABLE([eager],
[AS_HELP_STRING([--enable-eager]),
[Enable building libasync for eager stack allocation])],
[],
[enable_eager="no"])
AS_IF([test "x$enable_eager" = "xyes"],
[
EAGER_OR_LAZY=""
AC_MSG_NOTICE([Building for EAGER stack allocation.])
],
[
EAGER_OR_LAZY="-DCONFIG_LAZY_THC"
AC_MSG_NOTICE([Building for LAZY stack allocation.])
])
AC_SUBST(EAGER_OR_LAZY)
# --------------------------------------------------
# Checks for programs.
# --------------------------------------------------
......@@ -114,6 +131,9 @@ AC_CONFIG_FILES(
src/tests/simple/Kbuild
src/tests/dispatch/Kbuild
src/tests/ctx-switch/Kbuild
src/tests/async-msg-benchmarks/Kbuild
src/tests/abort/Kbuild
)
AC_OUTPUT
fast-ipc-module @ c27711e3
Compare 255e1b8c...c27711e3
Subproject commit 255e1b8cbdba34386bacaac425792d91478675e2
Subproject commit c27711e3dc09dfbf6132c1dbc82eb73ff44bb126
src/Makefile.am
View file @ 02123f67
......@@ -15,7 +15,7 @@ AM_CPPFLAGS = \
-I$(abs_top_builddir)/src/include \
-I$(LIBFIPC_DIR)/include
AM_CFLAGS = -DLINUX_KERNEL -DCONFIG_LAZY_THC
AM_CFLAGS = -DLINUX_KERNEL $(EAGER_OR_LAZY)
COMMON_SRCS= \
common/thc.c \
......
src/common/awe_mapper.c
View file @ 02123f67
......@@ -14,9 +14,18 @@
#include <lcd_config/pre_hook.h>
#endif
#ifdef LINUX_KERNEL
#include <linux/bug.h>
#include <linux/string.h>
#include <linux/slab.h>
#define printf printk
#else
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#endif
#include <awe_mapper.h>
#ifdef LCD_DOMAINS
......@@ -28,29 +37,57 @@
#define EXPORT_SYMBOL(x)
#endif
/*
* NOTE: This implementation right now is just a ring buffer.
* In the future, we probably want to change this to something
* like a red black tree or a B-tree to account for differing
* storage size requirements.
/* Linux kernel only provides ffs variant, which operates on 32-bit registers.
* For promoting the bsf instruction to 64-bit, intel manual suggests to use
* REX.W prefix to the instruction. However, when the operands are 64-bits, gcc
* already promotes bsf to 64-bit.
*/
static __always_inline int ffsll(long long x)
{
long long r;
/*
* AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
* dest reg is undefined if x==0, but their CPU architect says its
* value is written to set it to the same as before, except that the
* top 32 bits will be cleared.
*
* We cannot do this on 32 bits because at the very least some
* 486 CPUs did not behave this way.
*/
asm("bsf %1,%0"
: "=r" (r)
: "rm" (x), "0" (-1));
return r + 1;
}
/*
* Initilaizes awe mapper.
*/
void
LIBASYNC_FUNC_ATTR
void
inline
LIBASYNC_FUNC_ATTR
awe_mapper_init(void)
{
void* awe_map_ptr = kzalloc(sizeof(awe_table_t), GFP_KERNEL);
if( !awe_map_ptr )
#ifdef LINUX_KERNEL
awe_table_t* awe_map;
awe_map = kzalloc(sizeof(awe_table_t), GFP_KERNEL);
#else
awe_table_t* awe_map = calloc(sizeof(awe_table_t), 1);
#endif
if (!awe_map)
{
pr_err("No space left for awe_map_ptr\n");
printf("No space left for awe_map_ptr\n");
return;
}
set_awe_map((awe_table_t*) awe_map_ptr);
#ifdef LINUX_KERNEL
awe_map->awe_bitmap = ~0UL;
#else
awe_map->awe_bitmap = ~0LL;
#endif
set_awe_map((awe_table_t*) awe_map);
current->ptstate = PTS();
}
......@@ -58,111 +95,54 @@ awe_mapper_init(void)
/*
* Uninitilaizes awe mapper.
*/
void
LIBASYNC_FUNC_ATTR
void
inline
LIBASYNC_FUNC_ATTR
awe_mapper_uninit(void)
{
awe_table_t *awe_map = get_awe_map();
kfree(awe_map);
}
static bool is_slot_allocated(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
return ((awe_map->awe_list)[id] != NULL);
if (awe_map) {
#ifdef LINUX_KERNEL
kfree(awe_map);
#else
free(awe_map);
#endif
set_awe_map(NULL);
}
}
/*
* Returns new available id.
*/
uint32_t
LIBASYNC_FUNC_ATTR
awe_mapper_create_id(void)
int inline
_awe_mapper_create_id(awe_table_t *awe_map)
{
awe_table_t *awe_map = get_awe_map();
BUG_ON((awe_map->used_slots >= AWE_TABLE_COUNT) && "Too many slots have been requested.");
do
{
awe_map->next_id = (awe_map->next_id + 1) % AWE_TABLE_COUNT;
}
while( is_slot_allocated(awe_map->next_id) );
(awe_map->awe_list)[awe_map->next_id] = (void*)0xdeadbeef;
awe_map->used_slots++;
return awe_map->next_id;
}
EXPORT_SYMBOL(awe_mapper_create_id);
/*
* Marks provided id as available
*/
void
LIBASYNC_FUNC_ATTR
awe_mapper_remove_id(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
BUG_ON(id >= AWE_TABLE_COUNT);
if( awe_map->used_slots > 0 )
{
awe_map->used_slots--;
}
(awe_map->awe_list)[id] = NULL;
}
EXPORT_SYMBOL(awe_mapper_remove_id);
int id;
#ifdef LINUX_KERNEL
id = ffsll(awe_map->awe_bitmap);
#elif defined(linux)
id = __builtin_ffsll(awe_map->awe_bitmap);
#endif
awe_map->awe_bitmap &= ~(1LL << (id - 1));
/*
* Links awe_ptr with id.
*/
void
LIBASYNC_FUNC_ATTR
awe_mapper_set_id(uint32_t id, void* awe_ptr)
{
awe_table_t *awe_map = get_awe_map();
BUG_ON(id >= AWE_TABLE_COUNT);
(awe_map->awe_list)[id] = awe_ptr;
assert(id != 0);
#ifndef NDEBUG
if (!id)
printk("%s, id %d, bitmap %016llx\n", __func__, id, awe_map->awe_bitmap);
#endif
return id;
}
/*
* Returns awe_ptr that corresponds to id.
* Returns new available id.
*/
void*
LIBASYNC_FUNC_ATTR
awe_mapper_get_awe_ptr(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
BUG_ON(id >= AWE_TABLE_COUNT);
return (awe_map->awe_list)[id];
}
void
int inline
LIBASYNC_FUNC_ATTR
awe_mapper_print_list(void)
awe_mapper_create_id(int *id)
{
int i,j;
awe_table_t *awe_map = get_awe_map();
for(i = 0; i < AWE_TABLE_COUNT; i+= 8)
{
printk(KERN_ERR "\n");
for(j = i; j < i + 8; j++)
{
printk(KERN_ERR "0x%p ", (awe_map->awe_list)[j]);
}
}
printk(KERN_ERR "\n");
*id = _awe_mapper_create_id(awe_map);
return 0;
}
EXPORT_SYMBOL(awe_mapper_print_list);
EXPORT_SYMBOL(awe_mapper_create_id);
src/common/thc.c
View file @ 02123f67
This diff is collapsed.
src/common/thc_ipc.c
View file @ 02123f67
This diff is collapsed.
src/include/awe_mapper.h
View file @ 02123f67
......@@ -14,15 +14,17 @@
#include <thc.h>
#ifdef LINUX_KERNEL
#define AWE_TABLE_COUNT 128
#define AWE_TABLE_ORDER 10
#define AWE_TABLE_COUNT (1 << AWE_TABLE_ORDER)
//assert(AWE_TABLE_ORDER == 6);
struct awe_table
{
void* awe_list[AWE_TABLE_COUNT];
uint32_t used_slots;
uint32_t next_id;
awe_t awe_list[AWE_TABLE_COUNT];
long long awe_bitmap;
// uint32_t used_slots;
// uint32_t next_id;
};
typedef struct awe_table awe_table_t;
......@@ -38,37 +40,130 @@ void awe_mapper_init(void);
void awe_mapper_uninit(void);
/*
* Returns new available id.
* Returns new available id as out param.
*
* Returns non-zero on failure.
*/
int awe_mapper_create_id(int *);
static inline awe_table_t* get_awe_map(void)
{
return PTS()->awe_map;
}
/*
* Called in awe_mapper_init.
* Sets awe_map struct of the current PTS to a specific awe_table.
*/
uint32_t awe_mapper_create_id(void);
static inline void set_awe_map(awe_table_t * map_ptr)
{
PTS()->awe_map = map_ptr;
}
static inline void
_awe_mapper_remove_id(awe_table_t *awe_map, uint32_t id)
{
assert(id <= AWE_TABLE_COUNT);
assert(!(awe_map->awe_bitmap & (1 << (id - 1))));
awe_map->awe_bitmap |= (1 << (id - 1));
}
/*
* Marks provided id as available
*/
void awe_mapper_remove_id(uint32_t id);
static inline void
LIBASYNC_FUNC_ATTR
awe_mapper_remove_id(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
_awe_mapper_remove_id(awe_map, id);
}
//static inline void
//_awe_mapper_set_id(awe_table_t *awe_map, uint32_t id, void* awe_ptr)
//{
// assert(id < AWE_TABLE_COUNT);
// (awe_map->awe_list)[id] = awe_ptr;
//}
/*
* Links awe_ptr with id.
*/
void awe_mapper_set_id(uint32_t id, void* awe_ptr);
//static inline void
//awe_mapper_set_id(uint32_t id, void* awe_ptr)
//{
// awe_table_t *awe_map = get_awe_map();
// _awe_mapper_set_id(awe_map, id, awe_ptr);
//}
static inline int _is_slot_allocated(awe_table_t *awe_map, uint32_t id)
{
return !(awe_map->awe_bitmap & (1 << (id - 1)));
}
static inline int is_slot_allocated(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
return _is_slot_allocated(awe_map, id);
}
static inline awe_t *
_awe_mapper_get_awe(awe_table_t *awe_map, uint32_t id)
{
//assert(id >= AWE_TABLE_COUNT);
if (id > AWE_TABLE_COUNT) {
printk("%s, id(%u) > table count(%u)\n", __func__, id, AWE_TABLE_COUNT);
}
if (!awe_map) {
printk("%s, awemap for id %d = %p\n", __func__, id, awe_map);
#ifdef LCD_DOMAINS
BUG();
#endif
return NULL;
}
if (!_is_slot_allocated(awe_map, id))
return NULL;
assert(id <= AWE_TABLE_COUNT);
return &(awe_map->awe_list[id]);
}
/*
* Returns awe_ptr that corresponds to id.
*/
void* awe_mapper_get_awe_ptr(uint32_t id);
static inline awe_table_t* get_awe_map(void)
static inline awe_t*
LIBASYNC_FUNC_ATTR
awe_mapper_get_awe(uint32_t id)
{
return PTS()->awe_map;
awe_table_t *awe_map = get_awe_map();
if (!awe_map) {
printk("%s, awe_map for id %d = %p\n", __func__, id, awe_map);
#ifdef LCD_DOMAINS
BUG();
#endif
return NULL;
}
#ifdef LCD_DOMAINS
// printk("%s, awe_map %p | pts()->awemap %p, awemap2 %p | id %d\n",
// __func__, awe_map, PTS()->awe_map, PTS()->awe_map2, id);
#endif
return _awe_mapper_get_awe(awe_map, id);
}
static inline void set_awe_map(awe_table_t * map_ptr)
static inline awe_t *
LIBASYNC_FUNC_ATTR
_awe_mapper_get_awe_ptr_trusted(awe_table_t *awe_map, uint32_t id)
{
PTS()->awe_map = map_ptr;
return &(awe_map->awe_list[id]);
}
void awe_mapper_print_list(void);
#endif /* LINUX_KERNEL */
static inline awe_t*
LIBASYNC_FUNC_ATTR
awe_mapper_get_awe_ptr_trusted(uint32_t id)
{
awe_table_t *awe_map = get_awe_map();
return _awe_mapper_get_awe_ptr_trusted(awe_map, id);
}
#endif
src/include/thc.h
View file @ 02123f67
/* Barrelfish THC language extensions */
#ifndef _THC_H_
#define _THC_H_
#ifndef LIBASYNC_FUNC_ATTR
#define LIBASYNC_FUNC_ATTR
#define LIBASYNC_FUNC_ATTR
#endif
#define USE_ASSERT
#ifdef LINUX_KERNEL
#include <linux/types.h>
#include <linux/printk.h>
#define assert(XX) do { \
if (!(XX)) { \
printk("assertion failure at %s:%d\n", \
__FILE__, __LINE__); \
} \
} while(0)
#ifdef USE_ASSERT
#define assert(XX) do { \
if (!(XX)) { \
printk("assertion failure at %s:%d\n", \
__FILE__, __LINE__); \
} \
} while(0)
#else
#define assert(XX) do {} while(0)
#endif
#else
#include <stdint.h>
#include <stdlib.h>
#endif
#include <assert.h>
#endif /* LINUX_KERNEL */
#ifndef BARRELFISH
typedef int errval_t;
#define SYS_ERR_OK 0
#define THC_CANCELED 1
#endif
// The implementation of do..finish relies on shadowing so that
// _fb_info always refers to the closest enclosing do..finish block.
#pragma GCC diagnostic ignored "-Wshadow"
/*......................................................................*/
// Macros for use from user code
//
// Cancel. _TAG is the name of an enclosig DO_FINISH block.
#define CANCEL(_TAG) \
do { \
_thc_do_cancel_request(_fb_info_ ## _TAG); \
} while (0)
// DO_FINISH block. Non-cancellable:
#define DO_FINISH_NX(_CODE) DO_FINISH__(__,_CODE,1)
// Cancellable do-finish, with tag:
#define DO_FINISH_(_TAG,_CODE) DO_FINISH__(_TAG,_CODE,0)
// Cancellable do-finish, no tag:
#define DO_FINISH(_CODE) DO_FINISH__(__,_CODE,0)
// ASYNC implementation. __COUNTER__ is a GCC extension that will
// allocate a unique integer ID on each expansion. Hence use it once
// here and use the resulting expanded value in ASYNC_:
#define ASYNC(_BODY) ASYNC_(_BODY, __COUNTER__)
#ifdef CONFIG_LAZY_THC
// DO_FINISH implementation:
#define DO_FINISH__(_TAG,_CODE,_IS_NX) \
do { \
finish_t _fb; \
finish_t *_fb_info __attribute__((unused)) = &_fb; \
finish_t *_fb_info_ ## _TAG __attribute__((unused)) = _fb_info; \
void *_fb_curr_stack __attribute__((unused)); \
FORCE_FRAME_POINTER_USE; \
GET_STACK_POINTER(_fb_info->old_sp); \
_thc_startfinishblock(_fb_info, _IS_NX); \
do { _CODE } while (0); \
GET_STACK_POINTER(_fb_curr_stack); \
_thc_endfinishblock(_fb_info, _fb_curr_stack); \
if (_fb_info->old_sp != _fb_curr_stack) { \
RESTORE_OLD_STACK_POINTER(_fb_info->old_sp); \
_thc_pendingfree(); \
} \
} while (0)
// The basic idea for ASYNC_ is that the contents of the block becomes
// a nested function. For the lazy implementation, we create an AWE for
// the continuation of the block and store a reference to it on the nested
// function's stackframe. We then mark this as an async by replacing the
// return address of the nested function with a special marker. If we block,
// the runtime system looks through the stack for this marker, allocating
// a lazy stack for any AWE continuations it finds.
//
// We are careful to avoid taking the address of the nested function.
// This prevents GCC trying to generate stack-allocated trampoline functions
// (this is not implemented on Beehive where the I and D caches are not
// coherent).
//
// There are several hacks:
//
// 1. The nested function is given an explicit name in the generated
// assembly language code (NESTED_FN_STRING(_C)). This means that
// inline asm can refer to it without needing to take the address
// of the nested function.
//
// 2. The nested function specifically jumps to the point after the async
// continuation rather than returning normally, since (i) we have fiddled
// with the return address to make it a marker and (ii), changing it back
// then returning normally confuses the branch prediction hardware leading
// to an increase in async cost by about 40 cycles (25 cycles -> 65 cycles)
//
#define ASYNC_(_BODY, _C) \
do { \
awe_t _awe; \
extern void * CONT_RET_FN_NAME(_C) (void); \
\
/* Define nested function containing the body */ \
\
/* XXX: With these attributes, we're no longer portable with other \
* compilers ... */ \
auto void \
noinline \
__attribute__((optimize("no-ipa-sra","no-ipa-cp"))) \
_thc_nested_async(FORCE_ARGS_STACK awe_t *awe) \
__asm__(NESTED_FN_STRING(_C)); \
\
noinline \
void _thc_nested_async(FORCE_ARGS_STACK awe_t *awe) { \
void *_my_fb = _fb_info; \
_awe.current_fb = _my_fb; \
INIT_LAZY_AWE(awe, &_thc_lazy_awe_marker); \
do { _BODY; } while (0); \
/* If return address is NULLed then we blocked */ \
if (__builtin_return_address(0) == NULL) { \
/* thc_startasync is performed lazily, we should run */ \
/* _thc_endasync if we blocked*/ \
_thc_endasync(_my_fb, __builtin_frame_address(0)+(2*__WORD_SIZE));\
} \
/* Otherwise, return */ \
RETURN_CONT(CONT_RET_FN_STRING(_C)); \
} \
\
_awe.status = LAZY_AWE; \
_awe.lazy_stack = NULL; \
_awe.pts = NULL; \
\
SCHEDULE_CONT(&_awe, _thc_nested_async); \
__asm__ volatile ( \
" .globl " CONT_RET_FN_STRING(_C) "\n\t" \
" " CONT_RET_FN_STRING(_C) ": \n\t" \
); \
} while (0)
// DO_FINISH implementation:
#define DO_FINISH_(_TAG,_CODE) DO_FINISH__(_TAG,_CODE)
#else // EAGER_THC
// Cancellable do-finish, no tag:
// DO_FINISH implementation:
#define DO_FINISH(_CODE) DO_FINISH__(__,_CODE)
#define DO_FINISH__(_TAG,_CODE,_IS_NX) \
#define DO_FINISH__(_TAG, _CODE) \
do { \
finish_t _fb; \
finish_t *_fb_info __attribute__((unused)) = &_fb; \
finish_t *_fb_info_ ## _TAG __attribute__((unused)) = _fb_info; \
_thc_startfinishblock(_fb_info, _IS_NX); \
_thc_startfinishblock(_fb_info); \
do { _CODE } while (0); \
_thc_endfinishblock(_fb_info, NULL); \
_thc_endfinishblock(_fb_info); \
} while (0)
// The basic idea for ASYNC_ is that the contents of the block becomes
......@@ -195,31 +89,33 @@ typedef int errval_t;
do { \
awe_t _awe; \
void *_new_stack = _thc_allocstack(); \
_awe.current_fb = _fb_info; \
int volatile done_once = 0; \
/* Define nested function containing the body */ \
auto void _thc_nested_async(void) __asm__(NESTED_FN_STRING(_C)); \
__attribute__((noinline,used)) void _thc_nested_async(void) { \
auto void noinline \
__attribute__((used)) _thc_nested_async(void) __asm__(NESTED_FN_STRING(_C)); \
__attribute__((used)) void noinline _thc_nested_async(void) { \
void *_my_fb = _fb_info; \
void *_my_stack = _new_stack; \
_thc_startasync(_my_fb, _my_stack); \
do { _BODY; } while (0); \
_thc_endasync(_my_fb, _my_stack); \
assert(0 && "_thc_endasync returned"); \
/*assert(0 && "_thc_endasync returned");*/ \
} \
\
/* Define function to enter _nested on a new stack */ \
auto void _swizzle(void) __asm__(SWIZZLE_FN_STRING(_C)); \
SWIZZLE_DEF(_swizzle, _new_stack, NESTED_FN_STRING(_C)); \
/*_awe.current_fb = _fb_info;*/ \
\
/* Add AWE for our continuation, then run "_nested" on new stack */ \
if (!SCHEDULE_CONT(&_awe)) { \
_swizzle(); \
assert(0 && "_swizzle returned"); \
} \
SCHEDULE_CONT(&_awe); \
if(!done_once) { \
done_once = 1; \
_swizzle(); \
}/* else */ \
_thc_pendingfree(); \
} while (0)
#endif // CONFIG_LAZY_THC
// Helper macros used in creating the assembly language name for the
// nested function
......@@ -232,7 +128,6 @@ typedef int errval_t;
#define CONT_RET_FN_NAME(C) _thc_cont_return_ ## C
#define CONT_RET_FN_STRING(C) THC_STR(CONT_RET_FN_NAME(C))
/*......................................................................*/
// Prototypes for functions to be called by user code (as opposed to
// by the implementations of compiler intrinsics)
......@@ -282,17 +177,54 @@ void THCSuspendThen(awe_t **awe_ptr_ptr, THCThenFn_t then, void *arg);
// Schedule execution of a given AWE at the head/tail of the queue.
void THCSchedule(awe_t *awe_ptr);
void THCScheduleBack(awe_t *awe_ptr);
// Finish the current AWE, returning to the scheduler.
//
// NOTE: If we were waiting for this awe to finish in order to exit
// a do-finish block (there is a do-finish awe that is waiting for the
// do-finish's awe count to reach zero), this *will not* schedule that
// awe. I'm not sure why this wasn't done in the first place.
//
// And so, TODO: clean up enclosing do-finish if we are the last awe.
void THCFinish(void);
// Yields and saves awe_ptr to correspond to the provided id number
// Invoke this to signal to all awe's that they should exit. Right now,
// exiting is voluntary: An awe should call THCShouldStop to detect
// if should terminate. Likely, the awe will want to call THCAbort (see
// below).
void THCStopAllAwes(void);
// Returns non-zero if the calling awe should terminate as soon as
// possible. The awe should probably call THCAbort.
int THCShouldStop(void);
// Like THCFinish, but if we are the last awe in our enclosing do-finish,
// and the do-finish is waiting for us, we schedule the do-finish awe.
//
// XXX: This should only be called in exceptional conditions. It doesn't
// try to free stacks like _thc_endasync. (The stacks will be freed when
// the runtime exits. It's possible to check if we are on a "lazy stack"
// and free it when we're in the LAZY case, but for EAGER, there's no
// marker we can walk back to.)
void THCAbort(void);
//Yields and saves AWE for continuation (associates it with an id using
//the awe_mapper).
void THCYieldAndSave(uint32_t id_num);
void THCYieldAndSavePTS(uint32_t id_num);
//Yields and saves AWE for continuation (associates it with an id using
//the awe_mapper),but will not put AWE for the continuation in the dispatch
//queue.
void THCYieldAndSaveNoDispatch(uint32_t id_num);
// Finish the current AWE, creating a new AWE from its continuation, and
// passing this immediately to the scheduler.
void THCYield(void);
void THCYieldPTS(void);
// Hint that the runtime system should switch from the current AWE to the
// indicated awe_ptr. (Currently, the implementation does this immediately
......@@ -300,15 +232,43 @@ void THCYield(void);
// to THCYieldTo on the run-queue.)
void THCYieldTo(awe_t *awe_ptr);
//Same as THCYieldTo except that an id number is provided to locate a particular
//awe that was set to correspond to this id_num.
//id_to is the id that corresponds to the awe that should be yielded to.
//id_from is the id that corresponds to the awe that should save the
//current context.
void THCYieldToIdAndSave(uint32_t id_to, uint32_t id_from);
// Same as THCYieldTo except that an id number is provided to locate a
// particular awe that was set to correspond to this id_num.
//
// id_to is the id that corresponds to the awe that should be yielded to.
//
// id_from is the id that corresponds to the awe that should save the
// current context.
//
// If id_to is invalid, returns -EINVAL, zero otherwise.
int THCYieldToIdAndSave(uint32_t id_to, uint32_t id_from);
//Same as THCYieldToIdAndSave, but assumes AWEs don't use dispatch queue.
int THCYieldToIdAndSaveNoDispatch(uint32_t id_to, uint32_t id_from);
int THCYieldToIdAndSaveNoDispatchDirect(uint32_t id_to, uint32_t id_from);
void inline THCYieldToIdAndSaveNoDispatchDirectTrusted(uint32_t id_to, uint32_t id_from);
void inline THCYieldWithAwe(awe_t *awe_from);
//Yields to without saving curent awe id.
void THCYieldToId(uint32_t id_to);
void inline THCYieldToAwe(awe_t *awe_to, awe_t *awe_from);
int THCYieldToAweNoDispatch_TopLevel(awe_t *awe_to);
// Yields to without saving curent awe id.
//
// If there is no awe associated with id_to, returns non-zero.
int THCYieldToId(uint32_t id_to);
//Same as THCYieldToId, but does not put current AWE in dispatch queue.
//Also assumes the AWE that is being yielded to is not in the dispatch queue.
int THCYieldToIdNoDispatch(uint32_t id_to);
//Same as THCYieldToIdNoDispatch except that it assumes the caller should
//be put in the dispatch queue (but that the AWE being yielded to is not
//in the dispatch queue).
int THCYieldToIdNoDispatch_TopLevel(uint32_t id_to);
// Cancellation actions. These are executed in LIFO order when cancellation
// occurs. Once cancellation has been requested, it is assumed that no
......@@ -341,9 +301,23 @@ void THCDumpStats(int clear_stats);
void THCIncSendCount(void);
void THCIncRecvCount(void);
// Dump all stacks that are in use
//
// XXX NOTE: You will most likely need to modify dump_stack. For example,
// in the LCD microkernel, normally the stack walk stops after we reach
// a certain address boundary. But when you call this function, you need
// the stack walk to continue until you reach a null frame address or
// return address. You basically want your stack walk to look like this:
//
// while (frame_address != NULL && ret_address != NULL) {
// ... walk next frame ...
// }
void THCDumpAllStacks(void);
/*......................................................................*/
#include "thcsync.h"
#include "thcinternal.h"
#endif // _THC_H_
#endif
src/include/thc_ipc.h
View file @ 02123f67
......@@ -3,27 +3,391 @@
#include <libfipc.h>
#include <thc_ipc_types.h>
#include <awe_mapper.h>
#include <asm/atomic.h>
#define THC_MSG_TYPE(msg) ((msg)->regs[(FIPC_NR_REGS) - 2])
#define THC_MSG_ID(msg) ((msg)->regs[(FIPC_NR_REGS) - 1])
/* THC MESSAGES ------------------------------------------------------------ */
int thc_ipc_recv(struct fipc_ring_channel *chnl,
unsigned long msg_id,
struct fipc_message** out_msg);
#define THC_RESERVED_MSG_FLAG_BITS (2 + AWE_TABLE_ORDER)
int thc_poll_recv_group(struct thc_channel_group* chan_group,
struct thc_channel_group_item** chan_group_item,
struct fipc_message** out_msg);
/* Message type is in low 2 bits of flags */
static inline uint32_t thc_get_msg_type(struct fipc_message *msg)
{
/* Caught at compile time */
BUILD_BUG_ON(THC_RESERVED_MSG_FLAG_BITS > 32);
return fipc_get_flags(msg) & 0x3;
}
static inline void thc_set_msg_type(struct fipc_message *msg, uint32_t type)
{
fipc_set_flags(msg,
/* erase old type, and mask off low 2 bits of type */
(fipc_get_flags(msg) & ~0x3) | (type & 0x3));
}
/* Message id is in bits 2..(2 + AWE_TABLE_ORDER) bits */
static inline uint32_t thc_get_msg_id(struct fipc_message *msg)
{
/* shift off type bits, and mask off msg id */
return (fipc_get_flags(msg) >> 2) & ((1 << AWE_TABLE_ORDER) - 1);
}
static inline void thc_set_msg_id(struct fipc_message *msg,
uint32_t msg_id)
{
uint32_t flags = fipc_get_flags(msg);
/* erase old msg id, if any */
flags &= ~(((1 << AWE_TABLE_ORDER) - 1) << 2);
/* mask off relevant bits of msg id (to ensure it is in range),
* and install in flags. */
flags |= (msg_id & ((1 << AWE_TABLE_ORDER) - 1)) << 2;
fipc_set_flags(msg, flags);
}
static inline uint32_t thc_get_request_cookie(struct fipc_message *request)
{
return thc_get_msg_id(request);
}
static inline void thc_kill_request_cookie(uint32_t request_cookie)
{
awe_mapper_remove_id(request_cookie);
}
/* THC CHANNELS ------------------------------------------------------------ */
/*
* thc_channel_init
*
* Initialize a thc channel struct with the given fipc channel.
*
* The channel item's reference count is initialized to 1, and
* state to LIVE.
*
* Returns non-zero on failure.
*/
int thc_channel_init(struct thc_channel *chnl,
struct fipc_ring_channel *async_chnl);
int thc_channel_init_0(struct thc_channel *chnl,
struct fipc_ring_channel *async_chnl);
/*
* thc_channel_inc_ref
*
* Bump the reference count on the channel.
*/
static inline void thc_channel_inc_ref(struct thc_channel *chnl)
{
atomic_inc(&chnl->refcnt);
}
/*
* thc_channel_group_item_dec_ref
*
* Decrement the reference count on the channel.
*
* Returns non-zero if the ref count reached zero.
*/
static inline int thc_channel_dec_ref(struct thc_channel *chnl)
{
return atomic_dec_and_test(&chnl->refcnt);
}
static inline int thc_channel_is_live(struct thc_channel *chnl)
{
return chnl->state == THC_CHANNEL_LIVE;
}
static inline int thc_channel_is_dead(struct thc_channel *chnl)
{
return chnl->state == THC_CHANNEL_DEAD;
}
static inline void thc_channel_mark_dead(struct thc_channel *chnl)
{
chnl->state = THC_CHANNEL_DEAD;
}
static inline struct fipc_ring_channel *
thc_channel_to_fipc(struct thc_channel *chnl)
{
return chnl->fipc_channel;
}
/* IPC FUNCTIONS ----------------------------------------------------------- */
/*
* NOTE: The caller of these functions should ensure they hold a reference
* to the thc_channel (i.e., the caller is the creator of the channel, or
* has explicitly taken a reference via thc_channel_inc_ref). This is
* especially important in thc_ipc_recv_response/thc_ipc_poll_recv, where
* the "state of the world" can change when we yield and then get scheduled
* again later on.
*/
/*
* thc_ipc_send_request
*
* Use case: You are expecting a response after
* this send, and you want the calling awe to handle
* the response. (If the send can be immediately
* followed by the receive, use thc_ipc_call
* instead.)
*
* Associates a "request cookie" (the message id)
* with the request and does the send (this is
* essentially the top half of thc_ipc_call).
*
* The request cookie is returned as an out param.
*
* The caller should invoke a subsequent thc_ipc_recv
* with the request cookie in order to get the response.
*
* (If you don't expect a response that targets the
* calling awe, you should just do a regular fipc
* send instead.)
*
* If you know the other side of the fipc channel
* won't be sending a response and you want to
* delete the request_cookie, you should call
* thc_kill_request_cookie to make it available
* for future use again.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_ipc_send_request(struct thc_channel *chnl,
struct fipc_message *request,
uint32_t *request_cookie);
/*
* thc_ipc_recv_response
*
* Receive the response from a prior thc_ipc_send_request.
*
* thc_ipc_recv_response will peek at the fipc channel to see
* if the response has arrived. It uses the request_cookie to
* identify the response.
*
* There are a few things that can happen:
*
* 1 - No message is in the channel (i.e., the rx buffer).
* thc_ipc_recv_response will yield back to the dispatch
* loop, to be scheduled at a later time. (If there are
* no other awe's, the dispatch loop will just keep
* switching back to thc_ipc_recv_response, and thc_ipc_recv_response
* keep yielding.)
* 2 - A response arrives from the other side, and its
* request cookie matches request_cookie. thc_ipc_recv_response
* will return the response as an out parameter. This
* is the target scenario.
* 3 - A response arrives with a request cookie that does not match
* request_cookie. There are two subcases here:
*
* 3a - The response has a request_cookie that
* belongs to a pending awe. thc_ipc_recv_response
* will switch to that pending awe X. If X
* had not called thc_ipc_recv_response when
* it yielded, or does not end up calling
* thc_ipc_recv_response when it is woken up,
* you could run into trouble.
*
* 3b - The response has a request_cookie that
* *does not* belong to a pending awe.
* thc_ipc_recv_response will print an error notice
* to the console, receive and drop the message,
* and start over.
*
* XXX: This case may pose a security issue if the sender of the
* response passes a request cookie for an awe that didn't
* even send a request ...
*
* 4 - A *request* arrives from the other side, rather than
* a response. (This can happen if you are using the
* same fipc channel for requests and responses, i.e.,
* you receive requests and responses in the same rx
* buffer.) thc_ipc_recv_response will ignore the message, and
* yield as if no message was received. (Some other
* awe will need to pick up the request.)
*
* It is the caller's responsibility to mark the message as having completed
* the receive. (ex: fipc_recv_msg_end(chnl, msg))
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_ipc_recv_response(struct thc_channel *chnl,
uint32_t request_cookie,
struct fipc_message **response);
int thc_ipc_recv_response_ts(struct thc_channel *chnl,
uint32_t request_cookie,
struct fipc_message **response, uint64_t *ts);
int thc_poll_recv(struct thc_channel_group_item* item,
struct fipc_message** out_msg);
int thc_ipc_recv_response_new(struct thc_channel *chnl,
uint32_t request_cookie,
struct fipc_message **response);
int thc_ipc_recv_dispatch(struct thc_channel* channel,
struct fipc_message ** out,
int id,
int (*sender_dispatch)(struct thc_channel*, struct fipc_message *, void *),
void *);
int thc_ipc_recv_req_resp(struct thc_channel* channel,
struct fipc_message ** out, int id,
int (*sender_dispatch)(struct thc_channel*,
struct fipc_message *, void *),
void *ptr);
/* thc_poll_recv
*
* Looks at the fipc channel, and does the following:
*
* -- If a request is waiting in the rx buffer, returns
* it.
* -- If a response is waiting, attempts to dispatch to
* the matching pending awe (like thc_ipc_recv_response does).
* If no awe matches, prints error to console, drops
* message, and returns -ENOMSG.
* -- If no message is waiting in the rx buffer, returns
* -EWOULDBLOCK (rather than yieldling).
*
* There is no request_cookie passed in because thc_ipc_poll_recv should
* not expect a specific message.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_ipc_poll_recv(struct thc_channel *chnl,
struct fipc_message **out_msg);
int thc_ipc_poll_recv_2(struct thc_channel *chnl,
struct fipc_message **out_msg);
/*
* thc_ipc_call
*
* Performs a thc_ipc_send_request followed by an immediate
* thc_ipc_recv_response.
*
* It is the caller's responsibility to mark the message as having completed
* the receive. (ex: fipc_recv_msg_end(chnl, msg))
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_ipc_call(struct thc_channel *chnl,
struct fipc_message *request,
struct fipc_message **response);
/*
* thc_ipc_reply
*
* Send a response with request cookie request_cookie
* on the channel's tx buffer.
*
* This is for the receiver of a request. The receiver does
* some work, and then wants to send a response.
*
* You can get the request_cookie in a request message via
* thc_get_request_cookie.
*
* Sets the request_cookie of response and marks the response
* message as having a response message type. Sends response
* message on the provided channel.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_ipc_reply(struct thc_channel *chnl,
uint32_t request_cookie,
struct fipc_message *response);
/* CHANNEL GROUPS ---------------------------------------------------------- */
/*
* thc_channel_group_init
*
* Initializes thc_channel_group structure after it has been allocated.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_channel_group_init(struct thc_channel_group* channel_group);
/*
* thc_channel_group_item_init
*
* Initializes a channel group item with the provided ring channel and
* dispatch function.
*
* The channel item's reference count is initialized to 1.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_channel_group_item_init(struct thc_channel_group_item *item,
struct thc_channel *chnl,
int (*dispatch_fn)(struct thc_channel_group_item*,
struct fipc_message*));
/*
* thc_channel_group_item_add
*
* Adds a thc_channel_group_item to a thc_channel_group.
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_channel_group_item_add(struct thc_channel_group* channel_group,
struct thc_channel_group_item* item);
/*
* thc_channel_group_item_remove
*
* Removes a thc_channel_group_item from a thc_channel_group.
*/
void thc_channel_group_item_remove(struct thc_channel_group* channel_group,
struct thc_channel_group_item* item);
/*
* thc_channel_group_item_get
*
* Gets a thc_channel_group_item at a particular index in a channel_group.
* Puts the thc_channel_group_item in the out parameter (*out_item) on success.
*
* Returns 0 on success. Returns 1 if index is out of range.
*/
int thc_channel_group_item_get(struct thc_channel_group* channel_group,
int index,
struct thc_channel_group_item **out_item);
static inline struct thc_channel *
thc_channel_group_item_channel(struct thc_channel_group_item *item)
{
return item->channel;
}
static inline struct fipc_ring_channel *
thc_channel_group_item_to_fipc(struct thc_channel_group_item *item)
{
return thc_channel_group_item_channel(item)->fipc_channel;
}
/*
* thc_poll_recv_group
*
* Calls thc_poll_recv on a list of thc_channel_group_items represented
* by thc_channel_group. If there is a message available that does not
* correspond to a pending AWE, the thc_channel_group_item that corresponds
* to the received message is set to the out param (*chan_group_item),
* and the received message is set to the out param (*out_msg).
* If there is no message available in any of the thc_channel_group_items,
* then this function returns -EWOULDBLOCK.
*
* NOTE: The caller should "own" a reference to each of the thc channels
* in the group (i.e., should ensure the reference count for every
* channel in the group cannot suddenly drop to zero while this function
* is in progress).
*
* Returns 0 on success, non-zero otherwise.
*/
int thc_poll_recv_group(struct thc_channel_group* chan_group,
struct thc_channel_group_item** chan_group_item,
struct fipc_message** out_msg);
int thc_poll_recv_group_2(struct thc_channel_group* chan_group,
struct thc_channel_group_item** chan_group_item,
struct fipc_message** out_msg);
#endif
src/include/thc_ipc_types.h
View file @ 02123f67
......@@ -14,18 +14,52 @@ typedef enum {
struct predicate_payload
{
unsigned long expected_msg_id;
unsigned long actual_msg_id;
uint32_t expected_msg_id;
uint32_t actual_msg_id;
msg_type_t msg_type;
};
enum {
THC_CHANNEL_LIVE,
THC_CHANNEL_DEAD,
};
/*
* struct thc_channel
*
* Wraps around a libfipc channel. One of the motivations is we need to
* do reference counting due to interesting computation patterns with
* awe's that share async channels.
*/
struct thc_channel
{
int state;
atomic_t refcnt;
struct fipc_ring_channel *fipc_channel;
bool one_slot;
};
/*
* struct thc_channel_group_item
*
* Contains a channel and a function that should get called when a message
* is received on the channel.
*/
struct thc_channel_group_item
{
struct list_head list;
struct fipc_ring_channel * channel;
int (*dispatch_fn)(struct fipc_ring_channel*, struct fipc_message*);
bool xmit_channel;
struct thc_channel *channel;
int (*dispatch_fn)(struct thc_channel_group_item*, struct fipc_message*);
};
/*
* struct thc_channel_group
*
* Represents a linked list of thc_channel_group_items.
*/
struct thc_channel_group
{
struct list_head head;
......
src/include/thcinternal.h
View file @ 02123f67
This diff is collapsed.
src/tests/Kbuild.in
View file @ 02123f67
obj-m += simple/
obj-m += dispatch/
obj-m += ctx-switch/
obj-m += async-msg-benchmarks/
obj-m += abort/
src/tests/Makefile.am
View file @ 02123f67
......@@ -3,3 +3,6 @@
install-tests:
cp $(LIBASYNC_TESTS_KBUILD)/simple/libasync_test_simple.ko $(D)
cp $(LIBASYNC_TESTS_KBUILD)/dispatch/libfipc_test_dispatch_loop.ko $(D)
cp $(LIBASYNC_TESTS_KBUILD)/ctx-switch/ctx_switch_test.ko $(D)
cp $(LIBASYNC_TESTS_KBUILD)/async-msg-benchmarks/libasync_test_msgs.ko $(D)
cp $(LIBASYNC_TESTS_KBUILD)/abort/libasync_test_abort.ko $(D)
src/tests/abort/Kbuild.in 0 → 100644
View file @ 02123f67
# @ASYNC_AUTOCONF_NOTICE@
# Magic line so we can do out-of-source build
src = @abs_top_srcdir@/src/tests/abort
obj-m = libasync_test_abort.o
# Paths are relative to abort/ build dir
libasync_test_abort-y += main.o
# LIBASYNC_PATH and LIBFIPC_LIB are defined (and exported) in
# src/Makefile.am.
libasync_test_abort-y += ../../$(LIBASYNC_PATH)
libasync_test_abort-y += ../../$(LIBFIPC_LIB)
ccflags-y += $(CFLAGS) $(AM_CPPFLAGS) $(AM_CFLAGS)
src/tests/abort/main.c 0 → 100644
View file @ 02123f67
/*
* main.c
*/
#include <libfipc.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <thc.h>
#include "../test_helpers.h"
MODULE_LICENSE("GPL");
static void abort_example_init(void)
{
DO_FINISH(
ASYNC(
int i;
printk(KERN_ERR "ASYNC 0 started\n");
for (i = 0; i < 4; i++) {
THCYield();
printk(KERN_ERR "ASYNC 0 scheduled\n");
}
printk(KERN_ERR "ASYNC 0 signaling all awe's\n");
// THCStopAllAwes();
printk(KERN_ERR "ASYNC 0 aborting\n");
THCAbort();
);
ASYNC(
printk(KERN_ERR "ASYNC 1 started\n");
while(!THCShouldStop()) {
THCYield();
printk(KERN_ERR "ASYNC 1 Scheduled\n");
}
printk(KERN_ERR "ASYNC 1 aborting\n");
THCAbort();
<