Commit 424ac36d authored by Charlie Jacobsen's avatar Charlie Jacobsen Committed by Vikram Narayanan

Adds full functioning example with glue code.

Example is in virt/lcd-domains/test-mods/glue-example. It consists
of a fake minix and fake vfs. The original "unmodified" code is
in minix/original/main.c, vfs/original/main.c, and include/vfs.h.
The glue code (written by hand) is in minix/glue and vfs/glue. The
IDL is in idl/.

You can build the fake minix and vfs for isolation or as regular
modules. To build for isolation, run `make menuconfig' and under
Test Modules --> Example Exercising IDL and Glue.

To run minix and vfs in isolation, install the "boot" module,
lcd-test-mod-glue-example-boot.ko. (Its code is under isol-boot/main.c.)

The example will print status messages to the kernel logs so you
can see a trace of the interaction.

A few hacks were necessary to fully exercise all of the code. For example,
the fake vfs is the one that invokes new_file and rm_file, in the middle
of its dispatch loop.

The code for the trampolines in vfs/glue/vfs_caller.{c,lds} is probably
the ugliest part. That required a little bit of trickery and low-level
hacking.
parent bc7624d5
......@@ -414,7 +414,6 @@ static inline void __lcd_free_cptr(struct cptr_cache *cache, cptr_t c)
*/
struct dstore;
struct dstore_node;
typedef struct { unsigned long dptr; } dptr_t;
/**
* These tags are reserved.
......
......@@ -45,13 +45,13 @@
#undef panic
#define panic(fmt, args...) do { \
lcd_printk(fmt, args); \
lcd_printk(fmt, ##args); \
lcd_exit(-1); \
} while(0)
#undef printk
#define printk(fmt, args...) do { \
lcd_printk(fmt, args); \
lcd_printk(fmt, ##args); \
} while(0)
#undef printk_ratelimit
......
......@@ -63,6 +63,14 @@ LCD_MK_REG_ACCESS(5)
LCD_MK_REG_ACCESS(6)
LCD_MK_REG_ACCESS(7)
static inline u64 lcd_debug_reg(struct lcd_utcb *utcb) \
{ \
return __lcd_debug_reg(lcd_get_utcb()); \
} \
static inline void lcd_set_debug_reg(u64 val) \
{ \
__lcd_set_debug_reg(lcd_get_utcb(), val); \
} \
/* LOW LEVEL SYSCALLS -------------------------------------------------- */
......@@ -96,6 +104,7 @@ LCD_MK_SYSCALL(LCD_SYSCALL_PAGE_ALLOC)
LCD_MK_SYSCALL(LCD_SYSCALL_PAGE_MAP)
LCD_MK_SYSCALL(LCD_SYSCALL_PAGE_UNMAP)
LCD_MK_SYSCALL(LCD_SYSCALL_CAP_DELETE)
LCD_MK_SYSCALL(LCD_SYSCALL_SYNC_EP)
#define __LCD_DO_SYSCALL(num) lcd_syscall_##num()
#define LCD_DO_SYSCALL(name) __LCD_DO_SYSCALL(name)
......@@ -293,7 +302,6 @@ static inline struct lcd_boot_info * lcd_get_boot_info(void)
*/
struct dstore;
struct dstore_node;
typedef struct { unsigned long dptr; } dptr_t;
/**
* These tags are reserved.
......@@ -332,7 +340,8 @@ int lcd_dstore_insert(struct dstore *dstore, void *object, int tag,
*/
void lcd_dstore_delete(struct dstore *dstore, dptr_t d);
/**
* Tears down data store.
* Tears down data store and frees it (so the pointer arg is invalid
* after this call).
*/
void lcd_dstore_destroy(struct dstore *dstore);
/**
......
......@@ -18,6 +18,7 @@
#define LCD_SYSCALL_PAGE_MAP 7
#define LCD_SYSCALL_PAGE_UNMAP 8
#define LCD_SYSCALL_CAP_DELETE 9
#define LCD_SYSCALL_SYNC_EP 10
#endif /* LCD_DOMAINS_SYSCALL_H */
......@@ -104,6 +104,26 @@ struct cptr_cache {
unsigned long *bmaps[1 << LCD_CPTR_DEPTH_BITS];
};
/* DPTRS -------------------------------------------------- */
typedef struct { unsigned long dptr; } dptr_t;
static inline dptr_t __dptr(unsigned long dptr)
{
return (dptr_t){ dptr };
}
static inline unsigned long dptr_val(dptr_t d)
{
return d.dptr;
}
#define LCD_DPTR_NULL (__dptr(0))
static inline int dptr_is_null(dptr_t d)
{
return dptr_val(d) == dptr_val(LCD_DPTR_NULL);
}
/* ADDRESS SPACE TYPES ---------------------------------------- */
/* XXX: Assumes host and guest run in 64-bit mode */
......
......@@ -20,6 +20,7 @@
* -- mr are general registers and are just copied during a send/recv
* -- cr are capability pointer registers, and the corresponding
* capabilities are granted during a send/recv
* -- one debug register that won't squash the others
*
* NOTE: This needs to fit inside a page (4 KBs). It goes at the bottom
* of the lcd's stack (so it should be reasonably small).
......@@ -30,6 +31,7 @@
struct lcd_utcb {
u64 mr[LCD_NUM_REGS];
cptr_t cr[LCD_NUM_REGS];
u64 debug_reg;
};
#define UTCB_MK_REG_ACCESS(idx) \
......@@ -59,4 +61,14 @@ UTCB_MK_REG_ACCESS(5)
UTCB_MK_REG_ACCESS(6)
UTCB_MK_REG_ACCESS(7)
static inline u64 __lcd_debug_reg(struct lcd_utcb *utcb) \
{ \
return utcb->debug_reg; \
} \
static inline void __lcd_set_debug_reg(struct lcd_utcb *utcb, u64 val) \
{ \
utcb->debug_reg = val; \
} \
#endif /* LCD_DOMAINS_UTCB_H */
......@@ -33,22 +33,6 @@
/* DPTR DEFS -------------------------------------------------- */
static inline dptr_t __dptr(unsigned long dptr)
{
return (dptr_t){ dptr };
}
static inline unsigned long dptr_val(dptr_t d)
{
return d.dptr;
}
#define LCD_DPTR_NULL (__dptr(0))
static inline int dptr_is_null(dptr_t d)
{
return dptr_val(d) == dptr_val(LCD_DPTR_NULL);
}
static inline unsigned long lcd_dptr_slot(dptr_t d)
{
/*
......@@ -684,6 +668,13 @@ int lcd_dstore_insert(struct dstore *dstore, void *object, int tag,
struct dstore_node *dstore_node;
int ret;
dptr_t d;
/* Ensure tag is not one of the reserved tags. */
if (tag == LCD_DSTORE_TAG_NULL ||
tag == LCD_DSTORE_TAG_DSTORE_NODE) {
return -EINVAL;
}
/*
* Alloc a dptr
*/
......@@ -692,6 +683,7 @@ int lcd_dstore_insert(struct dstore *dstore, void *object, int tag,
LIBLCD_ERR("error alloc dptr");
goto fail1;
}
/*
* Get cnode
*/
......
......@@ -6,7 +6,31 @@
int lcd_create_sync_endpoint(cptr_t *slot_out)
{
return -ENOSYS;
int ret;
/*
* Alloc cptr
*/
ret = lcd_alloc_cptr(slot_out);
if (ret) {
LIBLCD_ERR("cptr alloc");
goto fail1;
}
/*
* Get new endpoint
*/
lcd_set_cr0(*slot_out);
ret = LCD_DO_SYSCALL(LCD_SYSCALL_SYNC_EP);
if (ret) {
LIBLCD_ERR("create sync ep syscall");
goto fail2;
}
return 0;
fail2:
lcd_free_cptr(*slot_out);
fail1:
return ret;
}
int lcd_send(cptr_t endpoint)
......
......@@ -7,7 +7,7 @@
int lcd_put_char(char c)
{
lcd_set_r0(c);
lcd_set_debug_reg(c);
return LCD_DO_SYSCALL(LCD_SYSCALL_PUTCHAR);
}
......
......@@ -161,7 +161,45 @@ config LCD_TEST_MOD_PMFS_BOOT
config LCD_TEST_MOD_PMFS_LCD
tristate
depends on LCD_TEST_MOD_PMFS
menuconfig LCD_TEST_MOD_GLUE_EXAMPLE
bool "Example exercising IDL and glue"
depends on LCD_DOMAINS
default y
select LCD_TEST_MOD_GLUE_EXAMPLE_VFS
select LCD_TEST_MOD_GLUE_EXAMPLE_MINIX
select LCD_TEST_MOD_GLUE_EXAMPLE_BOOT
---help---
Shows interoperation of IDL, glue code, LCD runtime, and more.
if LCD_TEST_MOD_GLUE_EXAMPLE
config ISOLATE_GLUE_EXAMPLE
bool "Run fake minix and vfs in isolation"
default y
---help---
When set, the fake minix and vfs will be built so they can run in
isolation. This will also build and link the glue code. When not set,
the fake vfs and minix are built as regular kernel modules, and none
of the glue code is linked with them.
config LCD_TEST_MOD_GLUE_EXAMPLE_VFS
tristate
depends on LCD_TEST_MOD_GLUE_EXAMPLE
default m
config LCD_TEST_MOD_GLUE_EXAMPLE_MINIX
tristate
depends on LCD_TEST_MOD_GLUE_EXAMPLE
default m
config LCD_TEST_MOD_GLUE_EXAMPLE_BOOT
tristate
depends on LCD_TEST_MOD_GLUE_EXAMPLE && ISOLATE_GLUE_EXAMPLE
default m
endif
# --------------------------------------------------
endif
......@@ -16,7 +16,7 @@
* --------------------------------------------------
*/
#define LCD_DEBUG_LVL 0
#define LCD_DEBUG_LVL 1
#define LCD_ERR(msg...) __lcd_err(__FILE__, __LINE__, msg)
static inline void __lcd_err(char *file, int lineno, char *fmt, ...)
......@@ -49,6 +49,7 @@ static inline void __lcd_warn(char *file, int lineno, char *fmt, ...)
#define LCD_DEBUG_ERR 1
#define LCD_DEBUG_WARN 2
#define LCD_DEBUG_MSG 3
#define LCD_DEBUG_VERB 4
#define LCD_DEBUG(lvl, msg...) { \
if (lvl <= LCD_DEBUG_LVL) \
......
......@@ -29,22 +29,6 @@
/* DPTR DEFS -------------------------------------------------- */
static inline dptr_t __dptr(unsigned long dptr)
{
return (dptr_t){ dptr };
}
static inline unsigned long dptr_val(dptr_t d)
{
return d.dptr;
}
#define LCD_DPTR_NULL (__dptr(0))
static inline int dptr_is_null(dptr_t d)
{
return dptr_val(d) == dptr_val(LCD_DPTR_NULL);
}
static inline unsigned long lcd_dptr_slot(dptr_t d)
{
/*
......
......@@ -237,6 +237,10 @@ static int find_cnode(struct cspace *cspace, struct cnode_table *old,
return 1; /* signal we found the slot and are done */
} else {
LCD_DEBUG(LCD_DEBUG_ERR, "Error looking up cnode: ");
LCD_DEBUG(LCD_DEBUG_ERR, " cnode type is %d, alloc is %d",
old->cnode[level_id].type, alloc);
/*
* invalid indexing, etc.
*/
......@@ -382,6 +386,7 @@ int __lcd_cap_insert(struct cspace *cspace, cptr_t c, void *object,
{
struct cnode *cnode;
int ret;
/*
* Get cnode
*/
......
......@@ -18,6 +18,8 @@
static int lookup_ep(struct cspace *cspace, cptr_t slot, struct cnode **cnode)
{
int ret;
/*
* Look up
*/
......@@ -29,9 +31,9 @@ static int lookup_ep(struct cspace *cspace, cptr_t slot, struct cnode **cnode)
*/
if ((*cnode)->type != LCD_CAP_TYPE_SYNC_EP) {
LCD_ERR("not a sync ep");
ret = -EINVAL;
goto fail2;
}
return 0;
fail2:
......@@ -153,8 +155,9 @@ static void copy_msg_caps(struct lcd *sender, struct lcd *receiver)
*/
from = sender->utcb->cr[i];
to = receiver->utcb->cr[i];
if (!cptr_is_null(from) && !cptr_is_null(to))
if (!cptr_is_null(from) && !cptr_is_null(to)) {
copy_msg_cap(sender, receiver, from, to);
}
}
}
......@@ -235,6 +238,7 @@ static int wait_for_transmit(struct lcd *lcd, struct lcd_sync_endpoint *ep)
ret = 0;
goto out;
case LCD_XMIT_FAILED:
LCD_DEBUG(LCD_DEBUG_ERR, "xmit failed");
ret = -EIO;
goto out;
default:
......@@ -251,6 +255,8 @@ static int wait_for_transmit(struct lcd *lcd, struct lcd_sync_endpoint *ep)
* testing it ...
*/
if (lcd_status_dead(lcd)) {
LCD_DEBUG(LCD_DEBUG_ERR,
"lcd died inside xmit loop");
ret = -EIO;
goto out;
}
......@@ -268,6 +274,7 @@ static int wait_for_transmit(struct lcd *lcd, struct lcd_sync_endpoint *ep)
/*
* We were interrupted
*/
LCD_DEBUG(LCD_DEBUG_ERR, "lcd xmit loop interrupted");
ret = -EINTR;
break;
}
......@@ -280,12 +287,36 @@ out:
return ret;
}
static void debug_dump_utcb(struct lcd *lcd)
{
int i;
printk(KERN_ERR "LCD %p UTCB DUMP\n", lcd);
printk(KERN_ERR "---------------------------------\n");
/* General regs */
for (i = 0; i < LCD_NUM_REGS; i++) {
printk(" mr[%d] = %llx\n", i,
lcd->utcb->mr[i]);
}
/* Cptrs */
for (i = 0; i < LCD_NUM_REGS; i++) {
printk(" cr[%d] = %lx\n", i,
cptr_val(lcd->utcb->cr[i]));
}
}
static int do_send(struct lcd *sender, struct cnode *cnode,
struct lcd_sync_endpoint *ep, int making_call,
int doing_reply)
{
int ret;
struct lcd *receiver;
if (LCD_DEBUG_LVL >= 3)
debug_dump_utcb(sender);
if (list_empty(&ep->receivers)) {
/*
* No one receiving; put myself in ep's sender list
......@@ -408,6 +439,10 @@ static int do_recv(struct lcd *receiver, struct cnode *cnode,
{
int ret;
struct lcd *sender;
if (LCD_DEBUG_LVL >= 3)
debug_dump_utcb(receiver);
if (list_empty(&ep->senders)) {
/*
* No one sending; put myself in ep's receiver list
......
......@@ -1017,7 +1017,7 @@ static int __lcd_put_char(struct lcd *lcd)
/*
* Char is in r0
*/
c = __lcd_r0(lcd->utcb);
c = __lcd_debug_reg(lcd->utcb);
/*
* Put char in lcd's console buff
*/
......@@ -1204,6 +1204,14 @@ static int do_cap_delete(struct lcd *lcd)
return 0;
}
static int do_create_sync_endpoint(struct lcd *lcd)
{
cptr_t slot;
slot = __lcd_cr0(lcd->utcb);
return __lcd_create_sync_endpoint(lcd, slot);
}
static int lcd_handle_syscall(struct lcd *lcd, int *lcd_ret)
{
int syscall_id;
......@@ -1212,7 +1220,8 @@ static int lcd_handle_syscall(struct lcd *lcd, int *lcd_ret)
syscall_id = lcd_arch_get_syscall_num(lcd->lcd_arch);
LCD_DEBUG(LCD_DEBUG_MSG, "got syscall %d from lcd %p", syscall_id, lcd);
LCD_DEBUG(LCD_DEBUG_VERB,
"got syscall %d from lcd %p", syscall_id, lcd);
switch (syscall_id) {
case LCD_SYSCALL_EXIT:
......@@ -1312,6 +1321,13 @@ static int lcd_handle_syscall(struct lcd *lcd, int *lcd_ret)
ret = do_cap_delete(lcd);
lcd_arch_set_syscall_ret(lcd->lcd_arch, ret);
break;
case LCD_SYSCALL_SYNC_EP:
/*
* Create synchronous endpoint
*/
ret = do_create_sync_endpoint(lcd);
lcd_arch_set_syscall_ret(lcd->lcd_arch, ret);
break;
default:
LCD_ERR("unimplemented syscall %d", syscall_id);
ret = -ENOSYS;
......@@ -1325,13 +1341,13 @@ static int lcd_kthread_run_once(struct lcd *lcd, int *lcd_ret)
ret = lcd_arch_run(lcd->lcd_arch);
if (ret < 0) {
LCD_ERR("running lcd");
LCD_ERR("running lcd %p", lcd);
goto out;
}
switch(ret) {
case LCD_ARCH_STATUS_PAGE_FAULT:
LCD_ERR("page fault");
LCD_ERR("page fault for lcd %p", lcd);
ret = -ENOSYS;
goto out;
break;
......
......@@ -15,3 +15,5 @@ obj-$(CONFIG_LCD_TEST_MOD_LIBLCD_TESTS) += liblcd-tests/
obj-$(CONFIG_LCD_TEST_MOD_CREATE_KLCD) += create-klcd/
obj-$(CONFIG_LCD_TEST_MOD_PMFS) += pmfs/
obj-$(CONFIG_LCD_TEST_MOD_GLUE_EXAMPLE) += glue-example/
obj-$(CONFIG_LCD_TEST_MOD_GLUE_EXAMPLE_VFS) += vfs/
obj-$(CONFIG_LCD_TEST_MOD_GLUE_EXAMPLE_MINIX) += minix/
obj-$(CONFIG_LCD_TEST_MOD_GLUE_EXAMPLE_BOOT) += isol-boot/
include "vfs.idl"
module minix {
requires interface vfs(vfs_chan);
}
/* CHARLIE: Where does vfs_chan come from? (It sounds like
* you're saying the IDL compiler just auto generates a
* channel for the interface. But how does that work if
* multiple domains use this interface? What are the
* implications of that for the glue code generated for
* each domain? Does it matter? */
/* TODO: Discuss errors.
*
* 1 - Projection type declared as taking arguments in more than one
* instance. Is that a problem or an error?
* 2 - Name collisions
*/
/* Why do we need this include? */
/* CHARLIE: Because the compiler needs to know where the original defs
* are. It needs the original defs in the glue code (to call the original
* functions, e.g. on the callee side). The other option is to have the
* idl compiler just generate declarations in the glue code so that
* including the original header is unnecessary. */
include ../original/include/vfs.h;
/* - I suggest to use "<" and ">" to improve readability,
- "struct fs" which is used as part of the projection definition
is a verbatim C type, which is required so IDL can use it to
generate typecast operations
CHARLIE: I agree
- Projections and interfaces can take channels as constructor argurments.
A channel then is used by each function inside projection.
CHARLIE: Our solution for passing channels into projections looks
complicated. This is why I advocated moving a projection that contains
function pointers out into a separate interface of some kind. In the
particular example below, I wasn't planning to associate an ipc channel
with a struct fs, so it seems weird to pull that out of its container
and pass that to the constructor for the fs_operations. It also seems
limiting (but maybe this is all that's needed anyway).
- rpc [alloc] -- means that caller and callee stubs have to be
allocated dynamically
CHARLIE: What about function pointers that are passed directly into
functions? (I'm encountering this right now in the PMFS glue.)
- I don't think we need "bind" --- all four operations on projections
(in, out, alloc, and dealloc), imply bind, no? Ok we might need
bind when we run into exotic case later, but for now I removed it.
CHARLIE: Yeah that makes sense for now.
- I decided to have more explicit syntax for alloc and dealloc,
specifically require to have explicit keywords (caller, callee)
to specify more explicitly on which end the data structure needs to
be allocated and deallocated (mostly to simplify the reasoning
about code), e.g.:
[out, alloc(caller)] - means allocate on the caller side, and is
incorrect without the "out" specifier
[alloc(caller, calle)] -- allocate on both sides, for now this is
only for the channels, since all other datastructures are allocated at
least on one end
CHARLIE: Good idea
*/
/* - The interface needs a channel (or a capability to a channel) as a
construction argument to identify which communication channel
should be used by this interface.
- module declares a static inteface, vfs_chnl is generated as a
global vairable inside the module
*/
module vfs (channel vfs_chnl) {
projection <struct file_system> fs;
projection <struct fs_ops> fs_ops (channel chnl) {
/* I suggest we generate a capability field for binding remote
copies automatically. The only case when we need to refer to
the remote references explicitly is when we project the same
data structure to two different domains, e.g., a file into VFS
and block, but I'm not sure it's needed.
The assumption is: each data structure has a shadow on the
other end.
*/
// capability [bind] container->self;
/* CHARLIE: Could the compiler just always generate two
* capability/reference fields? One for my ref, one for the
* other domain's ref? And again as you say, this won't work
* as nicely when we have multiple domains.
*/
{
/* - [out, alloc(caller)] means the callee allocates the struct in the
original code; caller allocates their own private copy
- [in, alloc(callee)] means that the data structure is already allocated
on the caller's side, but callee's glue code will allocate
a shadow copy
*/
projection <struct file> file {
/* this filed is automatically generated */
/* CHARLIE: So you mean the IDL writer doesn't even put it here? */
//capability [bind] container->self;
int [out] id;
}
/* - I just want to show an alternative syntax for a function with
implicit argument (chnl)
- We just bind to the fs projection (no fields marshalled), fs is defined
as a dummy projection above
*/
rpc [alloc] int (*new_file) (channel chnl = chnl,
projection fs *fs, // same as below
projection file [alloc(caller)] **file_out); // is this supposed to be a file projection? type was partly missing before
/* CHARLIE: Yes it is */
}
{
/* - [in, dealloc] will instruct _only_ the caller's glue
code to look up and free its private copy, the callee's
copy will be deallocated as part of the rm_file() invocation
on the callee's side
- the chnl is implicit here, it's taken from the projection
*/
projection <struct file> file; // still same problem. Do not know what fields because the other projection file definition is in different scope.
/* CHARLIE: Same problem means what? */
rpc [alloc] void (*rm_file) (projection fs *fs, // same as below
projection file [dealloc(caller)] *file); // need to name the projection type.
}
}
{
/* REGISTER_FS */
projection <struct fs> fs {
int [in] id;
int [in, out] size;
/* - Instruct IDL that a new channel should be constructed
- What if we decide to register two fs implementations? Can we
say that channel exists? For example, what if inside fs_ops
one of the functions registers more fs_ops?
*/
channel [alloc (caller, callee)] container->chnl;
/* CHARLIE: Does this mean the struct fs container should have
* a channel attached or associated with it? What does this mean
* in terms of the glue? Is this a completely new channel? */
/* Pass the channel as a constructor argument into fs_ops */
projection fs_ops [alloc(callee)] *fs_ops (container->chnl);
}
rpc int register_fs(projection fs [alloc(callee)] *fs);
}
{
/* UNREGISTER_FS */
projection <struct fs> fs {
/* [dealloc (caller, callee)] -- means deallocate objects on both ends, this
is in contrast to [dealloc (caller)] which means deallocate the shadow
copy on the caller side only */
channel [dealloc(caller, callee)] container->chnl;
projection fs_ops [dealloc(callee)] *fs_ops;
}
rpc int unregister_fs(projection fs [dealloc(callee)] *fs);
}
}
/**
* vfs.h - VFS interface.
*/
#ifndef EXAMPLE_VFS_H
#define EXAMPLE_VFS_H
/* DATA ---------------------------------------- */
struct file {
int id;
int private;
};
struct fs {
int id;
int size;
struct fs_operations *fs_operations;
};
struct fs_operations {
int (*new_file)(int id, struct file **file_out);
void (*rm_file)(struct file *file);