Commit a3d2cf4c authored by Leigh B Stoller's avatar Leigh B Stoller

Merge branch 'master' of

parents e2a8dbf5 4ef9397e
......@@ -139,8 +139,9 @@ my %xmlfields =
"description" => ["description", $SLOT_OPTIONAL],
"path" => ["path", $SLOT_OPTIONAL],
"mtype_*" => ["mtype", $SLOT_OPTIONAL],
"load_address" => ["load_address", $SLOT_ADMINONLY],
"frisbee_pid" => ["frisbee_pid", $SLOT_ADMINONLY]);
#"load_address" => ["load_address", $SLOT_ADMINONLY],
#"frisbee_pid" => ["frisbee_pid", $SLOT_ADMINONLY]
# Need a list of node types. We join this over the nodes table so that
# we get a list of just the nodes that are currently in the testbed, not
......@@ -376,28 +377,28 @@ UserError("Node Types: Must select at least one node type")
# Only admins can edit the load_address or the frisbee pid.
if ($isadmin) {
if (exists($editimageid_args{"load_address"}) &&
$editimageid_args{"load_address"} ne "") {
$foo = escapeshellarg($editimageid_args{"load_address"});
if ($editimageid_args{"load_address"} ne $foo) {
UserError("Load Address: Contains illegal characters!");
# Frisbee_pid has already been checked to be a valid int coming from XML.
else { # Not isadmin.
if (exists($editimageid_args{"load_address"}) &&
$editimageid_args{"load_address"} ne $image->load_address()) {
UserError("Load Address: No permission, admin-only");
if (exists($editimageid_args{"frisbee_pid"}) &&
$editimageid_args{"frisbee_pid"} ne $image->frisbee_pid()) {
UserError("Frisbee PID: No permission, admin-only");
# XXX: Bring back functionally is some way
#if ($isadmin) {
# if (exists($editimageid_args{"load_address"}) &&
# $editimageid_args{"load_address"} ne "") {
# $foo = escapeshellarg($editimageid_args{"load_address"});
# if ($editimageid_args{"load_address"} ne $foo) {
# UserError("Load Address: Contains illegal characters!");
# }
# }
# # Frisbee_pid has already been checked to be a valid int coming from XML.
#else { # Not isadmin.
# if (exists($editimageid_args{"load_address"}) &&
# $editimageid_args{"load_address"} ne $image->load_address()) {
# UserError("Load Address: No permission, admin-only");
# }
# if (exists($editimageid_args{"frisbee_pid"}) &&
# $editimageid_args{"frisbee_pid"} ne $image->frisbee_pid()) {
# UserError("Frisbee PID: No permission, admin-only");
# }
# Mereusers are not allowed to create more than one osid/imageid mapping
......@@ -94,12 +94,15 @@ use vars qw(@ISA @EXPORT);
......@@ -128,6 +131,7 @@ use vars qw(@ISA @EXPORT);
......@@ -399,6 +403,9 @@ sub TBDB_NODESTATE_PXEWAKEUP() { "PXEWAKEUP"; }
sub TBDB_NODEOPMODE_ANY { "*"; } # A wildcard opmode
......@@ -410,6 +417,8 @@ sub TBDB_NODEOPMODE_MINIMAL { "MINIMAL"; }
sub TBDB_NODEOPMODE_BOOTWHAT { "_BOOTWHAT_"; } # A redirection opmode
......@@ -495,6 +504,9 @@ sub TBDB_STATS_FLAGS_MODSWAPOUT() { 0x40; }
# Frisbee.
# Reserved node "roles"
sub TBDB_RSRVROLE_NODE() { "node"; }
sub TBDB_RSRVROLE_VIRTHOST() { "virthost"; }
......@@ -127,9 +127,9 @@ sub part4_osid($) { return field($_[0], "part4_osid"); }
sub default_osid($) { return field($_[0], "default_osid"); }
sub path($) { return field($_[0], "path"); }
sub magic($) { return field($_[0], "magic"); }
sub load_address($) { return field($_[0], "load_address"); }
sub frisbee_pid($) { return field($_[0], "frisbee_pid"); }
sub load_busy($) { return field($_[0], "load_busy"); }
#sub load_address($) { return field($_[0], "load_address"); }
#sub frisbee_pid($) { return field($_[0], "frisbee_pid"); }
#sub load_busy($) { return field($_[0], "load_busy"); }
sub ezid($) { return field($_[0], "ezid"); }
sub shared($) { return field($_[0], "shared"); }
sub global($) { return field($_[0], "global"); }
......@@ -146,7 +146,7 @@ sub ActiveImages($)
my @result = ();
my $query_result =
DBQueryWarn("select imageid from images where frisbee_pid!=0");
DBQueryWarn("select imageid from frisbee_blobs where frisbee_pid!=0");
return undef
if (!defined($query_result));
......@@ -370,7 +370,8 @@ sub EditImageid($$$$)
# (Others above already did their own updates.)
my %updates;
foreach my $col ("description", "path", "load_address", "frisbee_pid" ) {
#foreach my $col ("description", "path", "load_address", "frisbee_pid" ) {
foreach my $col ("description", "path") {
# Copy args we want so that others can't get through.
if (exists($argref->{$col})) {
$updates{$col} = $mods{$col} = $argref->{$col};
......@@ -659,25 +660,6 @@ sub UnLockTables($)
return 0;
# Bump the busy indicator to keep the frisbeed going.
sub KeepBusy($)
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $imageid = $self->imageid();
DBQueryFatal("update images set load_busy=GREATEST(load_busy,1) " .
"where imageid='$imageid'");
return 0;
# Mark the update time in the record,
......@@ -72,6 +72,7 @@ my $debug = 0;
"virt_tiptunnels" => [ "host", "vnode" ],
"virt_parameters" => [ "name", "value" ],
"virt_paths" => [ "pathname", "segmentname"],
"experiment_blobs" => [ "path", "action" ],
......@@ -1193,5 +1194,10 @@ use vars qw(@ISA);
@ISA = "VirtExperiment::VirtTableRow";
use VirtExperiment;
package VirtExperiment::VirtTableRow::experiment_blobs;
use vars qw(@ISA);
@ISA = "VirtExperiment::VirtTableRow";
use VirtExperiment;
# _Always_ make sure that this 1 is at the end of the file...
......@@ -53,10 +53,10 @@ open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");
# Clear the load_address field of the images table since any frisbee's
# Clear the frisbee_blobs table since any frisbee's
# that were running are obviously not running anymore!
DBQueryFatal("update images set load_address='',frisbee_pid=0,load_busy=0");
DBQueryFatel("delete from frisbee_blobs");
# Clear the event scheduler pids. Its okay to set them to zero since
......@@ -1408,8 +1408,10 @@ sub TBNodeDiskloadOSID($)
my $mfs;
return 0
if ($node->NodeTypeAttribute("diskloadmfs_osid", \$mfs) != 0);
if ($node->NodeAttribute("diskloadmfs_osid", \$mfs) || !defined($mfs)) {
return 0
if ($node->NodeTypeAttribute("diskloadmfs_osid", \$mfs) != 0);
if (!defined($mfs));
......@@ -122,6 +122,9 @@ my %virtual_tables =
"virt_paths" => { rows => undef,
tag => "path_members",
row => "path_member"},
"experiment_blobs" => { rows => undef,
tag => "blobs",
row => "blob"},
# This is a fake table. See below. If we add more, lets generalize.
"external_sourcefiles" => { rows => undef,
tag => "nsfiles",
......@@ -63,7 +63,7 @@ EmulabResponse::~EmulabResponse()
ServerProxy::ServerProxy(xmlrpc_c::clientXmlTransportPtr transport,
ServerProxy::ServerProxy(xmlrpc_c::clientXmlTransport *transport,
bool wbxml_mode,
const char *url)
......@@ -135,7 +135,7 @@ class ServerProxy
* @param wbxml_mode Indicate whether or not wbxml should be used.
* @param url The url to use.
ServerProxy(xmlrpc_c::clientXmlTransportPtr transport,
ServerProxy(xmlrpc_c::clientXmlTransport *transport,
bool wbxml_mode = false,
const char *url = "");
......@@ -176,7 +176,7 @@ class ServerProxy
xmlrpc_c::value call(xmlrpc_c::rpcPtr rpc);
xmlrpc_c::clientXmlTransportPtr transport;
xmlrpc_c::clientXmlTransport *transport;
std::string server_url;
......@@ -997,7 +997,7 @@ SetExpPath(const char *path)
AddUserEnv(char *name, char *path)
AddUserEnv(const char *name, const char *path)
int retval = 0;
FILE *file;
......@@ -1042,7 +1042,8 @@ AddUserEnv(char *name, char *path)
AddAgent(event_handle_t handle,
char *vname, char *vnode, char *nodeid, char *ipaddr, char *type)
const char *vname, const char *vnode, const char *nodeid,
const char *ipaddr, const char *type)
struct agent *agentp;
......@@ -1173,7 +1174,7 @@ AddAgent(event_handle_t handle,
* Add an agent group to the list.
AddGroup(event_handle_t handle, char *groupname, char *agentname)
AddGroup(event_handle_t handle, const char *groupname, const char *agentname)
struct agent *group = NULL, *agent;
int retval = 0;
......@@ -1238,8 +1239,9 @@ AddGroup(event_handle_t handle, char *groupname, char *agentname)
AddEvent(event_handle_t handle, address_tuple_t tuple,
char *exidx, char *ftime, char *objname, char *exargs,
char *objtype, char *evttype, char *parent, char *triggertype)
const char *exidx, const char *ftime, const char *objname,
const char *exargs, const char *objtype, const char *evttype,
const char *parent, const char *triggertype)
timeline_agent_t ta = NULL;
sched_event_t event;
......@@ -299,14 +299,15 @@ RPC_metadata(char *pid, char *eid)
int lpc;
for (lpc = 0; lpc < userenv.size(); lpc++) {
char *name, *value;
string name, value;
map<string, xmlrpc_c::value> ue;
ue = (xmlrpc_c::value_struct)userenv[lpc];
name = (char *)((string)(xmlrpc_c::value_string)ue["name"]).c_str();
value = (char *)((string)(xmlrpc_c::value_string)ue["value"]).c_str();
name = (xmlrpc_c::value_string)ue["name"];
value = (xmlrpc_c::value_string)ue["value"];
if ((retval = AddUserEnv(name, value)) != 0)
if ((retval = AddUserEnv(name.c_str(),
value.c_str())) != 0)
return retval;
......@@ -585,17 +586,18 @@ RPC_agentlist(event_handle_t handle, char *pid, char *eid)
for (i = 0; i < agents.size(); i++) {
char *vname, *vnode, *nodeid, *ipaddr, *type;
string vname, vnode, nodeid, ipaddr, type;
vector<xmlrpc_c::value>agent =
vname = (char *)((string)(xmlrpc_c::value_string)agent[0]).c_str();
vnode = (char *)((string)(xmlrpc_c::value_string)agent[1]).c_str();
nodeid = (char *)((string)(xmlrpc_c::value_string)agent[2]).c_str();
ipaddr = (char *)((string)(xmlrpc_c::value_string)agent[3]).c_str();
type = (char *)((string)(xmlrpc_c::value_string)agent[4]).c_str();
info("D: adding agent %s\n", vname);
if (AddAgent(handle, vname, vnode, nodeid, ipaddr, type) < 0) {
vname = (xmlrpc_c::value_string)agent[0];
vnode = (xmlrpc_c::value_string)agent[1];
nodeid = (xmlrpc_c::value_string)agent[2];
ipaddr = (xmlrpc_c::value_string)agent[3];
type = (xmlrpc_c::value_string)agent[4];
info("D: adding agent %s\n", (char *)vname.c_str());
if (AddAgent(handle, vname.c_str(), vnode.c_str(),
nodeid.c_str(), ipaddr.c_str(), type.c_str()) < 0) {
return -1;
......@@ -616,16 +618,19 @@ RPC_grouplist(event_handle_t handle, char *pid, char *eid)
for (i = 0; i < groups.size(); i++) {
char *groupname, *agentname;
string groupname, agentname;
vector<xmlrpc_c::value> group =
groupname = (char *)((string)(xmlrpc_c::value_string)group[0]).c_str();
info("D: \tIn GroupList() parsed name %s\n", groupname);
agentname = (char *)((string)(xmlrpc_c::value_string)group[1]).c_str();
info("D: \tIn GroupList() parsed agent %s\n", agentname);
groupname = (xmlrpc_c::value_string)group[0];
info("D: \tIn GroupList() parsed name %s\n",
(char *)groupname.c_str());
agentname = (xmlrpc_c::value_string)group[1];
info("D: \tIn GroupList() parsed agent %s\n",
(char *)agentname.c_str());
if (AddGroup(handle, groupname, agentname) != 0) {
if (AddGroup(handle, groupname.c_str(),
agentname.c_str()) != 0) {
return -1;
......@@ -656,25 +661,27 @@ RPC_eventlist(char *pid, char *eid,
// info(" Interating over invoke results\n");
// XXX
for (i = 0; i < events.size(); i++) {
char *exidx, *extime, *objname, *objtype, *evttype, *exargs;
char *parent, *triggertype;
string exidx, extime, objname, objtype, evttype, exargs;
string parent, triggertype;
vector<xmlrpc_c::value> event =
exidx = (char *)((string)(xmlrpc_c::value_string)event[0]).c_str();
extime = (char *)((string)(xmlrpc_c::value_string)event[1]).c_str();
objname = (char *)((string)(xmlrpc_c::value_string)event[2]).c_str();
objtype = (char *)((string)(xmlrpc_c::value_string)event[3]).c_str();
evttype = (char *)((string)(xmlrpc_c::value_string)event[4]).c_str();
exargs = (char *)((string)(xmlrpc_c::value_string)event[5]).c_str();
parent = (char *)((string)(xmlrpc_c::value_string)event[6]).c_str();
triggertype = (char *)((string)(xmlrpc_c::value_string)event[7]).c_str();
exidx = (xmlrpc_c::value_string)event[0];
extime = (xmlrpc_c::value_string)event[1];
objname = (xmlrpc_c::value_string)event[2];
objtype = (xmlrpc_c::value_string)event[3];
evttype = (xmlrpc_c::value_string)event[4];
exargs = (xmlrpc_c::value_string)event[5];
parent = (xmlrpc_c::value_string)event[6];
triggertype = (xmlrpc_c::value_string)event[7];
// XXX
// info(" Adding an event\n");
// XXX
if (AddEvent(handle, tuple, exidx,
extime, objname, exargs, objtype, evttype,
parent, triggertype) < 0) {
if (AddEvent(handle, tuple, exidx.c_str(),
extime.c_str(), objname.c_str(),
exargs.c_str(), objtype.c_str(),
evttype.c_str(), parent.c_str(),
triggertype.c_str()) < 0) {
return -1;
......@@ -67,17 +67,19 @@ int RPC_eventlist(char *pid, char *eid,
event_handle_t handle, address_tuple_t tuple);
extern int SetExpPath(const char *path);
extern int AddUserEnv(char *name, char *value);
extern int AddUserEnv(const char *name, const char *value);
extern int AddAgent(event_handle_t handle,
char *vname, char *vnode, char *nodeid,
char *ipaddr, char *type);
const char *vname, const char *vnode, const char *nodeid,
const char *ipaddr, const char *type);
extern int AddGroup(event_handle_t handle, char *groupname, char *agentname);
extern int AddGroup(event_handle_t handle, const char *groupname,
const char *agentname);
extern int AddEvent(event_handle_t handle, address_tuple_t tuple,
char *exidx, char *ftime, char *objname, char *exargs,
char *objtype, char *evttype, char *parent, char *triggertype);
const char *exidx, const char *ftime, const char *objname,
const char *exargs, const char *objtype, const char *evttype,
const char *parent, const char *triggertype);
extern int AddRobot(event_handle_t handle,
struct agent *agent,
Some notes about the TPM-enforced boot path (SECURELOAD state machine).
The current implementation requires that we boot from a flash device
BEFORE we network boot (see the paper for details). Unfortunately,
there is a lot of magic associated with the PXEBOOTING state, which is
assumed to always be the first state we will see when a node boots.
However, in the secure boot path the first thing we see is a secure
transition to the GPXEBOOTING state, so magic had to be added for that!
In particular:
* When stated gets a transition to GPXEBOOTING, it forces the node into
the SECURELOAD op_mode. This is a new trigger called SECURELOAD and
a new trigger table entry:
insert into state_triggers values \
* Later, when we do get a PXEBOOTING state, we DON'T push the machine
into the PXEBOOT op_mode. We do this with an override trigger:
insert into state_triggers values \
This is supposed to override the more general any ('*') op_mode trigger
for the PXEBOOTING state and will just make sure we state in SECURELOAD.
Updates on 9/7/10:
We need two secure state machines, one for loading and one for booting.
The latter will be used for all boots on machines which have gPXE dongles,
regardless of what they are booting. So we need to amend the above to
account for:
* some nodes will always see GPXEBOOTING first
* nodes which have just undergone the secure load will then reboot into
the secure boot path
* if a node was in the SECURELOAD op_mode but didn't finish, it cannot
be allowed to SECUREBOOT (which shouldn't happen, it should wind up in
I think what we needs is a SECUREBOOT op_mode which is entered whenever
a node transitions to GPXEBOOTING. This is allowed from any op_mode/state
except from */SECVIOLATION.
SECURELOAD consists of:
PXEBOOTING -- "bootinfo" --> BOOTING
BOOTING -- "quote ok" --> RELOADSETUP
RELOADSETUP -- "reload ready" --> RELOADING
RELOADING -- "image ok" --> RELOADDONE
SECUREBOOT consists of:
PXEBOOTING -- "bootinfo" --> BOOTING
BOOTING -- "quote ok" --> TPMSIGNOFF
How do we differentiate the two op_modes when they arrive at GPXEBOOTING?
Well, os_load should have set the next_op_mode to SECURELOAD (assuming the
MFS is listed with that as its op_mode), so we check that in the SECUREBOOT
trigger used by a transition to GPXEBOOTING.
For a regular "untrusted" node boot through gPXE, we will arrive at
GPXEBOOTING from some arbitrary state, get forced into SECUREBOOT and
go through the steps. We neuter the actions that PXEBOOTING and BOOTING
normally take. As part of the TPMSIGNOFF, we perform those actions,
which include putting the node into PXEKERNEL op_mode (PXEBOOT action) and
then setting the nodes.op_mode as appropriate for the next boot (BOOTING
For a secure load, we again arrive at GPXEBOOTING from any state, get
forced into SECURELOAD, and work through the state machine. Again we
perform none of the normal triggers along the way: PXEBOOTING, BOOTING,
which leave the node in SECURELOAD/TPMSIGNOFF. Now we have a problem
because next_op_mode is still set to SECURELOAD which will tell the
SECUREBOOT trigger to keep us in the SECURELOAD op_mode. Hmm...I think
next_op_mode will get set by the os_select in RESET.
How do we do this? If we associate the SECUREBOOT trigger with
SECURELOAD/TPMSIGNOFF, then as soon as we get the TPMSIGNOFF event
(from the MFS?) it will get to step #5 below where it will trigger
RESET (clear the one-time boot info), RELOADDONEV2 (send apod), and
then SECUREBOOT. The last should:
* set DB next_op_mode to SECUREBOOT?
* directly change op_mode?
What happens if a node in a secure reload reboots in the middle?
stated will see a transition from SECURELOAD/!TPMSIGNOFF -> GPXEBOOTING
which is okay; since the op_mode is still SECURELOAD we will just restart.
For nodes that have a gPXE dongle, should we allow them to boot via the
"standard" boot path or require that they always boot via the dongle?
If the latter, we would have to be a bit more rigerous about making sure
they boot through GPXEBOOTING.
Notes on state transitions:
If a node state transition is reported, stated:
1. Check for invalid state transitions, reporting them if so
NEW: if mode==SECURELOAD, set state to SECVIOLATION
2. Update nodes.eventstate in DB
3. Check to see if there is a TBCOMMAND "timeout" for this node.
If so and command is REBOOT, see if we have rebooted and report if so.
NOTE: this step appears to just report and not ever do anything else.
4. Queue any per-node timeout associated with the current mode and new state
(from state_timeouts). This removes old timeouts for the node queued by
the previous state.
5. Check for per-node triggers associated with the current mode and new state
(from state_triggers) and execute them. Several of these triggers can
force op_mode transitions: PXEBOOT (to PXEBOOT), SECURELOAD (to
SECURELOAD), BOOTING (to DB bootopmode if in PXEKERNEL op_mode, to DB
nodes.next_op_mode if curstate==ISUP).
6. Check if current mode and new state should trigger an op_mode transition
(allowed transitions from mode_transitions). The actual next op_mode
comes from nodes.next_op_mode which is cleared afterward.
Notes on triggers:
* Transitions node into PXEKERNEL op_mode
Queries DB to find what opmode/osid to boot next, order is: next, temp, def.
Updates DB nodes table with osid. Update op_mode of node:
* if current mode is PXEKERNEL, transition into new mode
* else if came from ISUP state, transition into mode nodes.next_op_mode (?)
* else if next mode is RELOAD, update DB and stated state to make current
mode be RELOAD (does not do an opModeTransition like the others)
Marks end of successful boot. Resets internal stated flags, calls osselect
to clear DB one-time (next) boot info.
Marks end of reload and reboots machine (V2 only). Clears DB reload state
and sends an apod (V2 only).
Sends ISUP if node cannot.
* if nodes.osfeatures includes "isup", do nothing
* else if includes "ping", fire off "eventping" to send ISUP when node pings
* else generate an ISUP event for the node
Always generates an ISUP event, regardless of osfeatures
(used by the ALWAYSUP op_mode).
Ensures that certain DB port_registration entries exist (backward compat
for older images). Right now: if emulab_syncd port is not registered,
insert an entry with the default port.
Invokes the "power" command to perform the appropriate action.
Send an email notification to testbed-ops informing them that the node
has entered a particular op_mode/state.
Invokes the given script in the background, not waiting for the result.
If <path> is not absolute, use /usr/testbed/sbin/<path>.
Special hack to detect old diskload MFSes that cannot load multiple images,
and sends email to testbed-ops.
Notes on timeouts.
Timeouts define the length of time a node should stay in a particular
op_mode/state combo. If the node is in that state longer, the associated
action is triggered.
Actions are:
Reboot the node. This doesn't really do anything right now, it just sends
email to testbed-ops that a reboot was requested. According to the comment,
os_load and os_setup are still doing the actual reboots.
Send email that node has timed out in a particular state, to testbed-ops.
Transition node into a particular state. Does not change the op_mode.
Timeouts can also be associated with commands that stated is performing,
indicated by the op_mode being "TBCOMMAND" and state being a command,
instead of a real op_mode/state. The only recognized action is CMDRETRY
to retry the command. This is only used for rebooting and power "events",
so that those actions can be retried if necessary.