Commit 9d021a07 authored by Leigh Stoller's avatar Leigh Stoller

Checkpoint my dynamic event stuff, crude as it is. The idea for this first

draft is that the user will at the end of an experiment run, log into one
of his nodes and perform some analysis which is intended to be repeated at
the end of the next run, and in future instantiations of the template.

A new table called experiment_template_events holds the dynamic events for
the template. Right now I am supporting just program events, but it will be
easy to support arbitrary events later. As an absurd example:

	node6> /usr/local/bin/template_analyze ~/data_analyze arg arg ...

The user is currently responsible for making sure the output goes into a
file in the archive. I plan to make the template_analyze wrapper handle
that automatically later, but for now what you really want is to invoke a
script that encapsulates that, redirecting output to $ARCHIVE (this
variable is installed in the environment template_analyze.

The wrapper script will save the current time, and then run the program.
If the program terminates with a zero exit status, it will ssh over to ops
and invoke an xmlrpc routine to tell boss to add a program event to both
the eventlist for the current instance, and to the template_eventlist for
future instances. The time of the event is the relative start time that was
saved above (remember, each experiment run replays the event stream from
time zero).

For the future, we want to allow this to be done on ops as well, but
that will take more infrastructure, to run "program agents" on ops.

It would be nice to install the ssl xmlrpc client side on our images so
that we do not have to ssh to ops to invoke the client.
parent 114de217
......@@ -47,6 +47,7 @@ client-install: client
-mkdir -p $(DESTDIR)$(CLIENT_MANDIR)/man1
$(INSTALL) -m 644 $(SRCDIR)/install-tarfile.1 $(DESTDIR)$(CLIENT_MANDIR)/man1/install-tarfile.1
$(INSTALL_PROGRAM) $(SRCDIR)/install-rpm $(LBINDIR)/install-rpm
$(INSTALL_PROGRAM) $(SRCDIR)/template_analyze $(LBINDIR)/template_analyze
ifeq ($(SYSTEM),FreeBSD)
$(INSTALL_PROGRAM) $(SRCDIR)/create-image $(LBINDIR)/
$(INSTALL_PROGRAM) $(SRCDIR)/create-swapimage $(LBINDIR)/
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
#
# Do some analysis.
#
sub usage()
{
print STDOUT "Usage: template_analyze <script> [args ...]\n";
exit(-1);
}
my $optlist = "";
#
# Turn off line buffering on output
#
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
#
use libsetup;
#
# Need the pid/eid.
#
my ($pid, $eid, $vname) = check_nickname();
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV < 1) {
usage();
}
my $scriptname = shift(@ARGV);
#
# Grab the user environment variables.
#
open(USRENV, "/var/emulab/boot/tmcc/userenv")
or die("Could not open user environment file!\n");
while (<USRENV>) {
if ($_ =~ /^(.*)=(.*)$/) {
$ENV{$1} = $2;
}
}
#
# Run the analysis.
# If it completes okay then schedule an event for the current time.
#
my $now = time();
my $childpid = fork();
if (! $childpid) {
#
# Child runs command
#
exec $scriptname, @ARGV;
die("Could not exec $scriptname\n");
}
#
# Wait until child exits or until user gets bored and types ^C.
#
waitpid($childpid, 0);
if ($?) {
exit(-1);
}
#
# Okay, ssh over to ops to run the xmlrpc client. SHould be installed locally.
#
system("ssh -q -o BatchMode=yes -o StrictHostKeyChecking=no ops ".
" /usr/testbed/bin/sslxmlrpc_client.py -m template addprogevent ".
" proj=$pid exp=$eid vnode=$vname when=$now ".
" cmd=\"'$scriptname @ARGV'\"");
exit 0;
......@@ -528,6 +528,22 @@ CREATE TABLE experiment_stats (
KEY rsrcidx (rsrcidx)
) TYPE=MyISAM;
--
-- Table structure for table `experiment_template_events`
--
CREATE TABLE experiment_template_events (
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
vname varchar(64) NOT NULL default '',
vnode varchar(32) NOT NULL default '',
time float(10,3) NOT NULL default '0.000',
objecttype smallint(5) unsigned NOT NULL default '0',
eventtype smallint(5) unsigned NOT NULL default '0',
arguments text,
PRIMARY KEY (parent_guid, parent_vers, vname)
) TYPE=MyISAM;
--
-- Table structure for table `experiment_template_graphs`
--
......
......@@ -3602,3 +3602,21 @@ last_net_act,last_cpu_act,last_ext_act);
4.80: Fix to previous revision; Skip to next entry.
4.81: Add a table to store dynamic template events. These events are
copied into the event stream when creating an instance (added to
the eventlist for the experiment).
**** Skip this stuff below if you just did 4.41 above.
CREATE TABLE experiment_template_events (
parent_guid varchar(16) NOT NULL default '',
parent_vers smallint(5) unsigned NOT NULL default '0',
vname varchar(64) NOT NULL default '',
vnode varchar(32) NOT NULL default '',
time float(10,3) NOT NULL default '0.000',
objecttype smallint(5) unsigned NOT NULL default '0',
eventtype smallint(5) unsigned NOT NULL default '0',
arguments text,
PRIMARY KEY (parent_guid, parent_vers, vname)
) TYPE=MyISAM;
......@@ -259,6 +259,25 @@ CREATE TABLE virt_parameters (
PRIMARY KEY (pid,eid,name)
) TYPE=MyISAM;
#
# Events that are dynamically created by the user, as for analysis.
# Assumed to be just program agent events; generalize later, perhaps.
#
CREATE TABLE experiment_template_events (
-- Globally Unique ID of the ExperimentTemplate this record belongs to.
parent_guid varchar(16) NOT NULL default '',
-- Version number for tracking modifications
parent_vers smallint(5) unsigned NOT NULL default '0',
-- stuff for the eventlist table
vname varchar(64) NOT NULL default '',
vnode varchar(32) NOT NULL default '',
time float(10,3) NOT NULL default '0.000',
objecttype smallint(5) unsigned NOT NULL default '0',
eventtype smallint(5) unsigned NOT NULL default '0',
arguments text,
PRIMARY KEY (parent_guid, parent_vers, vname)
) TYPE=MyISAM;
#
# This table is a wrapper around the current experiments table, which will
# continue to be the basic structure for the swapin. This includes all of
......
......@@ -1220,6 +1220,30 @@ sub NewInstance($$$)
return Template::Instance->Create(\%args);
}
#
# Lookup specific instance by its exptidx. The point of this is to ensure
# that the instance is really associated with the template, since by itself,
# the exptidx is enough to find the instance record.
#
sub LookupInstance($$)
{
my ($self, $exptidx) = @_;
# Must be a real reference.
return undef
if (! ref($self));
my $instance = Template::Instance->LookupByExptidx($exptidx);
return undef
if (!defined($instance));
return undef
if (! ($self->guid() == $instance->guid() &&
$self->vers() == $instance->vers()));
return $instance;
}
#
# Generate a list of instances for a template and save in the template
# structure. The argument indicates whether you want just active, or all
......@@ -2341,6 +2365,178 @@ sub UpdateExportTime($)
return Refresh($self);
}
#
# Add a dynamic event. A total hack.
#
sub AddEvent($$$$)
{
my ($self, $timestamp, $vnode, $commandline) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $guid = $self->guid();
my $vers = $self->vers();
my $pid = $self->pid();
my $eid = $self->eid();
my $cmd = DBQuoteSpecial("COMMAND=$commandline");
# We assume program agent events right now. Change later.
my $query_result =
DBQueryWarn("select idx from event_objecttypes where type='PROGRAM'");
return -1
if (!$query_result || ! $query_result->numrows);
my ($event_objecttype) = $query_result->fetchrow_array();
# And its a start event of course.
$query_result =
DBQueryWarn("select idx from event_eventtypes where type='START'");
return -1
if (!$query_result || ! $query_result->numrows);
my ($event_eventtype) = $query_result->fetchrow_array();
#
# Need to convert the timestamp from absolute unixtime to relative time.
#
my $runidx = $self->runidx();
my $exptidx = $self->exptidx();
$query_result =
DBQueryWarn("select UNIX_TIMESTAMP(start_time) from experiment_runs ".
"where exptidx='$exptidx' and idx='$runidx'");
return -1
if (!$query_result || ! $query_result->numrows);
my ($starttime) = $query_result->fetchrow_array();
my $firetime = $timestamp - $starttime;
# Make up a vname.
my $vname = "dprog" . $firetime;
#
# Add the event to the static event stream for the instance experiment.
#
$query_result =
DBQueryWarn("insert into eventlist set ".
" pid='$pid', eid='$eid', time='$firetime', ".
" vnode='$vnode', vname='$vname', ".
" objecttype='$event_objecttype', ".
" eventtype='$event_eventtype', ".
" atstring='', arguments=$cmd");
return -1
if (!$query_result || ! $query_result->numrows);
# Grab the new idx just in case we need to delete.
my $eventidx = $query_result->insertid;
# Need a virt_agent entry.
if (!DBQueryWarn("insert into virt_agents set ".
" pid='$pid', eid='$eid', ".
" vnode='$vnode', vname='$vname', ".
" objecttype='$event_objecttype'")) {
DBQueryWarn("delete from eventlist ".
"where pid='$pid' and eid='$eid' and idx='$eventidx'");
return -1;
}
# Need a virt_programs entry.
if (!DBQueryWarn("insert into virt_programs set ".
" pid='$pid', eid='$eid', ".
" vnode='$vnode', vname='$vname', ".
" dir='', timeout=0, expected_exit_code=0, ".
" command=$cmd")) {
DBQueryWarn("delete from virt_programs ".
"where pid='$pid' and eid='$eid' and ".
" vnode='$vnode' and vname='$vname'");
DBQueryWarn("delete from eventlist ".
"where pid='$pid' and eid='$eid' and idx='$eventidx'");
return -1;
}
# Add to template event list; we will need to hand copy these into
# the eventlist for new instances as they are created.
if (!DBQueryWarn("insert into experiment_template_events set ".
" parent_guid='$guid', parent_vers='$vers', ".
" time='$firetime', ".
" vnode='$vnode', vname='$vname', ".
" objecttype='$event_objecttype', ".
" eventtype='$event_eventtype', ".
" arguments=$cmd")) {
DBQueryWarn("delete from eventlist ".
"where pid='$pid' and eid='$eid' and idx='$eventidx'");
DBQueryWarn("delete from virt_agents ".
"where pid='$pid' and eid='$eid' and ".
" vnode='$vnode' and vname='$vname'");
DBQueryWarn("delete from virt_programs ".
"where pid='$pid' and eid='$eid' and ".
" vnode='$vnode' and vname='$vname'");
return -1;
}
return 0;
}
#
# Copy the events from the template to the instance.
#
sub CopyTemplateEvents($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $guid = $self->guid();
my $vers = $self->vers();
my $pid = $self->pid();
my $eid = $self->eid();
my $query_result =
DBQueryWarn("select * from experiment_template_events ".
"where parent_guid='$guid' and parent_vers='$vers'");
return -1
if (!$query_result);
while (my $rowref = $query_result->fetchrow_hashref()) {
my $vnode = $rowref->{"vnode"};
my $firetime = $rowref->{"time"};
my $vname = $rowref->{"vname"};
my $event_objecttype = $rowref->{"objecttype"};
my $event_eventtype = $rowref->{"eventtype"};
my $cmd = DBQuoteSpecial($rowref->{"arguments"});
#
# Any failure and and the instance will be killed by the
# caller, so no need to clean up any mess created below.
#
DBQueryWarn("insert into eventlist set ".
" pid='$pid', eid='$eid', time='$firetime', ".
" vnode='$vnode', vname='$vname', ".
" objecttype='$event_objecttype', ".
" eventtype='$event_eventtype', ".
" atstring='', arguments=$cmd")
or return -1;
# Need a virt_agent entry.
DBQueryWarn("insert into virt_agents set ".
" pid='$pid', eid='$eid', ".
" vnode='$vnode', vname='$vname', ".
" objecttype='$event_objecttype'")
or return -1;
# Need a virt_programs entry.
DBQueryWarn("insert into virt_programs set ".
" pid='$pid', eid='$eid', ".
" vnode='$vnode', vname='$vname', ".
" dir='', timeout=0, expected_exit_code=0, ".
" command=$cmd")
or return -1;
}
return 0;
}
# _Always_ make sure that this 1 is at the end of the file...
1;
......@@ -1629,6 +1629,11 @@ Simulator instproc make_event {outer event} {
set args "DIGESTER={$(-digester)}"
}
}
"snapshot" {
set otype SIMULATOR
set etype SNAPSHOT
set args "LOGHOLE_ARGS='-s'"
}
"cleanlogs" {
set otype SIMULATOR
set etype RESET
......
......@@ -57,6 +57,7 @@ sub ShowHidden();
sub HideHidden();
sub Activate();
sub InActivate();
sub AddEvent();
#
# Testbed Support libraries
......@@ -129,6 +130,9 @@ elsif ($action eq "activate") {
elsif ($action eq "inactivate") {
exit(InActivate());
}
elsif ($action eq "addevent") {
exit(AddEvent());
}
usage();
sub HideTemplate()
......@@ -207,6 +211,91 @@ sub InActivate()
return 0;
}
#
# Add a dynamic event.
#
sub AddEvent()
{
my $timestamp;
my $commandline;
my $vnode;
my $exptidx;
# Parse the rest of the options.
while (@ARGV) {
my $arg = shift(@ARGV);
if ($arg eq "-t") {
usage()
if (@ARGV == 0);
$timestamp = shift(@ARGV);
if ("$timestamp" eq "now") {
$timestamp = time();
}
elsif ($timestamp =~ /^([\d]*)$/) {
$timestamp = $1;
}
else {
tbdie("Illegal characters in timestamp!");
}
}
elsif ($arg eq "-c") {
usage()
if (@ARGV == 0);
$arg = shift(@ARGV);
if (! TBcheck_dbslot($arg, 'eventlist','arguments',
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tbdie("Illegal characters in commandline!");
}
$commandline = $arg;
}
elsif ($arg eq "-n") {
usage()
if (@ARGV == 0);
$arg = shift(@ARGV);
if (! TBcheck_dbslot($arg, 'virt_nodes','vname',
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tbdie("Illegal characters in node name!");
}
$vnode = $arg;
}
elsif ($arg eq "-i") {
usage()
if (@ARGV == 0);
$arg = shift(@ARGV);
if (! TBcheck_dbslot($arg, "default", "int",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tbdie("Illegal characters in experiment index!");
}
$exptidx = $arg;
}
else {
usage();
}
}
usage()
if (! (defined($timestamp) && defined($commandline) &&
defined($vnode) && defined($exptidx)));
my $instance = $template->LookupInstance($exptidx);
if (!defined($instance)) {
tbdie("Could not get instance record for experiment $exptidx!");
}
$instance->AddEvent($timestamp, $vnode, $commandline) == 0
or tbdie("Could not add dynamic event to $instance!");
return 0;
}
#
# Parse command arguments. Once we return from getopts, all that are
# left are the required arguments.
......
......@@ -469,6 +469,10 @@ $instance->CopyDataStore($template_tag,
"$instance_path/template_datastore") == 0
or fatal(-1, "Could not copy datastore ($template_tag) to instance");
# Ditto for dynamic events.
$instance->CopyTemplateEvents() == 0
or fatal(-1, "Could not copy template events to instance");
#
# Ick, Ick, Ick. I want these to be at the beginning of the enviroment
# strings so they are set in case the user has used any OPT variables
......@@ -491,6 +495,35 @@ DBQueryFatal("replace into virt_user_environment set ".
" name='ARCHIVE', value='$instance_path/archive',".
" idx=2, pid='$pid', eid='$eid'");
if ($experiment->dpdb() && $experiment->dpdbname() ne '') {
my $dpdbname = $experiment->dpdbname();
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_DBNAME', value='$dpdbname',".
" pid='$pid', eid='$eid'");
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_HOST', value='$CONTROL',".
" pid='$pid', eid='$eid'");
#
# XXX This needs to change to a per-experiment user/password.
#
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_USER', value='$dbuid',".
" pid='$pid', eid='$eid'");
my $query_result =
DBQueryFatal("select mailman_password from users where uid='$dbuid'");
my ($mailman_password) = $query_result->fetchrow_array();
if (defined($mailman_password)) {
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_PASSWORD', value='$mailman_password',".
" pid='$pid', eid='$eid'");
}
}
#
# Now do the swapin (or it gets queued if a batch experiment).
#
......
......@@ -469,6 +469,10 @@ $instance->CopyDataStore($template_tag,
"$instance_path/template_datastore") == 0
or fatal(-1, "Could not copy datastore ($template_tag) to instance");
# Ditto for dynamic events.
$instance->CopyTemplateEvents() == 0
or fatal(-1, "Could not copy template events to instance");
#
# Ick, Ick, Ick. I want these to be at the beginning of the enviroment
# strings so they are set in case the user has used any OPT variables
......@@ -491,6 +495,35 @@ DBQueryFatal("replace into virt_user_environment set ".
" name='ARCHIVE', value='$instance_path/archive',".
" idx=2, pid='$pid', eid='$eid'");
if ($experiment->dpdb() && $experiment->dpdbname() ne '') {
my $dpdbname = $experiment->dpdbname();
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_DBNAME', value='$dpdbname',".
" pid='$pid', eid='$eid'");
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_HOST', value='$CONTROL',".
" pid='$pid', eid='$eid'");
#
# XXX This needs to change to a per-experiment user/password.
#
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_USER', value='$dbuid',".
" pid='$pid', eid='$eid'");
my $query_result =
DBQueryFatal("select mailman_password from users where uid='$dbuid'");
my ($mailman_password) = $query_result->fetchrow_array();
if (defined($mailman_password)) {
DBQueryFatal("insert into virt_user_environment set ".
" name='DP_PASSWORD', value='$mailman_password',".
" pid='$pid', eid='$eid'");
}
}
#
# Now do the swapin (or it gets queued if a batch experiment).
#
......
......@@ -616,7 +616,7 @@ sub GraphDB(@)
print "DB '$dbname' does not exist!\n";
return -1;
}
my $gd = new GD::Graph::lines(700,425);
my $gd = new GD::Graph::lines(500,325);
if (!$gd) {
print "Could not create a new graph object!\n";
return -1;
......
......@@ -210,7 +210,7 @@ function ShowVis($pid, $eid, $zoom = 1.25, $detail = 1) {
" position: relative; z-index:1010; height: 450px; ".
" width: 90%; border: 2px solid black;'>\n".
" <div id=myvisdiv style='position:relative;'>\n".
" <img id=myvisimg border=0 ".
" <img id=myvisimg border=0 style='cursor: move;' ".
" onLoad=\"setTimeout('ShowVisInit();', 10);\" ".
" src='top2image.php3?pid=$pid&eid=$eid".
"&zoom=$zoom&detail=$detail'>\n".
......
......@@ -653,6 +653,7 @@ class Template
echo $imap;
echo "<img id=\"mygraphimg\" border=0 usemap=\"#TemplateGraph\" ";
echo " onLoad=\"setTimeout('ShowGraphInit();', 10);\" ";
echo " style='cursor: move;' ";
echo " src='template_graph.php?guid=$guid&now=$now'>\n";
echo "</div>\n";
echo "</div>\n";
......@@ -688,7 +689,7 @@ class Template
"width: 700px; border: 2px solid black;'>\n";
echo "<div id=\"myvisdiv\" style='position:relative;'>\n";
echo "<img id=\"myvizimg\" border=0 ";
echo "<img id=\"myvizimg\" border=0 style='cursor: move;' ";
echo " onLoad=\"setTimeout('ShowVisInit();', 100);\" ";
echo " src='top2image.php3?pid=$pid&eid=$eid".
"&zoom=$zoom&detail=$detail&now=$now'>\n";
......
......@@ -258,6 +258,20 @@ def CheckIsAdmin(uid):
return res[0][0];
#
# Template lookup, by exptidx of an experiment.
#
def TemplateLookup(exptidx):
res = DBQueryFatal("select parent_guid,parent_vers from "
" experiment_template_instances "
"where exptidx=%s",
(exptidx,))
if len(res) == 0:
return None
return (res[0][0], res[0][1])
#
# This is a wrapper class so that you can invoke methods in dotted form.
# For example experiment.swapexp(...).
......@@ -270,6 +284,7 @@ class EmulabServer:
self.uid = pwd.getpwuid(os.getuid())[0]
self.instances["experiment"] = experiment(readonly=self.readonly);
self.instances["template"] = template(readonly=self.readonly);
if readonly:
return
......@@ -4406,6 +4421,83 @@ class elabinelab:
return EmulabResponse(RESPONSE_SUCCESS, result, output=str(result))
pass
#
# This class implements the server side of the XMLRPC interface to experiments.
#
class template:
##
# Initialize the object. Currently only sets the objects 'VERSION' value.
#
def __init__(self, readonly=0):
self.readonly = readonly;
self.VERSION = VERSION
self.uid = pwd.getpwuid(os.getuid())[0]
return
def addprogevent(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")
if self.readonly:
return EmulabResponse(RESPONSE_FORBIDDEN,
output="Insufficient privledge to invoke method")
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
argerror = CheckRequiredArgs(argdict,
("proj", "exp", "when", "vnode", "cmd"))
if (argerror):
return argerror
if not (re.match("^[-\w]*$", argdict["proj"]) and
re.match("^[-\w]*$", argdict["exp"]) and
re.match("^[-\w]*$", argdict["vnode"]) and
re.match("^[-\w]*$", argdict["when"])):
return EmulabResponse(RESPONSE_BADARGS,
output="Improperly formed arguments")
#
# Check permission. This will check proj/exp for illegal chars.
#
permerror = CheckExptPermission(self.uid,
argdict["proj"], argdict["exp"])
if (permerror):
return permerror
# Need to pass experiment index to script.
dbres = DBQueryFatal("SELECT idx FROM experiments "
"WHERE pid=%s and eid=%s",
(argdict["proj"], argdict["exp"]))
if len(dbres) == 0:
return EmulabResponse(RESPONSE_ERROR,
output="No such experiment!")
exptidx = dbres[0][0]
(guid,version) = TemplateLookup(exptidx)
if guid == None:
return EmulabResponse(RESPONSE_ERROR,
output="No such template instance")
argstr = " -a addevent " + str(guid) + "/" + str(version)
argstr += " -i " + str(exptidx)
argstr += " -t " + str(argdict["when"])
argstr += " -n " + argdict["vnode"]