Commit 904d2b05 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Another request fullfilled; allow templates to be instantiated but

not swapped in (equiv of plain experiment preload), and then swapped
in later. This fulfills flyspray request FS#100.
parent 2e2cfef9
......@@ -25,7 +25,7 @@ BIN_STUFF = power snmpit tbend tbprerun tbreport \
template_swapin template_swapout template_graph \
template_exprun template_delete template_metadata \
template_export template_control template_commit \
template_analyze template_linkgraph
template_analyze template_linkgraph template_instantiate
SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
batch_daemon exports_setup reload_daemon sched_reserve \
......@@ -59,7 +59,7 @@ LIBEXEC_STUFF = rmproj wanlinksolve wanlinkinfo \
webtemplate_swapin webtemplate_swapout webtemplate_exprun \
webtemplate_graph webtemplate_metadata webtemplate_export \
webtemplate_control webtemplate_commit webtemplate_analyze \
webtemplate_linkgraph
webtemplate_linkgraph webtemplate_instantiate
LIB_STUFF = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm power_rpc27.pm \
......
......@@ -1246,7 +1246,7 @@ sub MetadataList($$)
#
sub NewInstance($$$)
{
my ($self, $eid, $creator) = @_;
my ($self, $eid, $creator, $description) = @_;
# Must be a real reference.
return undef
......@@ -1259,6 +1259,8 @@ sub NewInstance($$$)
$args{'pid'} = $self->pid();
$args{'eid'} = $eid;
$args{'uid'} = $creator;
$args{'description'} = $description
if (defined($description));
return Template::Instance->Create(\%args);
}
......@@ -1703,10 +1705,17 @@ sub LookupByExptidx($$)
sub Create($$)
{
my ($class, $argref) = @_;
my $description;
return undef
if (ref($class));
# must treat specially.
if (exists($argref->{'description'})) {
$description = DBQuoteSpecial($argref->{'description'});
delete($argref->{'description'});
}
my $query = "insert into experiment_template_instances set ".
join(",", map("$_='" . $argref->{$_} . "'", keys(%{$argref})));
......@@ -1714,6 +1723,8 @@ sub Create($$)
$query .= ", "
if (defined($argref) && scalar(keys%{$argref}));
$query .= "start_time=now(),continue_time=now() ";
$query .= ",description=$description"
if (defined($description));
my $query_result = DBQueryWarn($query);
return undef
......@@ -1741,6 +1752,7 @@ sub continue_time($) { return field($_[0], 'continue_time'); }
sub runtime($) { return field($_[0], 'runtime'); }
sub locked($) { return field($_[0], 'locked'); }
sub locker_pid($) { return field($_[0], 'locker_pid'); }
sub description($) { return field($_[0], 'description'); }
sub template($) { return ((!ref($_[0])) ? -1 : $_[0]->{'TEMPLATE'}); }
# The path is the path of the experiment.
......
......@@ -33,6 +33,7 @@ sub usage()
"-e <eid> <guid/vers>\n".
"switches and arguments:\n".
"-b - batchmode; insert into batch queue\n".
"-p - preload only; do not swapin\n".
"-w - wait for template to be instantiated\n".
"-q - be less chatty\n".
"-E <str> - A pithy sentence describing the instance\n".
......@@ -47,12 +48,13 @@ sub usage()
"<guid/vers> - GUID and version to swapin\n");
exit(-1);
}
my $optlist = "qwe:S:L:na:l:se:x:bE:t:r:f";
my $optlist = "qwe:S:L:na:l:se:x:bE:t:r:fp";
my %options = ();
my $quiet = 0;
my $waitmode = 0;
my $batchmode = 0;
my $foreground = 0;
my $preload = 0;
my $description;
my $paramfile;
my $guid;
......@@ -82,7 +84,6 @@ my $EVhandle;
my $exptidx;
my $template;
my $instance;
my $run;
my $logname;
my $template_tag;
my @ExptStates = ();
......@@ -99,7 +100,7 @@ my $replay_run; # Replay starting with a particular run in instance.
# Programs we need
my $checkquota = "$TB/sbin/checkquota";
my $batchexp = "$TB/bin/batchexp";
my $swapexp = "$TB/bin/swapexp";
my $swapin = "$TB/bin/template_swapin";
my $endexp = "$TB/bin/endexp";
my $dbcontrol = "$TB/sbin/opsdb_control";
my $archcontrol = "$TB/bin/archive_control";
......@@ -276,7 +277,7 @@ if (defined($paramfile)) {
# Generate a new template instance record.
# We will finish updating it later.
#
$instance = $template->NewInstance($eid, $dbuid);
$instance = $template->NewInstance($eid, $dbuid, $description);
if (!defined($instance)) {
fatal(-1, "Could not insert new experiment instance record!");
}
......@@ -508,9 +509,6 @@ $args{'template_tag'} = $template_tag;
$instance->Update(0, \%args) == 0
or fatal(-1, "Could not update experiment instance record!");
# Event connect now that experiment is created.
SetupEventHandler();
my $workdir = $experiment->WorkDir();
my $userdir = $experiment->UserDir();
......@@ -560,97 +558,26 @@ print "Writing program agent info ...\n";
$instance->WriteProgramAgents() == 0
or fatal(-1, "Could not write program agent info");
#
# Now do the swapin (or it gets queued if a batch experiment).
#
@arguments = ($swapexp, "-q", "-x", "-s", "in", $pid, $eid);
system(@arguments);
fatal($? >> 8, "Could not instantiate the experiment")
if ($?);
#
# We will spew forth info to the user each time the batch daemon tries to
# swap it in.
#
if ($batchmode) {
if (! $preload) {
#
# Spin waiting for the state to change in swapexp. We are waiting for
# it to swapin or go back to swapped.
# Now do the swapin (or it gets queued if a batch experiment).
#
my $queued = 0;
while (1) {
@ExptStates = ();
event_poll_blocking($EVhandle, 500);
next
if (! @ExptStates);
foreach my $state (@ExptStates) {
print "$state\n";
if ($state eq EXPTSTATE_ACTIVATING()) {
print "Experiment is starting a swapin attempt ...\n";
}
elsif ($state eq EXPTSTATE_ACTIVE()) {
print "Experiment swapped in!\n";
goto done;
}
elsif ($state eq EXPTSTATE_QUEUED()) {
# Failed to swapin; still queued in the batch system.
if (! $queued) {
print "Experiment has entered the batch system\n";
$queued = 1;
}
else {
print "Experiment swapin attempt failed.\n";
}
}
elsif ($state eq EXPTSTATE_SWAPPED()) {
# Dumped out of the batch system for some reason.
print "Experiment has been removed from the batch queue.\n";
#
# We are done; remove record of this attempt and exit.
#
fatal(1, "Experiment has been removed from the batch queue");
}
}
system("$swapin -w -f -e $eid $guid/$version");
if ($?) {
fatal(-1, "Could not swapin instance $instance!");
}
done:
}
#
# Lets commit the experiment archive now that it is active. The experiment is
# already running, but thats not a big deal.
#
system("$archcontrol -t instantiate commit $pid $eid");
if ($?) {
fatal(-1, "Could not commit archive!");
}
#
# All instances currently start with a default run.
#
$run = $instance->NewRun($eid, $description);
if (!defined($run)) {
fatal(-1, "Could not create new experiment run for $instance!");
}
#
# And the bindings for the default run ...
#
foreach my $name (keys(%parameters)) {
my $value = $parameters{$name};
$instance->NewRunBinding($name, $value) == 0
or fatal(-1, "Error inserting run binding into DB!");
else {
#
# Lets commit the experiment archive now. The experiment might already
# be running, but thats not a big deal.
#
system("$archcontrol -t instantiate commit $pid $eid");
if ($?) {
fatal(-1, "Could not commit archive!");
}
}
$instance->StartRun(Template::STARTRUN_FLAGS_FIRSTRUN()) == 0
or fatal(-1, "Could not update start time in instance record!");
# Stop the web interface from spewing.
TBExptCloseLogFile($pid, $eid)
if (defined($logname) && !$batchmode);
......@@ -712,6 +639,9 @@ sub ParseArgs()
if (defined($options{"f"})) {
$foreground = 1;
}
if (defined($options{"p"})) {
$preload = 1;
}
if (defined($options{"b"})) {
$batchmode = 1;
}
......@@ -786,7 +716,7 @@ sub ParseArgs()
if (defined($options{"E"})) {
if (! TBcheck_dbslot($options{"E"},
"experiment_templates", "description",
"experiment_template_instances", "description",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tbdie("Improper template description!");
}
......@@ -809,54 +739,6 @@ sub ParseArgs()
}
}
#
# Subscribe to experiment state change events.
#
sub SetupEventHandler()
{
my $port = @BOSSEVENTPORT@;
my $URL = "elvin://localhost:$port";
# Connect to the event system, and subscribe the the events we want
$EVhandle = event_register($URL, 0);
if (!$EVhandle) {
tbdie("Unable to register with event system\n");
}
my $tuple = address_tuple_alloc();
if (!$tuple) {
tbdie("Could not allocate an address tuple\n");
}
%$tuple = ( objtype => libdb::TBDB_TBEVENT_EXPTSTATE(),
objname => "$pid/$eid",
expt => "$pid/$eid",
host => $BOSSNODE,
);
if (!event_subscribe($EVhandle, \&EventHandler, $tuple)) {
tbdie("Could not subscribe to events\n");
}
}
#
# Callback for above.
#
sub EventHandler($$$) {
my ($handle,$notification,undef) = @_;
my $objname = event_notification_get_objname($handle,$notification);
my $eventtype = event_notification_get_eventtype($handle,$notification);
print "$objname, $eventtype\n";
return
if ($objname ne "$pid/$eid");
push(@ExptStates, $eventtype);
}
#
# Cleanup the mess.
#
......
......@@ -14,7 +14,7 @@ use XML::Simple;
use Data::Dumper;
#
# Create a new experiment template.
# Swapin a previously instantiated template.
#
# Exit codes are important; they tell the web page what has happened so
# it can say something useful to the user. Fatal errors are mostly done
......@@ -28,37 +28,22 @@ use Data::Dumper;
sub usage()
{
print(STDERR
"Usage: template_swapin [-q] [-w] [-s]\n".
" [-S reason] [-L reason] [-a <time>] [-l <time>] ".
"-e <eid> <guid/vers>\n".
"Usage: template_swapin [-q] [-w] -e <eid> <guid/vers>\n".
"switches and arguments:\n".
"-b - batchmode; insert into batch queue\n".
"-w - wait for template to be instantiated\n".
"-q - be less chatty\n".
"-E <str> - A pithy sentence describing the instance\n".
"-x <file>- XML file of parameter bindings\n".
"-S <str> - Instance cannot be swapped; must provide reason\n".
"-L <str> - Instance cannot be IDLE swapped; must provide reason\n".
"-n - Do not send idle email (internal option only)\n".
"-a <nnn> - Auto swapout nnn minutes after instance is swapped in\n".
"-l <nnn> - Auto swapout nnn minutes after instance goes idle\n".
"-s - Save disk state on swapout\n".
"-e <eid> - The instance name (unique, alphanumeric, no blanks)\n".
"<guid/vers> - GUID and version to swapin\n");
exit(-1);
}
my $optlist = "qwe:S:L:na:l:se:x:bE:t:r:f";
my $optlist = "qwe:f";
my %options = ();
my $quiet = 0;
my $waitmode = 0;
my $batchmode = 0;
my $foreground = 0;
my $description;
my $paramfile;
my $guid;
my $version;
my $eid;
my %parameters = ();
#
# Configure variables
......@@ -79,9 +64,11 @@ my $user_name;
my $user_email;
my $dbuid;
my $EVhandle;
my $exptidx;
my $template;
my $instance;
my $experiment;
my %parameters;
my $pid;
my $run;
my $logname;
my $template_tag;
......@@ -90,11 +77,6 @@ my @ExptStates = ();
my $cleaning = 0;
my $exptcreated = 0;
my $justexit = 1;
# For replay
my $replay_exptidx; # Instance to replay.
my $replay_runidx; # Optional run within instance to replay.
my $replay_instance; # Replay a complete instance, from first run.
my $replay_run; # Replay starting with a particular run in instance.
# Programs we need
my $checkquota = "$TB/sbin/checkquota";
......@@ -195,148 +177,28 @@ if (! TBProjAccessCheck($dbuid,
"$guid/$version");
exit(1);
}
$pid = $template->pid();
#
# Grab instance and/or run if this is a replay.
# Grab instance; better exist since this is a swapin of an instance!
#
if (defined($replay_exptidx)) {
$replay_instance =
Template::Instance->LookupByExptidx($replay_exptidx);
if (!defined($replay_instance)) {
tbdie("Replay Instance $replay_exptidx does not exist!");
}
if (!defined($replay_runidx)) {
#
# Default to first run.
#
# XXX Need to convert these other routines to return Run objects
my $row = $replay_instance->FirstRun();
if (!defined($row)) {
tbdie("Could not find first run for $replay_instance!");
}
$replay_runidx = $row->{'idx'};
}
$replay_run =
Template::Instance::Run->LookupByID($replay_exptidx, $replay_runidx);
if (!defined($replay_run)) {
tbdie("Replay Run $replay_runidx does not exist!");
}
}
#
# If we have a parameter file, we need to copyin the values and store
# them in the DB for this experiment. Note that these override existing
# values, so we start with those first.
# We need to find the experiment so we can find the instance.
# This is wrong, but necessary cause of how templates are layered over
# the existing experiment structure.
#
# XXX Do we keep these values in the DB forever? Or just keep the XML file
# around and archived?
#
$template->FormalParameterList(\%parameters) == 0
or tbdie("Could not get formal parameter list for $template");
if (defined($paramfile)) {
my $donebad = 0;
my $parse = XMLin($paramfile,
VarAttr => 'name',
ContentKey => '-content',
SuppressEmpty => undef);
foreach my $name (keys(%{ $parse->{'parameter'} })) {
my $value = $parse->{'parameter'}->{"$name"}->{'value'};
if (! TBcheck_dbslot($name,
"experiment_template_instance_bindings", "name",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tberror("Illegal characters in parameter name: $name");
$donebad++;
}
if (defined($value) &&
! TBcheck_dbslot($value,
"experiment_template_instance_bindings", "value",
TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
tberror("Illegal characters in parameter value: $value");
$donebad++;
}
# DB records inserted below, once experiment is pre-loaded.
# Watch for unwanted parameters.
$parameters{$name} = $value
if (exists($parameters{$name}));
}
# User sees this error.
exit(1)
if ($donebad);
$experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
tbdie("Experiment $pid/$eid does not exist!");
}
#
# Generate a new template instance record.
# We will finish updating it later.
#
$instance = $template->NewInstance($eid, $dbuid);
$instance = Template::Instance->LookupByExptidx($experiment->idx());
if (!defined($instance)) {
fatal(-1, "Could not insert new experiment instance record!");
tbdie("Could not get instance record for experiment $pid/$eid!");
}
#
# At this point, we need to force a cleanup no matter how we exit.
# See the END block below.
#
$justexit = 0;
#
# Now insert the binding records for the instance so that the parser
# can get them.
#
if ($paramfile) {
foreach my $name (keys(%parameters)) {
my $value = $parameters{$name};
$instance->NewBinding($name, $value) == 0
or fatal(-1, "Error inserting binding into DB!");
}
}
elsif (defined($replay_instance)) {
#
# Bindings come from the replay instance (well, run), although the
# use can still have provided a parameter file (above) to override
# the values from the record.
#
my %replay_bindings = ();
$replay_run->BindingList(\%replay_bindings) == 0
or fatal(-1, "Error getting bindings from $replay_run!");
foreach my $name (keys(%replay_bindings)) {
my $value = $replay_bindings{$name};
# Insert into new instance now, for the parser to pick up.
$instance->NewBinding($name, $value) == 0
or fatal(-1, "Error inserting binding into DB!");
# These are used below for inserting the run bindings.
$parameters{$name} = $value
if (exists($parameters{$name}));
}
}
#
# We make a copy of either the underlying template experiment, or of the
# replay argument.
#
my $pid = $template->pid();
my $copyid = $template->pid() . "," . $template->eid();
#
# Ah, but if this is a replay, then the copyid is really a tag in a
# previous experiment.
#
if (defined($replay_instance)) {
$copyid = $replay_instance->exptidx() . ":" . $replay_run->start_tag();
}
# Need these for default run below.
$instance->BindingList(\%parameters) == 0
or fatal(-1, "Error getting bindings from $instance!");
#
# Go to the background now so we have a proper log of what happened.
......@@ -347,10 +209,14 @@ $SIG{TERM} = \&sighandler;
#
# Use the logonly option to audit so that we get a record mailed.
#
if (! ($foreground || $batchmode)) {
if ($instance->CreateLogFile("swapin", \$logname) < 0) {
fatal(-1, "Could not create logfile!");
}
if (! ($foreground || $experiment->batchmode())) {
# Cleanup
$experiment->CleanLogFiles() == 0
or fatal(-1, "Could not clean up logfiles!");
$logname = TBExptCreateLogFile($pid, $eid, "swapin");
TBExptSetLogFile($pid, $eid, $logname);
TBExptOpenLogFile($pid, $eid);
if (my $childpid = AuditStart(LIBAUDIT_DAEMON, $logname,
LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE)) {
......@@ -373,7 +239,7 @@ if (! ($foreground || $batchmode)) {
sleep(2);
}
if ($batchmode) {
if ($experiment->batchmode()) {
print("Experiment $pid/$eid has entered the batch system.\n".
"You will be notified when it is fully instantiated.\n")
if (! $quiet);
......@@ -417,163 +283,30 @@ if (! ($foreground || $batchmode)) {
TBdbfork();
}
#
# Build up arguments to batchexp. I do not want to bother with shell
# escapes, hence the list argument to system instead of a long string.
# Be sure to leave it this way, or perl will invoke a shell and that
# would be a really bad thing.
#
# Note special -x option.
#
my @arguments =
($batchexp, "-x", $template->eid(), "-y", $instance->idx(), "-q", "-f",
"-p", $pid, "-e", $eid, "-g", $template->gid(),
"-E", "'Experiment Template Instantiation $guid/$version' ",
"-c" , $copyid);
# All the other goo.
push(@arguments, "-i")
if (!$batchmode);
push(@arguments, "-s")
if (defined($options{"s"}));
push(@arguments, "-n")
if (defined($options{"n"}));
push(@arguments, ("-S", $options{"S"}))
if (defined($options{"S"}));
push(@arguments, ("-L", $options{"L"}))
if (defined($options{"L"}));
push(@arguments, ("-l", $options{"l"}))
if (defined($options{"l"}));
push(@arguments, ("-a", $options{"a"}))
if (defined($options{"a"}));
push(@arguments, ("-t", $options{"t"}))
if (defined($options{"t"}));
# Now invoke batchexp.
system(@arguments);
fatal($? >> 8, "Could not pre-instantiate the experiment")
if ($?);
# Need to kill the experiment if we fail after this point.
$exptcreated = 1;
#
# Now we can do this ...
#
if (defined($logname) && ! ($foreground || $batchmode)) {
TBExptSetLogFile($pid, $eid, $logname);
TBExptOpenLogFile($pid, $eid);
}
# Grab the experiment record for below.
my $experiment = Experiment->Lookup($pid, $eid);
if (!defined($experiment)) {
fatal(-1, "Experiment $pid/$eid could not be found after creation!");
}
$exptidx = $experiment->idx();
#
# Templates always get a DB; override the NS file setting of dpdb, and
# then call out to have it created.
#
if ($OPSDBSUPPORT) {
my %args = ();
$args{'dpdb'} = 1;
$experiment->Update(\%args) == 0
or fatal(-1, "Could not update experiment record!");
if (system("$dbcontrol addexpdb $pid $eid")) {
fatal(-1, "$dbcontrol addexpdb failed!");
}
$experiment->Refresh();
}
#
# Need the current archive tag so that we can mark the new instance