Commit 765de560 authored by Chad Barb's avatar Chad Barb

Added new feature 'Experiment Modify'.

Now available (to admins only for now) from the showexp page.

Warning! doing a modify which alters the topology will probably
require a "reboot all nodes" afterwards.
(There will be a checkbox soon in the modify experiment page.)

Adding/removing delay nodes seems to work fine without reboots, though.

Warning! If the new version of the experiment cannot be mapped
 (not enough nodes available, for instance) the experiment will be
 swapped out! This will get fixed later.

Prerun backs up the experiment topology, so using a bad NS
file doesn't result in experiment termination.

As part of this, added library functions to libdb to
delete, backup, and restore both virtual and physical experiment state.
parent b7665f30
......@@ -123,6 +123,12 @@ use Exporter;
TBSaveExpLogFiles TBExptWorkDir TBExptUserDir TBExptLogDir
TBExptDestroy TBIPtoNodeID TBNodeBootReset TBNodeStateWait
TBExptRemoveVirtualState TBExptBackupVirtualState
TBExptRestoreVirtualState
TBExptRemovePhysicalState TBExptBackupPhysicalState
TBExptRestorePhysicalState
TBDB_WIDEAREA_LOCALNODE
TBWideareaNodeID TBTipServers
......@@ -2123,6 +2129,160 @@ sub TBExptDestroy($$)
return 0;
}
#
# List of tables used for experiment removal/backup/restore.
#
@virtualTables = ("virt_nodes",
"virt_lans",
"virt_trafgens",
"virt_agents",
"virt_routes",
"virt_vtypes",
"nseconfigs",
"eventlist",
#"ipport_ranges",
"ipsubnets");
@physicalTables = ("delays",
"vlans",
"tunnels",
"v2pmap",
"ipport_ranges",
"linkdelays");
#
# Remove the virtual state of an experiment from the DB,
# returning the number of queries which didn't work.
#
sub TBExptRemoveVirtualState($$)
{
my ($pid, $eid) = @_;
my $errors = 0;
foreach $table (@virtualTables) {
DBQueryWarn("DELETE FROM $table WHERE pid='$pid' AND eid='$eid'")
or $errors++;
}
return $errors;
}
#
# Remove the physical state of an experiment from the DB,
# returning the number of queries which didn't work.
#
sub TBExptRemovePhysicalState($$)
{
my ($pid, $eid) = @_;
my $errors = 0;
foreach $table (@physicalTables) {
DBQueryWarn("DELETE FROM $table WHERE pid='$pid' AND eid='$eid'")
or $errors++;
}
return $errors;
}
#
# Backs up specified virtual state of pid/eid into directory in tmp.
#
sub TBExptBackupVirtualState($$$)
{
my ($pid, $eid, $backupid) = @_;
my $errors = 0;
my $backupDir = "/tmp/vstate-$pid-$eid-$backupid";
mkdir($backupDir, 0777)
or $errors++;
if (! $errors) {
chmod 0777, $backupDir
or $errors++;
}
if (! $errors) {
foreach $table (@virtualTables) {
DBQueryWarn("SELECT * FROM $table WHERE pid='$pid' AND eid='$eid' ".
"INTO OUTFILE '$backupDir/$table' ")
or $errors++;
}
}
return $errors;
}
#
# Backs up specified physical state of pid/eid into directory in tmp.
#
sub TBExptBackupPhysicalState($$$)
{
my ($pid, $eid, $backupid) = @_;
my $errors = 0;
my $backupDir = "/tmp/pstate-$pid-$eid-$backupid";
mkdir($backupDir, 0777)
or $errors++;
if (! $errors) {
chmod 0777, $backupDir
or $errors++;
}
if (! $errors) {
foreach $table (@physicalTables) {
DBQueryWarn("SELECT * FROM $table WHERE pid='$pid' AND eid='$eid' ".
"INTO OUTFILE '$backupDir/$table' ")
or $errors++;
}
}
return $errors;
}
#
# Restores backed up virtual state of pid/eid from directory in /tmp.
#
sub TBExptRestoreVirtualState($$$)
{
my ($pid, $eid, $backupid) = @_;
my $errors = 0;
my $backupDir = "/tmp/vstate-$pid-$eid-$backupid";
foreach $table (@virtualTables) {
DBQueryWarn("LOAD DATA INFILE '$backupDir/$table' INTO TABLE $table")
or $errors++;
unlink("$backupDir/$table");
}
rmdir("$backupDir");
return $errors;
}
#
# Restores backed up virtual state of pid/eid from directory in /tmp.
#
sub TBExptRestorePhysicalState($$$)
{
my ($pid, $eid, $backupid) = @_;
my $errors = 0;
my $backupDir = "/tmp/pstate-$pid-$eid-$backupid";
foreach $table (@physicalTables) {
DBQueryWarn("LOAD DATA INFILE '$backupDir/$table' INTO TABLE $table")
or $errors++;
unlink("$backupDir/$table");
}
rmdir("$backupDir");
return $errors;
}
#
# Get the control network IP for a node (underlying physical node!).
#
......
......@@ -10,11 +10,13 @@ use English;
use Getopt::Std;
#
# This gets invoked from the Web interface. Swap an experiment in or out.
# This gets invoked from the Web interface.
# Swap an experiment in, swap it out, restart or modify.
#
sub usage()
{
print STDOUT "Usage: swapexp <-s in | out | restart> <pid> <eid>\n";
print STDOUT "Usage: swapexp <-s in | out | restart | modify> ".
"<pid> <eid> [<nsfile>]\n";
exit(-1);
}
my $optlist = "s:";
......@@ -72,15 +74,14 @@ umask(0002);
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV != 2) {
usage();
}
my $pid = $ARGV[0];
my $eid = $ARGV[1];
if (defined($options{"s"})) {
$inout = $options{"s"};
if ($inout ne "out" && $inout ne "in" && $inout ne "restart") {
if ($inout ne "out" &&
$inout ne "in" &&
$inout ne "restart" &&
$inout ne "modify") {
usage();
}
}
......@@ -88,6 +89,27 @@ else {
usage();
}
if (@ARGV != (($inout eq "modify") ? 3 : 2)) {
usage();
}
my $pid = $ARGV[0];
my $eid = $ARGV[1];
my $nsfile;
if ($inout eq "modify") {
$nsfile = $ARGV[2];
#
# Untaint nsfile argument; Allow slash.
#
if ($nsfile =~ /^([-\w.\/]+)$/) {
$nsfile = $1;
} else {
die("Tainted nsfile name: $nsfile");
}
}
#
# Untaint the arguments.
#
......@@ -124,13 +146,13 @@ if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
}
#
# Verify that this person is allowed to end the experiment.
# Verify that this person can muck with the experiment.
# Note that any script down the line has to do an admin check also.
#
if ($UID && !TBAdmin($UID) &&
!TBExptAccessCheck($dbuid, $pid, $eid, TB_EXPT_DESTROY)) {
die("*** $0:\n".
" You do not have permission to swap this experiment!\n");
" You do not have permission to swap or modify this experiment!\n");
}
#
......@@ -175,11 +197,11 @@ if (defined($hashrow{'expt_locked'})) {
#
if ($isbatchexpt) {
die("*** $0:\n".
" Batch experiments cannot be swapped yet!");
" Batch experiments cannot be swapped or modified yet!");
}
#
# Okay, check state. We do not allow swapping to start when the
# Okay, check state. We do not allow modification or swapping to start when the
# experiment is in transition. A future task would be to allow this,
# but for now the experiment must be in one of a few states to proceed
#
......@@ -205,6 +227,8 @@ if ($inout eq "restart" && $estate ne EXPTSTATE_ACTIVE) {
" It appears that experiment $pid/$eid is not active!");
}
# if $inout eq "modify", either EXPTSTATE_ACTIVE -or- EXPTSTATE_SWAPPED is ok.
#
# Set the timestamp now, and unlock the experiments table.
#
......@@ -227,6 +251,9 @@ if ($inout eq "out") {
if ($inout eq "restart") {
$action = "restarted";
}
if ($inout eq "modify") {
$action = "modified";
}
#
# Get email address of the experiment head, which may be different than
......@@ -268,6 +295,15 @@ if ($inout ne "restart" && -e $repfile) {
unlink("$repfile");
}
#
# Rerun tbprerun if modifying.
#
if ($inout eq "modify") {
if (system("$tbdir/tbprerun -m $pid $eid $nsfile") != 0) {
fatal("tbprerun failed!\n");
}
}
#
# Sanity check states in case someone changes something.
#
......@@ -294,8 +330,27 @@ elsif ($inout eq "in") {
}
system("$tbdir/tbreport -b $pid $eid 2>&1 > $repfile");
}
elsif ($inout eq "modify") {
#
# If experiment is currently swapped out, no need to do an update
# after modifying it.
#
if ($estate eq EXPTSTATE_ACTIVE) {
print STDOUT "Running 'tbswap update' with arguments: $pid $eid\n";
if (system("$tbdir/tbswap update $pid $eid") != 0) {
fatal("tbswap update failed!\n");
}
$estate = ExpState($pid, $eid);
if ($estate ne EXPTSTATE_ACTIVE) {
fatal("Experiment is in the wrong state: $estate\n");
}
system("$tbdir/tbreport -b $pid $eid 2>&1 > $repfile");
}
}
else {
else { # $inout eq "restart" assumed.
print STDOUT "Running tbrestart with arguments: $pid $eid\n";
if (system("$tbdir/tbrestart $pid $eid") != 0) {
fatal("tbrestart failed!\n");
......@@ -328,6 +383,23 @@ if ($batch) {
exit(0);
}
#
# HACK! if successful, put new NS file in DB.
#
if ($inout eq "modify") {
$nsdata_string = `cat $nsfile`;
if (defined($nsdata_string)) {
$nsdata_string = DBQuoteSpecial($nsdata_string);
DBQueryWarn("delete from nsfiles WHERE eid='$eid' and pid='$pid'");
DBQueryWarn("insert into nsfiles (pid, eid, nsfile) ".
"VALUES('$pid', '$eid', $nsdata_string)");
} else {
print "Warning!! Couldn't read nsfile '$nsfile'!\n";
}
}
#
# Clear the log file so the web page stops spewing.
#
......@@ -359,6 +431,36 @@ sub fatal($)
print STDOUT $mesg;
#
# if $hosed == 1, we entirely terminate the experiment.
#
my $hosed = 0;
# If we're doing a modify,
# and tbprerun sent the experiment to "NEW",
# we're hosed.
if ($inout eq "modify" && ExpState($pid,$eid) eq EXPTSTATE_NEW) {
$hosed = 1;
}
if ($hosed) {
#
# Note: $estate is still set to the state which the experiment was in
# when we began.
#
if ($estate eq EXPTSTATE_ACTIVE) {
print "Running 'tbswap out' with arguments: $pid $eid\n";
if (system("$tbdir/tbswap out -force $pid $eid") != 0) {
print "tbswap out failed!\n";
}
}
print "Running tbend with arguments: -force $pid $eid\n";
if (system("$tbdir/tbend -force $pid $eid") != 0) {
print "tbend failed!\n";
}
}
#
# Kill this for convenience later.
#
......@@ -394,6 +496,19 @@ sub fatal($)
"Cc: $TBOPS",
($logname));
if ($hosed) {
#
# Copy off the workdir to the user directory, Then back up both of
# them for post-mortem debugging.
#
system("/bin/cp -Rfp $workdir/ $userdir/tbdata");
system("/bin/rm -rf ${workdir}-failed");
system("/bin/mv -f $workdir ${workdir}-failed");
system("/bin/rm -rf ${userdir}-failed");
system("/bin/mv -f $userdir ${userdir}-failed");
TBExptDestroy($pid, $eid);
}
exit(-1);
}
......@@ -84,26 +84,7 @@ if (! SetExpState($pid, $eid, EXPTSTATE_TERMINATING)) {
}
print "Clearing out virtual state.\n";
DBQueryWarn("DELETE from virt_nodes where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from virt_lans where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from virt_trafgens where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from virt_agents where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from virt_routes where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from virt_vtypes where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from nseconfigs where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from eventlist where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from ipsubnets where pid='$pid' and eid='$eid'") or
$errors++;
DBQueryWarn("DELETE from ipport_ranges where pid='$pid' and eid='$eid'") or
$errors++;
$errors += TBExptRemoveVirtualState( $pid, $eid );
print "Removing visualization data...\n";
system("prerender -r -t $pid $eid");
......
......@@ -18,10 +18,11 @@ use English;
#
sub usage()
{
print STDERR "Usage: $0 [-force] pid eid nsfile\n";
print STDERR "Usage: $0 [-force | -m] pid eid nsfile\n";
exit(-1);
}
my $force = 0;
my $modify = 0;
my $state;
#
......@@ -55,6 +56,11 @@ if ($ARGV[0] eq "-force") {
$force = 1;
shift;
}
if ($ARGV[0] eq "-m") {
$modify = 1;
shift;
}
if (@ARGV != 3) {
usage();
}
......@@ -72,32 +78,63 @@ if (! ($state = ExpState($pid, $eid))) {
die("*** $0:\n".
" No such experiment $pid/$eid\n");
}
if (!$force && $state ne EXPTSTATE_NEW) {
die("*** $0:\n".
" Experiment is not in the proper state: $state\n");
}
if (! SetExpState($pid, $eid, EXPTSTATE_PRERUN)) {
die("*** $0:\n".
" Failed to set intermediate experiment state.\n");
if (! $modify) {
if (!$force && $state ne EXPTSTATE_NEW) {
die("*** $0:\n".
" Experiment is not in the proper state: $state\n");
}
if (! SetExpState($pid, $eid, EXPTSTATE_PRERUN)) {
die("*** $0:\n".
" Failed to set intermediate experiment state.\n");
}
} else {
if ($state ne EXPTSTATE_ACTIVE &&
$state ne EXPTSTATE_SWAPPED ) {
die("*** $0:\n".
" Experiment is in transition state: $state\n");
}
}
#
# Cleanup if something goes wrong.
#
sub cleanup {
if ($modify) {
print STDERR "Recovering from errors.\n";
print "Restoring old experiment state ... " . TBTimeStamp() . "\n";
if (0 == TBExptRestoreVirtualState($pid, $eid, $$)) {
print "Restoration done! " . TBTimeStamp() . "\n";
return;
} else {
print "Restoration failed... aborting! " . TBTimeStamp() . "\n";
# Fall through to full cleanup...
}
}
print STDERR "Cleaning up after errors.\n";
DBQueryWarn("DELETE from virt_nodes where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from virt_lans where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from virt_trafgens where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from virt_agents where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from virt_routes where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from nseconfigs where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from eventlist where pid='$pid' and eid='$eid'");
DBQueryWarn("DELETE from ipsubnets where pid='$pid' and eid='$eid'");
print "Removing experiment state ... " . TBTimeStamp() . "\n";
TBExptRemoveVirtualState($pid, $eid );
print "Removal done! " . TBTimeStamp() . "\n";
#
# If modify fails, and recovery fails,
# going to "NEW" tells swapexp to fully terminate the experiment.
#
SetExpState($pid, $eid, EXPTSTATE_NEW);
system("prerender -r $pid $eid");
}
if ($modify) {
print "Backing up old experiment state ... " . TBTimeStamp() . "\n";
if (TBExptBackupVirtualState($pid, $eid, $$)) {
die("*** $0:\n".
" Could not backup experiment state!\n");
}
TBExptRemoveVirtualState($pid, $eid);
print "Backup done! " . TBTimeStamp() . "\n";
}
# This setups virt_nodes, virt_names including all IP address calculation
# and tb-* handling.
print "Running parser ... " . TBTimeStamp() . "\n";
......@@ -124,10 +161,12 @@ if (system("staticroutes $pid $eid")) {
TBDebugTimeStamp("routes finished");
print "Static routing done! " . TBTimeStamp() . "\n";
if (!SetExpState($pid, $eid, EXPTSTATE_SWAPPED)) {
cleanup();
die("*** $0:\n".
" Failed to set experiment state!\n");
if (! $modify) {
if (!SetExpState($pid, $eid, EXPTSTATE_SWAPPED)) {
cleanup();
die("*** $0:\n".
" Failed to set experiment state!\n");
}
}
TBDebugTimeStamp("tbprerun finished");
......
......@@ -59,7 +59,9 @@ $| = 1;
my $restart = 0;
my $force = 0;
my $errors = 0;
my $retry = 0;
my $state;
my $os_setup_pid;
......@@ -405,6 +407,11 @@ sub doSwapout($) {
$swapout_errors = 1;
}
TBDebugTimeStamp("nfree finished");
#
# Since this is an actual swapout,
# reset our count of swap out nag emails sent.
#
DBQueryWarn("update experiments set swap_requests='' ".
"where eid='$eid' and pid='$pid'");
} else {
......@@ -491,18 +498,7 @@ sub doSwapout($) {
#
print STDERR "Resetting DB.\n";
DBQueryWarn("DELETE from delays where pid='$pid' and eid='$eid'")
or $swapout_errors++;
DBQueryWarn("DELETE from vlans where pid='$pid' and eid='$eid'")
or $swapout_errors++;
DBQueryWarn("DELETE from tunnels where pid='$pid' and eid='$eid'")
or $swapout_errors++;
DBQueryWarn("DELETE from v2pmap where pid='$pid' and eid='$eid'")
or $swapout_errors++;
DBQueryWarn("DELETE from ipport_ranges where pid='$pid' and eid='$eid'")
or $swapout_errors++;
DBQueryWarn("DELETE from linkdelays where pid='$pid' and eid='$eid'")
or $swapout_errors++;
TBExptRemovePhysicalState( $pid, $eid );
return $swapout_errors;
}
......
<?php
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
include("defs.php3");
include("showstuff.php3");
$parser = "$TB/libexec/ns2ir/parse-ns";
#
# Standard Testbed Header
#
PAGEHEADER("Modify Experiment");
#
# Only known and logged in users can modify experiments.
#
$uid = GETLOGIN();
LOGGEDINORDIE($uid);
$isadmin = ISADMIN($uid);
#
# Verify page arguments.
#
if (!isset($pid) ||
strcmp($pid, "") == 0) {
USERERROR("You must provide a Project ID.", 1);
}
if (!isset($eid) ||
strcmp($eid, "") == 0) {
USERERROR("You must provide an Experiment ID.", 1);
}
#
# Be paranoid.
#
$pid = addslashes($pid);
$eid = addslashes($eid);
#
# Check to make sure this is a valid PID/EID tuple.
#
$query_result =
DBQueryFatal("SELECT * FROM experiments WHERE ".
"eid='$eid' and pid='$pid'");
if (mysql_num_rows($query_result) == 0) {
USERERROR("The experiment $eid is not a valid experiment ".
"in project $pid.", 1);
}
$expstate = TBExptState($pid, $eid);
if (! TBExptAccessCheck($uid, $pid, $eid, $TB_EXPT_MODIFY ) ) {
USERERROR("You do not have permission to modify this experiment.", 1);
}
if (strcmp($expstate, $TB_EXPTSTATE_ACTIVE) &&
strcmp($expstate, $TB_EXPTSTATE_SWAPPED)) {
USERERROR("You cannot modify an experiment in transition.", 1);
}
if (! isset($go)) {
echo "<form action='modifyexp.php3' method='post'>";
echo "<textarea cols='100' rows='40' name='nsdata'>";
$query_result =
DBQueryFatal("SELECT nsfile from nsfiles where pid='$pid' and eid='$eid'");
if (mysql_num_rows($query_result)) {
$row = mysql_fetch_array($query_result);
$nsfile = stripslashes($row[nsfile]);
echo "$nsfile";
} else {
echo "# There was no stored NS file for $pid/$eid.\n";
}
echo "</textarea>";
echo "<br />";
# echo "<input type='checkbox' name='restart' value='1'>Restart experiment</input>";
echo "<br />";
echo "<input type='hidden' name='pid' value='$pid' />";
echo "<input type='hidden' name='eid' value='$eid' />";
echo "<button name='go'>Modify</button>";
echo "</form>\n";
} else {
if (! isset($nsdata)) {
USERERROR("NSdata CGI variable missing (How'd that happen?)",1);
}
# echo "<pre>$nsdata</pre>\n";
# echo "<h1>2</h1>\n";
$nsfile = tempnam("/tmp", "$pid-$eid.nsfile.");
if (! ($fp = fopen($nsfile, "w"))) {
TBERROR("Could not create temporary file $nsfile", 1);
}
$nsdata_string = urldecode($nsdata);
fwrite($fp, $nsdata_string);
fclose($fp);
chmod($nsfile, 0666);
# echo "<pre>$nsdata_string</pre>\n";
if (system("$parser -n -a $nsfile") != 0) {
USERERROR("Modified NS file contains syntax errors; aborting.", 1);
}
#
# Get exp group.
#
TBExptGroup($pid,$eid,$gid);
#
# We need the unix gid for the project for running the scripts below.
#
TBGroupUnixInfo($pid, $gid, $unix_gid, $unix_name);
# echo "<pre>".
# "$TBSUEXEC_PATH $uid $unix_gid ".
# "webswapexp -s modify $pid $eid $nsfile".
# "</pre>\n";
#
# Avoid SIGPROF in child.
#
set_time_limit(0);
$output = array();
$retval = 0;
$result = exec("$TBSUEXEC_PATH $uid $unix_gid ".
"webswapexp -s modify $pid $eid $nsfile",
$output, $retval);
if ($retval) {
echo "<br /><br />\n".
"<h2>Modify failure($retval): Output as follows:</h2>\n".
"<br />".
"<xmp>\n";
for ($i = 0; $i < count($output); $i++) {
echo "$output[$i]\n";
}
echo "</xmp>\n";
PAGEFOOTER();
die("");
} else {
#
# Exit status 0 means the experiment is swapping, or will be.
#
echo "<br /><br />";
echo "<font size=+1>
<p>Experiment
<a href='showexp.php3?pid=$pid&eid=$eid'>$eid</a>
in project <A href='showproject.php3?pid=$pid'>$pid</A>