Commit 1de6af90 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Rework experiment start/end: Remove tbdoit and tbstopit. Silly names.

Change to startexp/endexp, which are almost scriptable now (can be
called directly in addition from the web page). Add front ends to
these for the web page (webstartexp and webendexp). These changes are
mostly support for batch mode.
parent f75131f0
...@@ -17,7 +17,7 @@ ac_help="$ac_help ...@@ -17,7 +17,7 @@ ac_help="$ac_help
ac_help="$ac_help ac_help="$ac_help
--with-WWWDEFS=name Specify WWW defintions ("default" default)" --with-WWWDEFS=name Specify WWW defintions ("default" default)"
ac_help="$ac_help ac_help="$ac_help
--with-TBOPSEMAIL=name Specify ops email (testbed-ops@fast default)" --with-TBOPSEMAIL=name Specify ops email (testbed-ops@flux default)"
ac_help="$ac_help ac_help="$ac_help
--with-LEDA Specify LEDA path (/usr/testbed default)" --with-LEDA Specify LEDA path (/usr/testbed default)"
ac_help="$ac_help ac_help="$ac_help
...@@ -820,7 +820,7 @@ if test "${with_TBOPSEMAIL+set}" = set; then ...@@ -820,7 +820,7 @@ if test "${with_TBOPSEMAIL+set}" = set; then
else else
TBOPSEMAIL="testbed-ops@fast.cs.utah.edu" TBOPSEMAIL="testbed-ops@flux.cs.utah.edu"
fi fi
...@@ -956,7 +956,8 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -956,7 +956,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \ tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
tbsetup/os_load tbsetup/os_setup tbsetup/mkprojdir tbsetup/power \ tbsetup/os_load tbsetup/os_setup tbsetup/mkprojdir tbsetup/power \
tbsetup/resetvlans tbsetup/rmacct-ctrl tbsetup/rmproj \ tbsetup/resetvlans tbsetup/rmacct-ctrl tbsetup/rmproj \
tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \ tbsetup/sched_reload \
tbsetup/startexp tbsetup/endexp tbsetup/webstartexp tbsetup/webendexp \
tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \ tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \ tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \ tbsetup/ir/handle_virt \
......
...@@ -45,14 +45,14 @@ AC_SUBST(WWWDEFS) ...@@ -45,14 +45,14 @@ AC_SUBST(WWWDEFS)
# Okay, I know this is improper usage of --with. Too bad. # Okay, I know this is improper usage of --with. Too bad.
# #
AC_ARG_WITH(TBOPSEMAIL, AC_ARG_WITH(TBOPSEMAIL,
[ --with-TBOPSEMAIL=name Specify ops email (testbed-ops@fast default)], [ --with-TBOPSEMAIL=name Specify ops email (testbed-ops@flux default)],
[ [
if test "$withval" = "yes"; then if test "$withval" = "yes"; then
AC_MSG_ERROR(Invalid TBOPSEMAIL - Must specify the value) AC_MSG_ERROR(Invalid TBOPSEMAIL - Must specify the value)
fi fi
TBOPSEMAIL="$withval" TBOPSEMAIL="$withval"
],[ ],[
TBOPSEMAIL="testbed-ops@fast.cs.utah.edu" TBOPSEMAIL="testbed-ops@flux.cs.utah.edu"
]) ])
TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`" TBOPSEMAIL="`echo $TBOPSEMAIL | sed -e 's/@/\\\@/'`"
AC_SUBST(TBOPSEMAIL) AC_SUBST(TBOPSEMAIL)
...@@ -115,7 +115,8 @@ outfiles="$outfiles Makeconf GNUmakefile \ ...@@ -115,7 +115,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \ tbsetup/GNUmakefile tbsetup/console_setup tbsetup/mkacct-ctrl \
tbsetup/os_load tbsetup/os_setup tbsetup/mkprojdir tbsetup/power \ tbsetup/os_load tbsetup/os_setup tbsetup/mkprojdir tbsetup/power \
tbsetup/resetvlans tbsetup/rmacct-ctrl tbsetup/rmproj \ tbsetup/resetvlans tbsetup/rmacct-ctrl tbsetup/rmproj \
tbsetup/sched_reload tbsetup/tbdoit tbsetup/tbstopit \ tbsetup/sched_reload \
tbsetup/startexp tbsetup/endexp tbsetup/webstartexp tbsetup/webendexp \
tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \ tbsetup/ir/GNUmakefile tbsetup/ir/postassign tbsetup/snmpit \
tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \ tbsetup/ir/assign_wrapper tbsetup/ns2ir/GNUmakefile \
tbsetup/ir/handle_virt \ tbsetup/ir/handle_virt \
......
...@@ -11,12 +11,13 @@ include $(OBJDIR)/Makeconf ...@@ -11,12 +11,13 @@ include $(OBJDIR)/Makeconf
SUBDIRS = checkpass ir ns2ir SUBDIRS = checkpass ir ns2ir
BIN_STUFF = power snmpit tbend tbrun tbprerun tbreport \ BIN_STUFF = power snmpit tbend tbrun tbprerun tbreport \
os_load savevlans os_load savevlans startexp endexp
SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup
LIBEXEC_STUFF = mkprojdir rmproj mkacct-ctrl rmacct-ctrl \ LIBEXEC_STUFF = mkprojdir rmproj mkacct-ctrl rmacct-ctrl \
os_setup mkexpdir tbdoit tbstopit console_setup os_setup mkexpdir console_setup \
webstartexp webendexp
LIB_STUFF = libtbsetup.pm LIB_STUFF = libtbsetup.pm
......
#!/usr/bin/perl -wT
use English;
use Getopt::Std;
#
# This gets invoked from the Web interface. Terminate an experiment.
# Most of the STDOUT prints are never seen since the web interface
# repeats only errors. My plan is make this script the front end to
# experiment termination and make tbend a backend program that no one
# uses.
#
# The -b (batch) argument is so that this script can be part of a batchmode
# that starts/ends experiments offline. In that case, we don't want to put
# it into the background and send email, but just want an exit status
# returned to the batch system.
#
sub usage()
{
print STDOUT "Usage: endexp [-b] <pid> <eid>\n";
exit(-1);
}
my $optlist = "b";
#
# Configure variables
#
my $TB = "@prefix@";
my $DBNAME = "@TBDBNAME@";
my $TBOPS = "@TBOPSEMAIL@";
my $tbdir = "$TB/bin/";
my $projroot = "/proj";
my $tbdata = "tbdata";
my $logname = 0;
my $batch = 0;
#
# Untaint the path
#
$ENV{'PATH'} = '/bin:/usr/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV != 2) {
usage();
}
my $pid = $ARGV[0];
my $eid = $ARGV[1];
if (defined($options{"b"})) {
$batch = $options{"b"};
}
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
$pid = $1;
}
if ($eid =~ /^([-\@\w.]+)$/) {
$eid = $1;
}
my $piddir = "$projroot/$pid";
my $expdir = "$piddir/exp";
my $eiddir = "$expdir/$eid";
#
# Set up for querying the database.
#
use Mysql;
my $DB = Mysql->connect("localhost", $DBNAME, "script", "none");
#
# We have to protect against trying to end an experiment that is currently
# in the process of being terminated. We use a timestamp for this purpose.
# If the timestamp is ever non-null, then something is wrong and we should
# not proceed.
#
# We also have to guard against trying to terminate an experiment that
# is still in the process of configuring. Its easiest to force the user
# to wait!
#
$query_result =
$DB->query("SELECT expt_terminating,expt_ready FROM experiments ".
"WHERE eid='$eid' and pid='$pid'");
if (! $query_result) {
fatal("DB Error getting experiment termination date for $pid/$eid\n");
}
if ($query_result->numrows < 1) {
print STDOUT "No such experiment $pid/$eid exists!\n";
exit(1);
}
@row = $query_result->fetchrow_array();
if (defined($row[0])) {
print STDOUT
"It appears that $pid/$eid started terminating at $row[0]\n".
"You will be notified via email when the experiment has been ".
"torn down\n";
exit(1);
}
if (! $row[1]) {
print STDOUT
"It appears that experiment $pid/$eid is still configuring.\n".
"The user that created the experiment will be notified via email\n".
"when it has been fully configured and is ready for use\n";
exit(1);
}
#
# Get some user information.
#
$query_result = $DB->query("SELECT uid,usr_name,usr_email,admin from users ".
"WHERE unix_uid='$EUID'");
if (! $query_result) {
fatal("DB Error getting user information for uid $EUID\n");
}
if ($query_result->numrows < 1) {
print STDOUT "Go Away! You do not exist in the Emulab Database.\n";
exit(1);
}
@row = $query_result->fetchrow_array();
$uid = $row[0];
$user_name = $row[1];
$user_email = $row[2];
$isadmin = $row[3];
#
# Verify that this person is allowed to end the experiment. Must be
# in the project membership table, or must be an admin type. Note that
# any script down the line has to do an admin check also.
#
if (! $isadmin) {
$query_result =
$DB->query("SELECT pid FROM proj_memb ".
"WHERE uid=\"$uid\" and pid=\"$pid\"");
if (! $query_result) {
fatal("DB Error getting project membership for uid $uid\n");
}
if ($query_result->numrows == 0) {
print STDOUT "Go Away! You are not a member of project $pid\n";
exit(1);
}
}
#
# Set the timestamp.
#
$stamp = `date '+20%y-%m-%d %H:%M:%S'`;
$query_result = $DB->query("UPDATE experiments SET expt_terminating='$stamp' ".
"WHERE eid='$eid' and pid='$pid'");
if (! $query_result) {
fatal("DB Error setting expt_terminating for experiment $pid/$eid\n");
}
#
# If not in batch mode, go into the background. Parent exits.
#
if (! $batch) {
if (background()) {
#
# Parent exits normally
#
print STDOUT
"Experiment $pid/$eid is now terminating\n".
"You will be notified via email when the experiment has been\n".
"torn down, and you can reuse the experiment name.\n";
exit(0);
}
}
print STDOUT "Running tbend with arguments: -nologfile $pid $eid\n";
if (system("$tbdir/tbend -nologfile $pid $eid") != 0) {
fatal("tbend failed!\n");
}
#
# Try to remove experiment directory. We allow for it not being there
# cause we often run the tb programs directly. We also allow for not
# having permission, in the case that an admin type is running this,
# in which case it won't be allowed cause of directory permissions. Thats
# okay since admin types should rarely end experiments in other projects.
#
if (chdir($expdir)) {
print STDOUT "Removing experiment directory: $eiddir\n";
system("rm -rf $eid");
}
else {
print STDOUT "Not able to remove experiment directory: $eiddir\n";
print STDOUT "Someone will need to do this by hand.\n";
}
#
# Done! Remove all trace from the DB.
#
$query_result = $DB->query("DELETE from experiments ".
"WHERE eid='$eid' and pid='$pid'");
if (! $query_result) {
fatal("DB Error deleting experiment record for $pid/$eid\n");
}
print STDOUT "Termination Success\n";
#
# In batch mode, just exit without sending email.
#
if ($batch) {
exit(0);
}
#
# Send email notification if not in batch mode.
#
open(MAIL, "| /usr/bin/mail ".
"-s \"TESTBED: Experiment $pid/$eid Terminated\" ".
"-c $TBOPS \"$user_name <$user_email>\" >/dev/null 2>&1")
or die "Cannot start mail program: $!";
print MAIL "Experiment `$eid' in project `$pid' has been terminated.\n";
print MAIL "You may now reuse `$eid' as an experiment name.\n\n";
print MAIL "Appended below is the output of the experiment teardown.\n";
print MAIL "If you have any questions or comments, please include the\n";
print MAIL "output below in your message to $TBOPS\n";
print MAIL "\n\n---------\n\n";
if (open(IN, "$logname")) {
while (<IN>) {
print MAIL "$_";
}
close(IN);
}
close(MAIL);
unlink("$logname");
exit 0;
sub fatal()
{
my($mesg) = $_[0];
print STDOUT $mesg;
#
# In batch mode, exit without sending the email.
#
if ($batch) {
exit(-1);
}
#
# Send a message to the testbed list. Append the logfile if it got
# that far.
#
open(MAIL, "| /usr/bin/mail ".
"-s \"TESTBED: Termination Failure $pid/$eid\" ".
"$TBOPS >/dev/null 2>&1")
or die "Cannot start mail program: $!";
print MAIL $mesg;
if (open(IN, "$logname")) {
print MAIL "\n\n---------\n\n";
while (<IN>) {
print MAIL "$_";
}
close(IN);
}
close(MAIL);
unlink("$logname");
exit(-1);
}
#
# Put ourselves into the background so that caller sees immediate response.
# Mail notification will happen later.
#
sub background()
{
$mypid = fork();
if ($mypid) {
return $mypid;
}
#
# We have to disconnect from the caller by redirecting both STDIN and
# STDOUT away from the pipe. Otherwise the caller (the web server) will
# continue to wait even though the parent has exited.
#
open(STDIN, "< /dev/null") or
die("opening /dev/null for STDIN: $!");
#
# Create a temporary name for a log file and untaint it.
#
$logname = `mktemp /tmp/end-$pid-$eid.XXXXXX`;
# Note different taint check (allow /).
if ($logname =~ /^([-\@\w.\/]+)$/) {
$logname = $1;
} else {
die "Bad data in $logname";
}
open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");
return 0;
}
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
use English; use English;
use Getopt::Std;
# #
# This gets invoked from the Web interface. CD into the proper directory # This gets invoked from the Web interface. CD into the proper directory
# and do the tb stuff. # and do the tb stuff.
# #
# usage: tbdoit <pid> <eid> <temp_nsfile> # The -b (batch) argument is so that this script can be part of a batchmode
# that starts/ends experiments offline. In that case, we don't want to put
# it into the background and send email, but just want an exit status
# returned to the batch system.
# #
# usage: startexp [-b] <pid> <eid> <nsfile>
#
sub usage()
{
print STDOUT "Usage: startexp [-b] <pid> <eid> <nsfile>\n";
exit(-1);
}
my $optlist = "b";
# #
# Configure variables # Configure variables
...@@ -19,6 +31,7 @@ my $tbdir = "$TB/bin/"; ...@@ -19,6 +31,7 @@ my $tbdir = "$TB/bin/";
my $projroot = "/proj"; my $projroot = "/proj";
my $tbdata = "tbdata"; my $tbdata = "tbdata";
my $cleanme = 0; my $cleanme = 0;
my $batch = 0;
# #
# For debugging all this goo. Leaves the experiment directory intact, # For debugging all this goo. Leaves the experiment directory intact,
...@@ -38,15 +51,22 @@ $ENV{'PATH'} = '/bin:/usr/bin'; ...@@ -38,15 +51,22 @@ $ENV{'PATH'} = '/bin:/usr/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
# #
# Check args. # Parse command arguments. Once we return from getopts, all that should
# # left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV != 3) { if (@ARGV != 3) {
print STDOUT "Usage: tbdoit <pid> <eid> <temp_nsfile>\n"; usage();
exit(-1);
} }
my $pid = $ARGV[0]; my $pid = $ARGV[0];
my $eid = $ARGV[1]; my $eid = $ARGV[1];
my $tempfile = $ARGV[2]; my $tempfile = $ARGV[2];
if (defined($options{"b"})) {
$batch = $options{"b"};
}
# #
# Untaint the arguments. # Untaint the arguments.
...@@ -68,7 +88,7 @@ my $eiddir = "$expdir/$eid"; ...@@ -68,7 +88,7 @@ my $eiddir = "$expdir/$eid";
my $nsfile = "$eid.ns"; my $nsfile = "$eid.ns";
my $irfile = "$eid.ir"; my $irfile = "$eid.ir";
my $repfile = "$eid.report"; my $repfile = "$eid.report";
my $tempcopy= "$tempfile.$$"; my $tempns = "$tempfile.$$";
# #
# Set up for querying the database. # Set up for querying the database.
...@@ -102,10 +122,10 @@ if ($row[0]) { ...@@ -102,10 +122,10 @@ if ($row[0]) {
} }
# #
# Figure out who is going to get the email! # Get some user information.
# #
$query_result = $query_result =
$DB->query("SELECT usr_name,usr_email from users ". $DB->query("SELECT uid,usr_name,usr_email from users ".
"WHERE unix_uid='$EUID'"); "WHERE unix_uid='$EUID'");
if (! $query_result) { if (! $query_result) {
...@@ -117,56 +137,53 @@ if ($query_result->numrows < 1) { ...@@ -117,56 +137,53 @@ if ($query_result->numrows < 1) {
} }
@row = $query_result->fetchrow_array(); @row = $query_result->fetchrow_array();
$user_name = $row[0]; $uid = $row[0];
$user_email = $row[1]; $user_name = $row[1];
$user_email = $row[2];
#
# Verify that this person is allowed to start the experiment. Must be
# in the project membership table.
#
$query_result =
$DB->query("SELECT pid FROM proj_memb ".
"WHERE uid=\"$uid\" and pid=\"$pid\"");
if (! $query_result) {
fatal("DB Error getting project membership for uid $uid\n");
}
if ($query_result->numrows == 0) {
print STDOUT "Go Away! You are not a member of project $pid\n";
exit(1);
}
# #
# Copy the nsfile from wherever the web server stuffed it into a temporary # Copy the nsfile from wherever the web server stuffed it into a temporary
# file. The web server is going to delete it once this script returns. # file. The web server is going to delete it once this script returns.
# #
if (system("/bin/cp", "$tempfile", "$tempcopy") != 0) { if (system("/bin/cp", "$tempfile", "$tempns") != 0) {
fatal("Could not copy $tempfile to $tempcopy: $!\n"); fatal("Could not copy $tempfile to $tempns: $!\n");
} }
chmod(0770, "$tempns");
# #
# The rest of this goes into the background so that the user sees # The rest of this goes into the background so that the user sees
# immediate response. We will send email later when the experiment # immediate response. We will send email later when the experiment
# is actually torn down. # is actually torn down.
# #
$mypid = fork(); if (! $batch) {
if ($mypid) { if (background()) {
# #
# Parent exits normally # Parent exits normally
# #
print STDOUT print STDOUT
"Experiment $pid/$eid is now configuring\n". "Experiment $pid/$eid is now configuring\n".
"You will be notified via email when the experiment is ready to use\n"; "You will be notified via email when the experiment is ".
exit(0); "ready to use\n";
} exit(0);
}
#
# We have to disconnect from the caller by redirecting both STDIN and
# STDOUT away from the pipe. Otherwise the caller (the web server) will
# continue to wait even though the parent has exited.
#
open(STDIN, "< /dev/null") or
die("opening /dev/null for STDIN: $!");
#
# Create a temporary name for a log file and untaint it.
#
$logname = `mktemp /tmp/$pid-$eid.XXXXXX`;
# Note different taint check (allow /).
if ($logname =~ /^([-\@\w.\/]+)$/) {
$logname = $1;
} else {
die "Bad data in $logname";
} }
open(STDERR, ">> $logname") or die("opening $logname for STDERR: $!");
open(STDOUT, ">> $logname") or die("opening $logname for STDOUT: $!");
# #
# Create a directory structure for the experiment in the project directory. # Create a directory structure for the experiment in the project directory.
# #
...@@ -176,16 +193,15 @@ if (system("$TB/libexec/mkexpdir $pid $eid") != 0) { ...@@ -176,16 +193,15 @@ if (system("$TB/libexec/mkexpdir $pid $eid") != 0) {
# #
# Copy the nsfile from wherever the web server stuffed it, into the # Copy the nsfile from wherever the web server stuffed it, into the
# experiment directory. # experiment directory. We leave the tempns file around till later.
# #
if (! chdir("$eiddir/$tbdata")) { if (! chdir("$eiddir/$tbdata")) {
fatal("Could not chdir to $tbdata in $eiddir: $!\n"); fatal("Could not chdir to $tbdata in $eiddir: $!\n");
} }
if (system("/bin/cp", "$tempcopy", "$nsfile") != 0) { if (system("/bin/cp", "$tempns", "$nsfile") != 0) {
fatal("Could not copy $tempcopy to $eiddir/$tbdata/$nsfile: $!\n"); fatal("Could not copy $tempns to $eiddir/$tbdata/$nsfile: $!\n");
} }
unlink("$tempcopy");
# #
# Run the various scripts. # Run the various scripts.
...@@ -231,6 +247,14 @@ $expt_expires = $row[2]; ...@@ -231,6 +247,14 @@ $expt_expires = $row[2];
print STDOUT "Setup Success\n"; print STDOUT "Setup Success\n";
#
# In batch mode, just exit without sending email. Remove tempns file!
#
if ($batch) {
unlink("$tempns");
exit(0);
}
# #
# Dump the report file and the log file to the user via email. # Dump the report file and the log file to the user via email.
# #
...@@ -285,6 +309,7 @@ if (open(IN, "$nsfile")) { ...@@ -285,6 +309,7 @@ if (open(IN, "$nsfile")) {
} }
close(MAIL); close(MAIL);
unlink("$tempns");
unlink("$logname"); unlink("$logname");
exit 0; exit 0;
...@@ -321,6 +346,14 @@ sub fatal() ...@@ -321,6 +346,14 @@ sub fatal()
system("/bin/mv", "-f", "$eid", "$save"); system("/bin/mv", "-f", "$eid", "$save");
} }
#
# In batch mode, exit. Make sure to delete tempns file.
#
if ($batch) {
unlink("$tempns");
exit(-1);
}
# #
# Send a message to the testbed list. Append the logfile if it got # Send a message to the testbed list. Append the logfile if it got
# that far. # that far.
...@@ -332,13 +365,14 @@ sub fatal() ...@@ -332,13 +365,14 @@ sub fatal()
print MAIL $mesg; print MAIL $mesg;
if (open(IN, "$nsfile")) { if (open(IN, "$tempns")) {
print MAIL "\n\n---------\n\n"; print MAIL "\n\n---------\n\n";
while (<IN>) { while (<IN>) {
print MAIL "$_"; print MAIL "$_";
} }
close(IN); close(IN);
unlink("$tempns");
} }
if (open(IN, "$logname")) { if (open(IN, "$logname")) {
...@@ -351,6 +385,7 @@ sub fatal() ...@@ -351,6 +385,7 @@ sub fatal()
} }
close(MAIL); close(MAIL);
unlink("$tempns");
unlink("$logname"); unlink("$logname");
exit(-1); exit(-1);
} }
...@@ -366,3 +401,40 @@ sub tbendit() ...@@ -366,3 +401,40 @@ sub tbendit()
print STDOUT "tbend failed!\n"; print STDOUT "tbend failed!\n";
} }