Commit b674bc7b authored by Leigh Stoller's avatar Leigh Stoller

First cut at template checkout and commit from a checkout. The interface

described is the one exported to ops via the XMLRPC interface. This is
just playing aroundl no doubt this stuff is going to change.

* template_checkout guid/vers

  Checkout a copy of the template to the current working directory.

* template_commit

  Modify the previous template checkout, using the nsfile contained in
  the tbdata directory (subdir of the current directory). In other words,
  the current template is modified, creating a new template in the
  current working directory (the current directory refers to the new
  template).

  The datastore subdir is imported into the new template, but that is
  the only directory that is imported at present. Might change that.

So this sounds much cooler then it really is. Why?

* This only works from ops.

* The "current directory" must be one of the standard approved directories
  (/proj, /users, /groups).

* Cause, boss reads and writes that directory via NFS, as told to it
  by the xmlrpc client.

At some point in the future it would be nice to support something
fancier, using a custom transport, but lets see how this goes.
parent edfec844
......@@ -27,7 +27,7 @@ BIN_STUFF = power snmpit tbend tbprerun tbreport \
template_exprun template_delete template_metadata \
template_export template_control template_commit \
template_analyze template_linkgraph template_instantiate \
template_revise
template_revise template_checkout
SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
batch_daemon exports_setup reload_daemon sched_reserve \
......
......@@ -184,6 +184,20 @@ sub LookupByPidEid($$$)
return Template->Lookup($guid, $vers);
}
#
# Return the underlying experiment object for the template itself.
#
sub GetExperiment($)
{
my ($self) = @_;
# Must be a real reference.
return undef
if (! ref($self));
return Experiment->LookupByIndex($self->exptidx());
}
#
# Refresh a template instance by reloading from the DB.
#
......@@ -1793,6 +1807,20 @@ sub GetArchive($)
return Archive->Lookup($self->archive_idx());
}
#
# Get the current archive tag for the template from the underlying experiment.
#
sub ArchiveTag($)
{
my ($self) = @_;
# Must be a real reference.
return undef
if (! ref($self));
return $self->GetExperiment()->archive_tag();
}
#
# Return project object for the template.
#
......@@ -2197,6 +2225,7 @@ sub Experiment($)
return Experiment->LookupByIndex($self->exptidx());
}
sub GetExperiment($) { return $_[0]->Experiment(); }
#
# Delete a template instance record.
......
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use strict;
use Getopt::Std;
use POSIX qw(isatty setsid);
use POSIX qw(strftime);
use Errno qw(EDQUOT);
use XML::Simple;
use Data::Dumper;
#
# Checkout a copy of a template. Do not be confused by the name; this will
# make a copy of a template in the specified directory and drop a little
# cookie so that we can tell later what template it refers to. It only works
# when called from ops (the directory specified resides on ops, there is
# no remote checkout at this time). Later, you can commit the template, which
# creates a new template (template_modify) and changes the cookie so that the
# current directory refers to the new template.
#
# Eventually support remote checkout via the web interface.
#
# There is no "update" operation, as with "cvs update" so you can easily
# muck up things by having two directories that refer to the same template!
#
# $status < 0 - Fatal error. Something went wrong we did not expect.
# $status = 0 - Everything okay.
# $status > 0 - Expected error. User not allowed for some reason.
#
sub usage()
{
print(STDERR
"Usage: template_checkout [-q] -f <path> <guid/vers>\n".
"switches and arguments:\n".
"-q - be less chatty\n".
"-f - path to checkout directory on ops (via NFS).\n".
"<guid/vers> - GUID and version to swapin\n");
exit(-1);
}
my $optlist = "qdf:";
my %options = ();
my $quiet = 0;
my $debug = 0;
my $directory;
my $guid;
my $version;
#
# Configure variables
#
my $TB = "@prefix@";
my $EVENTSYS = @EVENTSYS@;
my $TBOPS = "@TBOPSEMAIL@";
my $TBLOGS = "@TBLOGSEMAIL@";
my $TBDOCBASE = "@TBDOCBASE@";
my $TBBASE = "@TBBASE@";
my $CONTROL = "@USERNODE@";
my $checkquota = "$TB/sbin/checkquota";
my $TAR = "/usr/bin/tar";
# Protos
sub ParseArgs();
sub fatal($);
sub cleanup();
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use libtblog;
use Template;
use Experiment;
use Archive;
#
# Turn off line buffering on output
#
$| = 1;
#
# Set umask for start/swap. We want other members in the project to be
# able to swap/end experiments, so the log and intermediate files need
# to be 664 since some are opened for append.
#
umask(0002);
#
# Untaint the path
#
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Verify user and get his DB uid.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
tbdie("You ($UID) do not exist!");
}
my $user_name = $this_user->name();
my $user_email = $this_user->email();
my $user_uid = $this_user->uid();
#
# Before doing anything else, check for overquota ... lets not waste
# our time. Make sure user sees the error by exiting with 1.
#
if (system("$checkquota $user_uid") != 0) {
tberror("You are over your disk quota on $CONTROL; ".
"please login there and cleanup!");
exit(1);
}
# Now parse arguments.
ParseArgs();
# A little cookie file in that indicates what template it refers to.
my $cookie = "$directory/.template";
if (-e $cookie) {
print STDERR "$directory already contains a checkout!\n";
print STDERR "There is no 'update' operation yet; delete or move it.\n";
exit(1);
}
my $template = Template->Lookup($guid, $version);
if (!defined($template)) {
tbdie("Experiment template $guid/$version does not exist!");
}
# Grab the archive tag from the template itself, for an archive checkout.
my $archive_tag = $template->ArchiveTag();
if (!defined($archive_tag) || $archive_tag eq "") {
tbdie("Could not determine archive_tag for $template!");
}
# And do the checkout to the directory specified.
my $archive = $template->GetArchive();
if (!defined($archive)) {
tbdie("Could not get archive object for $template!");
}
# This toplevel "exp" directory has turned out to be a pain in the ass.
# What was I thinking?
if (-e "$directory/exp" && ! -l "$directory/exp") {
print STDERR "Please remove '$directory/exp' first!\n";
exit(1);
}
if (! -e "$directory/exp") {
system("cd $directory; ln -s . exp") == 0 or
tbdie("Could not create exp symlink in $directory!");
}
if ($archive->Checkout($directory, $template->exptidx(), $archive_tag) != 0) {
fatal("Could not checkout $archive_tag from $archive to $directory!");
}
unlink("$directory/exp");
#
# If the checkout succeeds then drop a little cookie file in that indicates
# what template it refers to.
#
unlink($cookie)
if (-e $cookie);
open(COOKIE, "> $cookie") or
fatal("Could not create $cookie\n");
print COOKIE "# Do not remove this file!\n";
print COOKIE "GUID: $guid/$version\n";
print COOKIE "TIME: " . time() . "\n";
close(COOKIE);
exit(0);
#
# Parse command arguments. Once we return from getopts, all that are
# left are the required arguments.
#
sub ParseArgs()
{
if (! getopts($optlist, \%options)) {
usage();
}
#
# Parse out guid/vers argument.
#
usage()
if (@ARGV != 1);
my $tmp = shift(@ARGV);
if ($tmp =~ /^([\w]*)\/([\d]*)$/) {
$guid = $1;
$version = $2;
}
else {
tbdie("Bad data in argument: $tmp");
}
if (defined($options{"f"})) {
$directory = $options{"f"};
# The Archive library has a nice routine to validate this path.
if (Archive::ValidatePath(\$directory) != 0) {
tberror("Invalid path for checkout $directory");
exit(1);
}
}
else {
tbdie("Must supply a path argument with the -f option!");
}
if (defined($options{"q"})) {
$quiet = 1;
}
if (defined($options{"d"})) {
$debug = 2;
}
}
#
# Cleanup the mess.
#
sub cleanup()
{
}
sub fatal($)
{
my ($msg) = @_;
tberror $msg;
cleanup();
exit(-1);
}
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006 University of Utah and the Flux Group.
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
......@@ -28,7 +28,8 @@ use Data::Dumper;
sub usage()
{
print(STDERR
"Usage: template_commit [-q] [-e eid] <guid/vers>\n".
"Usage: template_commit [-q] -f <path>\n".
" template_commit [-q] [-e eid] <guid/vers>\n".
" template_commit [-q] -p pid -e eid\n".
"switches and arguments:\n".
"-q - be less chatty\n".
......@@ -37,10 +38,11 @@ sub usage()
"<guid/vers> - GUID and version to swapin\n");
exit(-1);
}
my $optlist = "qe:dp:";
my $optlist = "qe:dp:f:";
my %options = ();
my $quiet = 0;
my $debug = 0;
my $frompath;
my $eid;
my $pid;
my $guid;
......@@ -60,11 +62,12 @@ my $checkquota = "$TB/sbin/checkquota";
my $modify = "$TB/bin/template_create";
my $archcontrol = "$TB/bin/archive_control";
my $TAR = "/usr/bin/tar";
my $RSYNC = "/usr/local/bin/rsync";
# Locals
my $dbuid;
my $template;
my $child_template;
my $cookie;
# Protos
sub ParseArgs();
......@@ -73,6 +76,7 @@ sub sighandler($);
sub cleanup();
sub CommitFromInstance();
sub CommitFromTemplate();
sub CommitFromCheckout();
#
# Testbed Support libraries
......@@ -106,8 +110,9 @@ delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Verify user and get his DB uid.
#
if (! UNIX2DBUID($UID, \$dbuid)) {
tbdie("You do not exist in the Emulab Database!");
my $this_user = User->ThisUser();
if (! defined($this_user)) {
tbdie("You ($UID) do not exist!");
}
# Now parse arguments.
......@@ -133,6 +138,28 @@ if (defined($pid)) {
$version = $template->vers();
}
else {
if (defined($frompath)) {
#
# Figure out what template from cookie file.
#
$cookie = "$frompath/.template";
if (open(COOKIE, $cookie)) {
while (<COOKIE>) {
if ($_ =~ /^GUID:\s*([\w]*)\/([\d]*)$/) {
$guid = $1;
$version = $2;
}
}
close(COOKIE);
}
else {
tbdie("Could not open $cookie!");
}
if (!(defined($guid) && defined($version))) {
tbdie("Could not parse $cookie!");
}
}
$template = Template->Lookup($guid, $version);
if (!defined($template)) {
......@@ -140,7 +167,7 @@ else {
}
}
if (! $template->AccessCheck($dbuid, TB_EXPT_MODIFY)) {
if (! $template->AccessCheck($this_user, TB_EXPT_MODIFY)) {
tberror("You do not have permission to commit template $guid/$version");
exit(1);
}
......@@ -150,7 +177,10 @@ if (! $template->AccessCheck($dbuid, TB_EXPT_MODIFY)) {
#
$SIG{TERM} = \&sighandler;
if (defined($eid)) {
if (defined($frompath)) {
CommitFromCheckout();
}
elsif (defined($eid)) {
CommitFromInstance();
}
else {
......@@ -158,6 +188,68 @@ else {
}
exit(0);
#
# Commit from a checkout.
#
sub CommitFromCheckout()
{
my $pid = $template->pid();
my $tid = $template->tid();
my $gid = $template->gid();
my $nsfile = "$frompath/tbdata/nsfile.ns";
my $optarg = ($quiet ? "-q" : "");
#
# The NS file is taken from the checkout.
#
fatal(1, "There is no NS file in $frompath/tbdata!")
if (! -e $nsfile);
#
# Template modify to give us a new version. Giving it $frompath causes
# the datastore to be imported from the directory instead of the parent
# template.
#
system("$modify -m $guid/$version -w $optarg -g $gid -f $frompath ".
"$pid $tid $nsfile");
if ($?) {
fatal($? >> 8, "Failed to modify $template!");
}
# Pick up changes to child guid/vers.
$template->Refresh();
my $child_guid = $template->child_guid();
my $child_vers = $template->child_vers();
$child_template = Template->Lookup($child_guid, $child_vers);
if (!defined($child_template)) {
fatal(-1, "Lookup of child template failed!");
}
#
# Now we want to copy back to the $frompath to sync it up.
#
my $newpath = $child_template->path();
if (system("$RSYNC -rtgoDlz ${newpath}/ $frompath") != 0) {
fatal(-1, "Could not copy template back to checkout");
}
# Update the cookie so that the checkout refers to the new template
# not the original template.
unlink($cookie)
if (-e $cookie);
open(COOKIE, "> $cookie") or
fatal(-1, "Could not create $cookie\n");
print COOKIE "# Do not remove this file!\n";
print COOKIE "GUID: $child_guid/$child_vers\n";
print COOKIE "TIME: " . time() . "\n";
close(COOKIE);
return 0;
}
#
# Commit a template. This is basically a template modify operation.
#
......@@ -293,6 +385,14 @@ sub ParseArgs()
tbdie("Improper project name (pid)!");
}
}
elsif (defined($options{"f"})) {
$frompath = $options{"f"};
# The Archive library has a nice routine to validate this path.
if (Archive::ValidatePath(\$frompath) != 0) {
tbdie("Invalid path $frompath");
}
}
else {
usage();
}
......
......@@ -39,10 +39,11 @@ sub usage()
"<input> - Input file for experiment.\n");
exit(-1);
}
my $optlist = "qwE:g:m:";
my $optlist = "qwE:g:m:f:";
my $quiet = 0;
my $waitmode = 0;
my $modify = 0;
my $frompath;
my $description;
my $pid;
my $tid;
......@@ -245,7 +246,7 @@ if (my $childpid =
if (! $quiet);
exit(0);
}
print("Waiting for template $pid/$tid to be created\n")
print("Waiting for template $pid/$tid to be created. Please be patient!\n")
if (! $quiet);
# Give child a chance to run.
......@@ -379,9 +380,15 @@ if ($modify) {
print "Committing archive before copying data store\n";
libArchive::TBCommitExperimentArchive($pid, $eid, "template_modify")
>= 0 or fatal(-1, "Failed to commit experiment archive!");
$template->CopyDataStore($parent_template, $user_uid) == 0
or fatal(-1, "Failed to copy data store");
if (defined($frompath)) {
$template->ImportDataStore("$frompath/datastore") == 0
or fatal(-1, "Failed to import data store");
}
else {
$template->CopyDataStore($parent_template, $user_uid) == 0
or fatal(-1, "Failed to copy data store");
}
# and tell the archive library about the above files.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
......@@ -549,6 +556,15 @@ sub ParseArgs()
$gid = $pid;
}
if (defined($options{"f"})) {
$frompath = $options{"f"};
# The Archive library has a nice routine to validate this path.
if (Archive::ValidatePath(\$frompath) != 0) {
tbdie("Invalid path $frompath");
}
}
#
# Parent pointer, for modify. We always create a new template point
# it to the parent.
......
......@@ -33,7 +33,8 @@ SYMLINKS = node_admin node_reboot os_load create_image node_list \
readycount nscheck startexp batchexp startexp swapexp endexp \
modexp expinfo node_avail tbuisp expwait template_commit \
template_export template_swapin template_swapout \
template_stoprun template_instantiate template_startrun
template_stoprun template_instantiate template_startrun \
template_checkout
#
# Force dependencies on the scripts so that they will be rerun through
......
......@@ -4615,6 +4615,25 @@ class template:
argstr += " "
argstr += str(argdict["guid"])
pass
elif argdict.has_key("path"):
try:
path = nfspath(argdict["path"]) # Scrub the path
# Make sure the path is accessible,
if not os.access(path, os.X_OK):
raise OSError(errno.EPERM, "Path is not accessible", path)
pass
except OSError, e:
retval = EmulabResponse(RESPONSE_ERROR,
value=e,
output=(e.strerror +
": " + e.filename))
return
argstr = "-q -f "
argstr += escapeshellarg(argdict["path"]);
pass
(exitval, output) = runcommand(TBDIR +
"/bin/template_commit " + argstr)
......@@ -4623,6 +4642,56 @@ class template:
return EmulabResponse(RESPONSE_SUCCESS, output=output)
#
# Checkout a template.
#
def checkout(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")
if self.readonly:
return EmulabResponse(RESPONSE_FORBIDDEN,
output="Insufficient privledge to invoke method")
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
argerror = CheckRequiredArgs(argdict, ("guid", "path"))
if (argerror):
return argerror
if not (re.match("^[-\w\/]*$", argdict["guid"])):
return EmulabResponse(RESPONSE_BADARGS,
output="Improperly formed arguments")
try:
path = nfspath(argdict["path"]) # Scrub the path
# Make sure the path is accessible,
if not os.access(path, os.X_OK):
raise OSError(errno.EPERM, "Path is not accessible", path)
pass
except OSError, e:
retval = EmulabResponse(RESPONSE_ERROR,
value=e,
output=(e.strerror + ": " + e.filename))
pass
argstr = "-q -f " + escapeshellarg(argdict["path"])
argstr += " " + escapeshellarg(argdict["guid"])
(exitval, output) = runcommand(TBDIR +
"/bin/template_checkout " + argstr)
if exitval:
return EmulabResponse(RESPONSE_ERROR, exitval >> 8, output=output)
return EmulabResponse(RESPONSE_SUCCESS, output=output)
#
# Export a template.
#
......
......@@ -131,6 +131,8 @@ API = {
"help" : "Commit changes to template (modify)" },
"template_export" : { "func" : "template_export",
"help" : "Export template record" },
"template_checkout" : { "func" : "template_checkout",
"help" : "Checkout a template" },
"template_instantiate": { "func" : "template_instantiate",
"help" : "Instantiate a template" },
"template_swapin": { "func" : "template_swapin",
......@@ -1825,7 +1827,10 @@ class template_commit:
pass
# Try to infer the template guid/vers from the current path.
if (len(req_args) == 0 and pid == None and eid == None):
if os.access(".template", os.R_OK):
params["path"] = os.getcwd()
pass
elif (len(req_args) == 0 and pid == None and eid == None):
guid = infer_guid(os.getcwd())
if guid != None:
params["guid"] = guid
......@@ -1848,13 +1853,15 @@ class template_commit:
else:
self.usage();
return -1
print "Committing new template; please be (very) patient!"
rval,response = do_method("template", "template_commit", params);
return rval;
def usage(self):
print "template_commit [-e eid] [<guid/vers>]"
print "template_commit -p pid -e eid"
print "template_commit [-w] [-e eid] [<guid/vers>]"
print "template_commit [-w] -p pid -e eid"
print "where:";
print " -w - Wait for template to finish commit";
print " -e - Commit from specific template instance (eid)"
......@@ -1925,6 +1932,40 @@ class template_export:
return
pass
class template_checkout:
def __init__(self, argv=None):
self.argv = argv;
return
def apply(self):
try:
opts, req_args = getopt.getopt(self.argv, "d", [ "help" ]);
pass
except getopt.error, e:
print e.args[0]
self.usage();
return -1;
if (len(req_args) == 0):
self.usage()
return -1
params = {}
params["guid"] = req_args[0]
params["path"] = os.getcwd()
rval,response = do_method("template", "checkout", params);
return rval;
def usage(self):
print "template_checkout guid/vers";
print ""
print "Environment:"
print " cwd The template checkout is placed in the current dir"
wrapperoptions();
return
pass
#
# template_instantiate
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment