Support for a "Commit" button, to allow the user to force a commit

of the archive.

configure

@@ -2,7 +2,7 @@
 
 # Guess values for system-dependent variables and create Makefiles.
 # Generated automatically using autoconf version 2.13 
-# Copyright (C) 1992, 93, 94, 95, 96 Free Software Foundation, Inc.
+# Copyright (C) 1992, 93, 94, 95, 96, 05 Free Software Foundation, Inc.
 #
 # This configure script is free software; the Free Software Foundation
 # gives unlimited permission to copy, distribute and modify it.
@@ -2282,6 +2282,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/idleswap tbsetup/webidleswap tbsetup/switchmac \
 	tbsetup/newnode_reboot tbsetup/webnodeattributes \
         tbsetup/libtestbed.py \
+	tbsetup/libArchive.pm tbsetup/archive_control \
+	tbsetup/webarchive_control \
 	tbsetup/tarfiles_setup tbsetup/webtarfiles_setup \
 	tbsetup/fetchtar.proxy tbsetup/webfrisbeekiller \
 	tbsetup/plab/GNUmakefile tbsetup/plab/libplab.py \

configure.in

@@ -720,6 +720,8 @@ outfiles="$outfiles Makeconf GNUmakefile \
 	tbsetup/idleswap tbsetup/webidleswap tbsetup/switchmac \
 	tbsetup/newnode_reboot tbsetup/webnodeattributes \
         tbsetup/libtestbed.py \
+	tbsetup/libArchive.pm tbsetup/archive_control \
+	tbsetup/webarchive_control \
 	tbsetup/tarfiles_setup tbsetup/webtarfiles_setup \
 	tbsetup/fetchtar.proxy tbsetup/webfrisbeekiller \
 	tbsetup/plab/GNUmakefile tbsetup/plab/libplab.py \

tbsetup/GNUmakefile.in

@@ -20,7 +20,7 @@ BIN_STUFF	= power snmpit tbend tbprerun tbreport \
 		  node_reboot nscheck node_update savelogs node_control \
 		  portstats checkports eventsys_control os_select tbrestart \
 		  tbswap nseswap tarfiles_setup node_history tbrsync \
-		  node_attributes
+		  node_attributes archive_control
 
 SBIN_STUFF	= resetvlans console_setup.proxy sched_reload named_setup \
 		  batch_daemon exports_setup reload_daemon sched_reserve \
@@ -48,7 +48,7 @@ LIBEXEC_STUFF	= rmproj wanlinksolve wanlinkinfo \
 		  spewlogfile staticroutes routecalc wanassign \
 		  webnodereboot webrmuser webidleswap switchmac \
 		  spewrpmtar webtarfiles_setup webfrisbeekiller gentopofile \
-		  webnodeattributes
+		  webnodeattributes webarchive_control
 
 LIB_STUFF       = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
                   snmpit_cisco.pm snmpit_lib.pm snmpit_apc.pm power_rpc27.pm \

tbsetup/archive_control.in

+#!/usr/bin/perl -wT
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2005 University of Utah and the Flux Group.
+# All rights reserved.
+#
+use English;
+use Getopt::Std;
+
+#
+# Command line interface to experiment archive module.
+#
+sub usage()
+{
+    print STDOUT
+	"Usage: archive_control [-f] <commit> <pid> <eid>\n";
+    exit(-1);
+}
+my $optlist = "df";
+my $debug   = 0;
+my $force   = 0;
+my $dbuid;
+
+#
+# Configure variables
+#
+my $TB		    = "@prefix@";
+my $TBOPS           = "@TBOPSEMAIL@";
+my $NFSTRACESUPPORT = @NFSTRACESUPPORT@;
+my $NFSTRACE	    = "$TB/sbin/nfstrace";
+
+# Protos
+sub fatal($);
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+# un-taint path
+$ENV{'PATH'} = "/bin:/usr/bin:/usr/local/bin:$TB/bin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+use libaudit;
+use libArchive;
+
+#
+# Verify user and get his DB uid.
+#
+if (! UNIX2DBUID($UID, \$dbuid)) {
+    die("*** $0:\n".
+	"    You do not exist in the Emulab Database!\n");
+}
+
+#
+# Parse command arguments. Once we return from getopts, all that should be
+# left are the required arguments.
+#
+%options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (@ARGV != 3) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (defined($options{"f"})) {
+    $force = 1;
+}
+my $action = $ARGV[0];
+my $pid    = $ARGV[1];
+my $eid    = $ARGV[2];
+
+#
+# Untaint args.
+#
+if ($action =~ /^(commit)$/) {
+    $action = $1;
+}
+else {
+    usage();
+}
+if ($pid =~ /^([-\w]+)$/) {
+    $pid = $1;
+}
+else {
+    die("Bad data in pid: $pid.");
+}
+if ($eid =~ /^([-\w]+)$/) {
+    $eid = $1;
+}
+else {
+    die("Bad data in eid: $eid.");
+}
+
+if (! ($expstate = ExpState($pid, $eid))) {
+    fatal("No such experiment $pid/$eid!");
+}
+
+#
+# Check permission. Only people with permission to destroy the experiment
+# can do this.
+#
+if (! TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_UPDATE)) {
+    fatal("You do not have permission to control the archive for $pid/$eid!");
+}
+
+#
+# Do not allow an archive system to be controlled if the experiment is not
+# active or swapped. Will probably be changed later.
+#
+if ($expstate ne EXPTSTATE_ACTIVE && 
+    $expstate ne EXPTSTATE_SWAPPED) {
+    fatal("Experiment $pid/$eid must be active or swapped!");
+}
+
+#
+# This script is always audited. Mail is sent automatically upon exit.
+#
+if (AuditStart(0)) {
+    #
+    # Parent exits normally
+    #
+    exit(0);
+}
+
+# Temporary
+libArchive::setdebug(1);
+
+#
+# Allow the user to force a commit of the archive.
+#
+if ($action eq "commit") {
+    if ($NFSTRACESUPPORT) {
+	#
+	# This program asks the nodes for the trace results.
+	#
+	print "Getting files accessed via NFS.\n";
+	system("$NFSTRACE transfer $pid $eid");
+ 
+	#
+	# Add the files that have been detected by tracing to the archive.
+	#
+	if (libArchive::TBExperimentArchiveAddTracedFiles($pid, $eid) < 0) {
+	    fatal("Failed to add traced files to the experiment archive!");
+	}
+    }
+
+    #
+    # Add the special per-experiment archive directory.
+    #
+    print "Getting user added files.\n";
+    if (libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) < 0) {
+	fatal("Failed to add user specified files to the experiment archive!");
+    }
+
+    #
+    # Do a SavePoint on the experiment files.
+    #
+    print "Doing a savepoint on the experiment archive ...\n";
+    if (libArchive::TBExperimentArchiveSavePoint($pid, $eid) < 0) {
+	fatal("Failed to do a savepoint on the experiment archive!");
+    }
+
+    # Commit the archive after swapout
+    print "Doing a commit on the experiment archive ...\n";
+    if (libArchive::TBCommitExperimentArchive($pid, $eid, "user_commit") < 0) {
+	  fatal("Failed to commit experiment archive!");
+    }
+}
+exit(0);
+
+sub fatal($)
+{
+    my ($mesg) = $_[0];
+
+    die("*** $0:\n".
+	"    $mesg\n");
+}
+

tbsetup/libArchive.pm.in

@@ -46,7 +46,7 @@ my $SVNADMIN    = "/usr/local/bin/svnadmin";
 my $IMPORTER    = "$TB/sbin/svn_load_dirs.pl";
 my $inittag     = 'root';
 my $defaultview = 'head';
-my $debug       = 1;
+my $debug       = 0;
 my $svnopt      = ($debug ? "" : "-q");
 
 # Little helper and debug function.
@@ -59,6 +59,21 @@ sub mysystem($)
     system($command);
 }
 
+# Another little helper for scripts that include this library.
+sub setdebug($)
+{
+    my ($toggle) = @_;
+
+    if ($toggle) {
+	$debug  = 1;
+	$svnopt = "";
+    }
+    else {
+	$debug  = 0;
+	$svnopt = "-q";
+    }
+}
+
 #
 # Create a new archive. Returns -1 if any error. Otherwise return
 # the new record index.
@@ -153,11 +168,12 @@ sub ArchiveCreate(;$$)
     mysystem("cd $dir; mkdir ignore; cd ignore; mkdir $view; ".
              "         mkdir $view/trunk $view/savepoint $view/tags; ".
 	     "         mkdir $view/history; ".
-	     "$SVN import -m 'ArchiveCreate' $view file://$repodir/$view")
+	     "$SVN import $svnopt -m 'ArchiveCreate' "/
+	     "     $view file://$repodir/$view")
 	== 0 or goto bad;
 
     # Create a branch tag in the tags directory to base differences against.
-    mysystem("$SVN copy -m 'ArchiveCreate Branch' ".
+    mysystem("$SVN copy $svnopt -m 'ArchiveCreate Branch' ".
 	     "          file://$repodir/$view/trunk ".
 	     "          file://$repodir/$view/tags/${tag}-branch")
 	== 0 or goto bad;
@@ -271,7 +287,7 @@ sub ArchiveAdd($$;$$)
 	mysystem("$TAR cf - -C / $pathname | tar xf - -C $checkin");
     }
     else {
-	mysystem("$RSYNC -R -avx --delete /${pathname} $checkin");
+	mysystem("$RSYNC -R -ax --delete /${pathname} $checkin");
     }
     if ($?) {
 	print STDERR "ArchiveFile: Could not copy in /$pathname\n";
@@ -358,13 +374,13 @@ sub ArchiveSavePoint($;$$$)
 	goto bad;
     }
     mysystem("$IMPORTER -no_user_input file://$repodir ".
-	     "          $view/savepoint . ")
+	     "          $view/savepoint . > /dev/null")
 	== 0 or goto bad;
 
     #
     # Create the tag for this savepoint. 
     # 
-    mysystem("$SVN copy -m 'ArchiveCreate Branch' ".
+    mysystem("$SVN copy $svnopt -m 'ArchiveCreate Branch' ".
 	     "          file://$repodir/$view/savepoint ".
 	     "          file://$repodir/$view/tags/${savetag}")
 	== 0 or goto bad;
@@ -531,7 +547,7 @@ sub ArchiveCommit($;$$$)
     }
 
     # Create a tag in the tags directory for the commit.
-    mysystem("$SVN copy -m 'ArchiveCommit' ".
+    mysystem("$SVN copy $svnopt -m 'ArchiveCommit' ".
 	     "          file://$repodir/$view/trunk ".
 	     "          file://$repodir/$view/tags/${newtag}")
 	== 0 or goto bad;
@@ -539,13 +555,13 @@ sub ArchiveCommit($;$$$)
     # Create a tag in the history directory for the commit. The
     # history directory has just the commit tags, so its easy to go
     # back in time.
-    mysystem("$SVN copy -m 'ArchiveCommit' ".
+    mysystem("$SVN copy $svnopt -m 'ArchiveCommit' ".
 	     "          file://$repodir/$view/trunk ".
 	     "          file://$repodir/$view/history/${newtag}")
 	== 0 or goto bad;
     
     # Create a branch tag in the tags directory to base differences against.
-    mysystem("$SVN copy -m 'ArchiveCommit Branch' ".
+    mysystem("$SVN copy $svnopt -m 'ArchiveCommit Branch' ".
 	     "          file://$repodir/$view/trunk ".
 	     "          file://$repodir/$view/tags/${newtag}-branch")
 	== 0 or goto bad;
@@ -689,7 +705,7 @@ sub ArchiveFork($$;$$$)
     }
 
     # Create newview directory in the repo.
-    mysystem("$SVN mkdir -m 'ArchiveFork' ".
+    mysystem("$SVN mkdir $svnopt -m 'ArchiveFork' ".
 	     "          file://$repodir/$newview")
 	== 0 or goto bad;
 
@@ -708,7 +724,7 @@ sub ArchiveFork($$;$$$)
 	== 0 or goto bad;
 
     # Do not want to copy the tags/savepoints directories. Add new ones.
-    mysystem("$SVN mkdir -m 'ArchiveFork' ".
+    mysystem("$SVN mkdir $svnopt -m 'ArchiveFork' ".
 	     "          file://$repodir/$newview/savepoint ".
 	     "          file://$repodir/$newview/tags")
 	== 0 or goto bad;
@@ -839,12 +855,28 @@ sub ArchiveArchive($$)
     #
     my $directory;
     if (GetArchiveDirectory($archive_idx, \$directory) < 0) {
-	print STDERR "ArchiveCommit: ".
+	print STDERR "ArchiveArchive: ".
 	    "Archive '$archive_idx' does not exist in the DB!\n";
 	return -1;
     }
+
+    #
+    # Need additional check to make sure that it has not already been
+    # archived.
+    #
+    my ($archived, $date_archived);
+    
+    if (IsArchiveArchived($archive_idx, \$archived, \$date_archived) < 0) {
+	return -1;
+    }
+    if ($archived) {
+	print STDERR "ArchiveArchive: ".
+	    "Archive '$archive_idx' already archived on $date_archived!\n";
+	return 0;
+    }
+    
     if (! -d $directory || ! -w $directory) {
-	print STDERR "ArchiveFile: $directory cannot be written!\n";
+	print STDERR "ArchiveArchive: $directory cannot be written!\n";
 	return -1;
     }
 
@@ -872,7 +904,10 @@ sub ArchiveArchive($$)
     #
     # Update its location in the DB, and remove the old directory.
     # 
-    DBQueryWarn("update archives set directory='$target' ".
+    DBQueryWarn("update archives set ".
+		"   directory='$target', ".
+		"   archived=1, ".
+		"   date_archived=UNIX_TIMESTAMP(now()) ".
 		"where idx='$archive_idx'")
 	or return -1;
     
@@ -889,7 +924,7 @@ sub ArchiveArchive($$)
 # Destroy an archive. The DB state is retained unless optional flag says
 # to clean it.
 #
-sub ArchiveDestroy($;$)
+sub ArchiveDestroy($$)
 {
     my ($archive_idx, $clean) = @_;
 
@@ -902,6 +937,22 @@ sub ArchiveDestroy($;$)
 	    "Archive '$archive_idx' does not exist in the DB!\n";
 	return -1;
     }
+
+    #
+    # Need additional check to make sure that it has not already been
+    # archived. Do not want to do anything, unless clean is specified.
+    #
+    my ($archived, $date_archived);
+    
+    if (IsArchiveArchived($archive_idx, \$archived, \$date_archived) < 0) {
+	return -1;
+    }
+    if ($archived && !$clean) {
+	print STDERR "ArchiveDestroy: ".
+	    "Archive '$archive_idx' archived on $date_archived!\n";
+	return 0;
+    }
+    
     if (! -d $directory || ! -w $directory) {
 	return 0;
     }
@@ -912,7 +963,7 @@ sub ArchiveDestroy($;$)
 	    "Could not remove contents of $directory!\n";
 	return -1;
     }
-    if (defined($clean) && $clean) {
+    if ($clean) {
 	(DBQueryWarn("delete from archive_tags ".
 		     "where archive_idx='$archive_idx'") &&
 	 DBQueryWarn("delete from archive_views ".
@@ -943,6 +994,30 @@ sub GetArchiveDirectory($$)
     return 0;
 }
 
+#
+# See if the archive has already been archived away, and when.
+#
+sub IsArchiveArchived($$$)
+{
+    my ($idx, $parch, $pdate) = @_;
+    
+    my $query_result =
+	DBQueryWarn("select archived,FROM_UNIXTIME(date_archived) ".
+		    "  from archives where idx='$idx'");
+
+    return -1
+	if (!$query_result || !$query_result->numrows);
+    
+    my ($archived,$date_archived) = $query_result->fetchrow_array();
+
+    $$parch = $archived
+	if (defined($parch));
+    $$pdate = $date_archived
+	if (defined($pdate));
+    
+    return 0;
+}
+
 #
 # Get the current tag for an archive, given its index. Returns -1 on error,
 # zero otherwise. Place tag in the return pointer.
@@ -1142,11 +1217,14 @@ sub TBExperimentArchiveAddUserFiles($$)
 #
 # SavePoint an experiment archive. 
 #
-sub TBExperimentArchiveSavePoint($$$)
+sub TBExperimentArchiveSavePoint($$;$)
 {
     my ($pid, $eid, $tagext) = @_;
     my ($archive_idx, $view);
 
+    $tagext = "savepoint"
+	if (!defined($tagext));
+
     return 0
 	if (!$MAINSITE || $pid ne $ALLOWEDPID);
 

tbsetup/webarchive_control.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2006 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+use English;
+
+#
+# This gets invoked from the Web interface. Simply a wrapper ...
+#
+
+#
+# Configure variables
+#
+my $TB       = "@prefix@";
+
+#
+# Run the real thing, and never return.
+# 
+exec "$TB/bin/archive_control", @ARGV;
+
+die("webarchive_control: Could not exec archive_control: $!");

utils/webviewvc.in

@@ -57,6 +57,7 @@ cfg.general.cvs_roots    = {}
 cfg.options.docroot      = "/cvsweb/viewvc"
 cfg.options.http_expiration_time = 10
 cfg.options.generate_etags = 1
+cfg.options.use_localtime  = 1
 cfg.general.address      = "<a href='mailto:@TBOPSEMAIL_NOSLASH@'>@TBOPSEMAIL_NOSLASH@</a>"
 
 viewcvs.main(server, cfg)

www/archive_control.php3

+<?php
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2006 University of Utah and the Flux Group.
+# All rights reserved.
+#
+include("defs.php3");
+include("showstuff.php3");
+
+#
+# Only known and logged in users.
+#
+$uid = GETLOGIN();
+LOGGEDINORDIE($uid);
+$isadmin = ISADMIN($uid);
+
+#
+# Verify page arguments.
+# 
+if (!isset($pid) ||
+    strcmp($pid, "") == 0) {
+    USERERROR("You must provide a Project ID.", 1);
+}
+if (!isset($eid) ||
+    strcmp($eid, "") == 0) {
+    USERERROR("You must provide an Experiment ID.", 1);
+}
+if (!TBvalid_pid($pid)) {
+    PAGEARGERROR("Invalid project ID.");
+}
+if (!TBvalid_eid($eid)) {
+    PAGEARGERROR("Invalid experiment ID.");
+}
+
+#
+# Check to make sure this is a valid PID/EID tuple.
+#
+if (! TBValidExperiment($pid, $eid)) {
+    USERERROR("The experiment $eid is not a valid experiment ".
+	      "in project $pid.", 1);
+}
+
+#
+# Verify Permission.
+#
+if (! TBExptAccessCheck($uid, $pid, $eid, $TB_EXPT_MODIFY)) {
+    USERERROR("You do not have permission to view experiment $eid!", 1);
+}
+
+$exptidx = TBExptIndex($pid, $eid);
+if ($exptidx < 0) {
+    TBERROR("Could not get experiment index for $pid/$eid!", 1);
+}
+
+if (!TBExptGroup($pid, $eid, $gid)) {
+    TBERROR("Could not get experiment gid for $pid/$eid!", 1);
+}
+
+#
+# Not many actions to consider.
+#
+if (isset($commit) && $commit != "") {
+	SUEXEC($uid, "$pid,$gid",
+	       "webarchive_control commit $pid $eid",
+	       SUEXEC_ACTION_DIE);
+}
+
+$newurl = preg_replace("/archive_control/",
+		       "archive_view", $_SERVER['REQUEST_URI']);
+
+header("Location: $newurl");
+?>

www/archive_view.php3

+<?php
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2006 University of Utah and the Flux Group.
+# All rights reserved.
+#
+include("defs.php3");
+include("showstuff.php3");
+
+#
+# Only known and logged in users can look at experiments.
+#
+$uid = GETLOGIN();
+LOGGEDINORDIE($uid);
+$isadmin = ISADMIN($uid);
+
+#
+# Verify page arguments.
+# 
+if (!isset($pid) ||
+    strcmp($pid, "") == 0) {
+    USERERROR("You must provide a Project ID.", 1);
+}
+if (!isset($eid) ||
+    strcmp($eid, "") == 0) {
+    USERERROR("You must provide an Experiment ID.", 1);
+}
+if (!TBvalid_pid($pid)) {
+    PAGEARGERROR("Invalid project ID.");
+}
+if (!TBvalid_eid($eid)) {
+    PAGEARGERROR("Invalid experiment ID.");
+}
+
+#
+# Standard Testbed Header now that we have the pid/eid okay.
+#
+PAGEHEADER("Experiment Archive ($pid/$eid)");
+
+#
+# Check to make sure this is a valid PID/EID tuple.
+#
+if (! TBValidExperiment($pid, $eid)) {
+    USERERROR("The experiment $eid is not a valid experiment ".
+	      "in project $pid.", 1);
+}
+
+#
+# Verify Permission.
+#
+if (! TBExptAccessCheck($uid, $pid, $eid, $TB_EXPT_READINFO)) {
+    USERERROR("You do not have permission to view experiment $eid!", 1);
+}
+
+$exptidx = TBExptIndex($pid, $eid);
+if ($exptidx < 0) {
+    TBERROR("Could not get experiment index for $pid/$eid!", 1);
+}
+$url = "cvsweb/cvsweb.php3?exptidx=$exptidx";
+
+echo "<center>\n";
+echo "This is the Subversion archive for your experiment.<br>";
+echo "<form action='archive_control.php3' method=get>\n";
+echo "<b><input type=submit name=commit value='Force Commit'></b>\n";
+echo "<input type=hidden name=pid value='$pid'>";
+echo "<input type=hidden name=eid value='$eid'>";
+echo "</form>\n";
+if (isset($commit)) {
+    echo "<b>Archive sucessfully committed.</b><br>";
+}
+echo "</center>\n";
+
+echo "<iframe width=100% height=800 scrolling=yes src='$url' border=2>".
+     "</iframe>\n";
+
+#
+# Standard Testbed Footer
+# 
+PAGEFOOTER();
+?>

www/cvsweb/cvsweb.php3

@@ -138,7 +138,7 @@ elseif (isset($exptidx) && $exptidx != "") {
 	if (! ISADMIN($uid)) {
 	    USERERROR("Must be administrator to view historical archives!", 1);
 	}
-	$repodir = "$TBDIR/expinfo/${pid}-${eid}.${exptidx}/Archive";
+	$repodir = "$TBDIR/expinfo/${pid}-${eid}.${exptidx}/Archive/repo";
     }
     $use_viewvc = 1;
 }

www/showexp.php3

@@ -180,7 +180,7 @@ if ($expstate) {
     }
 }
 
-WRITESUBMENUBUTTON("Modify Experiment Metadata",
+WRITESUBMENUBUTTON("Modify Metadata",
 		   "editexp.php3?pid=$exp_pid&eid=$exp_eid");
 
 WRITESUBMENUDIVIDER();
@@ -253,7 +253,7 @@ if (STUDLY()) {
 
 if ($EXPOSEARCHIVE) {
     WRITESUBMENUBUTTON("Experiment File Archive",
-		       "cvsweb/cvswebwrap.php3?exptidx=$expindex");
+		       "archive_view.php3?pid=$exp_pid&eid=$exp_eid");
 }
 
 if ($types['garcia'] || $types['static-mica2'] || $types['robot']) {