From b898a8cc8b610cfe944118fd1a8685457c7a11f6 Mon Sep 17 00:00:00 2001
From: "Leigh B. Stoller" <stoller@flux.utah.edu>
Date: Fri, 8 Dec 2006 16:16:27 +0000
Subject: [PATCH] As discussed in meetings and email ... this commit changes
what is archived. Rather then a special "archive" directory in the
experiment directory, we know archive the entire experiment directory.
This change should be backwards compatable, but let me know if not.
Note that the nsdata directory is gone; the nsfile comes from the
tbdata, but I know place a copy in nsfile.ns so that the name is well
known.
---
tbsetup/Template.pm.in | 18 ++---
tbsetup/archive_control.in | 3 -
tbsetup/batchexp.in | 133 ++++++++++++++++----------------
tbsetup/eventsys_control.in | 2 +-
tbsetup/libArchive.pm.in | 28 ++-----
tbsetup/swapexp.in | 53 ++-----------
tbsetup/template_analyze.in | 2 +-
tbsetup/template_commit.in | 8 +-
tbsetup/template_create.in | 11 +--
tbsetup/template_exprun.in | 21 ++---
tbsetup/template_instantiate.in | 4 +-
11 files changed, 110 insertions(+), 173 deletions(-)
diff --git a/tbsetup/Template.pm.in b/tbsetup/Template.pm.in
index 6eced01226..a875b85ed9 100644
--- a/tbsetup/Template.pm.in
+++ b/tbsetup/Template.pm.in
@@ -2655,8 +2655,7 @@ sub LogHole($)
my $pid = $self->pid();
my $eid = $self->eid();
my $userdir = TBExptUserDir($pid, $eid);
- my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
- my $logdir = "$archivedir/logs";
+ my $logdir = "$userdir/logs";
if (! -e $logdir) {
mkdir($logdir, 0775)
@@ -2670,13 +2669,6 @@ sub LogHole($)
" LOGHOLE_ARGS='-c -s'") == 0
or return -1;
- #
- # Copy the experiment logs directory into the archive.
- #
- Template::mysystem("cd $userdir; ".
- " $RSYNC -rtgoDlz --delete logs $archivedir") == 0
- or return -1;
-
return 0;
}
@@ -2716,11 +2708,11 @@ sub DumpDB($)
my $pid = $self->pid();
my $eid = $self->eid();
-
- my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
+ my $userdir = TBExptUserDir($pid, $eid);
+ my $dumpdir = "$userdir/dbdata";
+
# The file is a gzipped.
- my $dumpdir = "$archivedir/dbdata";
- my $dumpfile = "$dumpdir/dbdump.gz";
+ my $dumpfile= "$dumpdir/dbdump.gz";
if (! -e $dumpdir) {
mkdir($dumpdir, 0775)
diff --git a/tbsetup/archive_control.in b/tbsetup/archive_control.in
index 318e680afc..9b2d00ed16 100755
--- a/tbsetup/archive_control.in
+++ b/tbsetup/archive_control.in
@@ -250,9 +250,6 @@ elsif ($action eq "addtoarchive") {
my $userdir = TBExptUserDir($pid, $eid);
my $errors = 0;
- # This is special by convention;
- my $subdir = "$userdir/archive";
-
usage()
if (! @ARGV);
diff --git a/tbsetup/batchexp.in b/tbsetup/batchexp.in
index ea7aa9d69b..ddb4824cb4 100755
--- a/tbsetup/batchexp.in
+++ b/tbsetup/batchexp.in
@@ -218,7 +218,7 @@ if (!defined($tempnsfile) && !defined($copyarg) && !TBAdmin($dbuid)) {
tbdie("Only admins can create experiments with no NS file");
}
my $nsfile = "$eid.ns";
-my $repfile = "$eid.report";
+my $repfile = "report";
# Defaults for the DB and for the email message.
$noswap_reason = "None Given"
@@ -448,6 +448,22 @@ if (system("/bin/cp", "$tempnsfile", "$nsfile")) {
}
chmod(0664, "$nsfile");
+# Future; do not name the ns as above, but use generic name.
+if (system("/bin/cp", "$tempnsfile", "nsfile.ns")) {
+ fatal({type => 'primary', severity => SEV_ERROR,
+ error => ['copy_ns_file_failed', $tempnsfile, "nsfile.ns"]},
+ "Could not copy $tempnsfile to $workdir/$nsfile");
+}
+
+# We created this file below so kill it.
+if ($copyarg) {
+ # This is tainted for reasons I do not understand.
+ if ($tempnsfile =~ /^([-\w\.\/]+)$/) {
+ $tempnsfile = $1;
+ }
+ unlink($tempnsfile);
+}
+
#
# Run parse in impotent mode on the NS file. This has no effect but
# will display any errors.
@@ -662,32 +678,7 @@ $experiment->SaveLogFiles();
#
$experiment->CopyLogFiles();
-#
-# Copy out some files to the archive directory so that they get saved later.
-#
-my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
-
-if (! -e "$archivedir/tbdata") {
- mkdir("$archivedir/tbdata", 0775)
- or fatal("Failed to mkdir $archivedir/tbdata");
-}
-if (! -e "$archivedir/nsdata") {
- mkdir("$archivedir/nsdata", 0775)
- or fatal("Failed to mkdir $archivedir/nsdata");
-}
-
-system("cp -fp $workdir/$nsfile $archivedir/nsdata/nsfile.ns") == 0
- or fatal("Failed to copy nsfile to $archivedir/tbdata");
-
-if (defined($logname)) {
- system("cp -fp $logname $archivedir/tbdata/log.batchexp") == 0
- or fatal("Failed to copy logfile to $archivedir/tbdata");
-}
-
-system("cp -fp $workdir/$repfile $archivedir/tbdata/tbreport.batchexp") == 0
- or fatal("Failed to copy nsfile to $archivedir/tbdata");
-
-# And tell the archive library about the above files.
+# Tell the archive library to add all files to the archive.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal("Failed to add user archive files to the archive!");
@@ -1220,56 +1211,36 @@ sub CopyInArchive()
return 0
if (!defined($copyarg));
- my $dstdir = "$userdir/archive/nsdata";
- $tempnsfile = "$dstdir/nsfile.ns";
+ $tempnsfile = "/tmp/$$.ns";
if ($copyfrom eq "exp") {
#
- # Copy in from a current experiment. This is probably a bad thing
- # to do, but hey.
+ # Grab a copy from the DB since we save all current NS files there.
#
- my $srcdir = $copy_experiment->UserDir() . "/archive/nsdata";
-
- if (-e $srcdir) {
- print "Copying experiment archive from ${srcdir}\n";
- system("$RSYNC -rtgoDlz --delete ${srcdir}/ $dstdir");
- if ($?) {
- fatal("CopyInArchive: Failed to copy ${srcdir} to $dstdir");
- }
- }
- else {
- #
- # This is a really old experiment with no archive yet.
- # Get the old experiment nsfile from the db.
- #
- my $query_result =
- DBQueryFatal("select nsfile from nsfiles ".
- "where pid='$copypid' and eid='$copyeid'");
- tbdie("No such experiment in DB for $copypid/$copyeid\n")
- if (!$query_result->numrows);
- my ($nsfile) = $query_result->fetchrow_array();
- tbdie("No nsfile in DB for $copypid/$copyeid\n")
- if (!defined($nsfile) || $nsfile eq "");
-
- if (! -e "$dstdir") {
- mkdir("$dstdir", 0775)
- or fatal("Failed to mkdir $dstdir for copy");
- }
-
- open(NS, "> $tempnsfile")
- or tbdie("Could not write ns code to $tempnsfile!\n");
- $nsfile =~ s/\r//g;
- print NS $nsfile;
- print NS "\n";
- close(NS);
- chmod(0664, "$tempnsfile");
- }
+ my $query_result =
+ DBQueryFatal("select nsfile from nsfiles ".
+ "where pid='$copypid' and eid='$copyeid'");
+ tbdie("No such experiment in DB for $copypid/$copyeid\n")
+ if (!$query_result->numrows);
+ my ($nsfile) = $query_result->fetchrow_array();
+ tbdie("No nsfile in DB for $copypid/$copyeid\n")
+ if (!defined($nsfile) || $nsfile eq "");
+
+ open(NS, "> $tempnsfile")
+ or tbdie("Could not write ns code to $tempnsfile!\n");
+ $nsfile =~ s/\r//g;
+ print NS $nsfile;
+ print NS "\n";
+ close(NS);
+ chmod(0664, "$tempnsfile");
}
else {
#
# Ask for a checkout of the archive.
#
- my $subdir = "/archive/nsdata";
+ my $dstdir = "$userdir/nsdata";
+ my $subdir = "/exp/tbdata";
+ my $oldns = "$dstdir/${copyeid}.ns";
print "Checking out experiment archive to $dstdir\n";
libArchive::TBCheckoutExperimentArchivebyExptIDX($copyidx,
@@ -1277,6 +1248,31 @@ sub CopyInArchive()
$copytag,
$subdir)
== 0 or fatal("CopyInArchive: Checking out archive");
+
+ #
+ # An old template will have the nsfile in a different place.
+ #
+ if (! -e $oldns) {
+ $subdir = "/archive/nsdata";
+ $oldns = "$dstdir/archive/nsdata/nsfile.ns";
+
+ print "Checking out experiment archive to $dstdir\n";
+ libArchive::TBCheckoutExperimentArchivebyExptIDX($copyidx,
+ $dstdir,
+ $copytag,
+ $subdir)
+ == 0 or fatal("CopyInArchive: Checking out archive");
+ }
+
+ fatal("Missing NS file: $oldns")
+ if (! -e $oldns);
+
+ #
+ # Now copy the old NS file out and delete the rest of it.
+ #
+ system("/bin/cp", "$oldns", "$tempnsfile") == 0
+ or fatal("Failed to copy $oldns to $tempnsfile");
+ system("/bin/rm -rf $dstdir");
}
}
@@ -1321,4 +1317,5 @@ END {
$cleaning = 1;
cleanup();
$? = $saved_exitcode;
+
}
diff --git a/tbsetup/eventsys_control.in b/tbsetup/eventsys_control.in
index 2f65cd857c..69857be7e5 100755
--- a/tbsetup/eventsys_control.in
+++ b/tbsetup/eventsys_control.in
@@ -207,7 +207,7 @@ $cmd .= " -d"
if ($debug);
$cmd .= " -a"
if ($agent);
-$cmd .= " -t " . TBExptUserDir($pid, $eid) . "/archive/events"
+$cmd .= " -t " . TBExptUserDir($pid, $eid) . "/logs/events"
if (Template::Instance->LookupByExptidx($exptidx));
$cmd .= " -l " . TBExptUserDir($pid, $eid) . "/logs/event-sched.log";
$cmd .= " -k " . TBDB_EVENTKEY($pid, $eid);
diff --git a/tbsetup/libArchive.pm.in b/tbsetup/libArchive.pm.in
index 98c93d9e57..bd73b5d4f5 100644
--- a/tbsetup/libArchive.pm.in
+++ b/tbsetup/libArchive.pm.in
@@ -366,7 +366,7 @@ sub ArchiveAdd($$;$$$)
#
# What does this do?
# Basically, we copy the last part (directory) to / of the checkin.
- # eg: cp /proj/pid/exp/eid/archive/... /archive of the checkins.
+ # eg: cp /proj/pid/exp/eid... /exp of the checkins.
# This avoids pid/eid tokens in the archive.
#
# Last part of path must be a directory.
@@ -375,11 +375,9 @@ sub ArchiveAdd($$;$$$)
print STDERR "ArchiveAdd: Must be a directory: $pathname\n";
return -1;
}
- my ($filename,$directory,undef) = fileparse($pathname);
-
- $rootdir = $filename;
- $sourcedir = $directory;
- $sourcefile = $filename . "/";
+ $rootdir = "exp";
+ $sourcedir = $pathname;
+ $sourcefile = "./";
}
elsif ($pathname =~ /^[\/]+(\w+)\/([-\w\/\.\+\@,~]+)$/) {
$rootdir = $1;
@@ -1500,7 +1498,7 @@ sub TBUserFileArchiveDirectory($$)
}
#
-# Add files the user explicitly wants archived
+# Add all files from the experiment directory to the archive.
#
sub TBExperimentArchiveAddUserFiles($$)
{
@@ -1517,25 +1515,13 @@ sub TBExperimentArchiveAddUserFiles($$)
if ($rval < 0);
my $userdir = TBExptUserDir($pid, $eid);
- my $userarch = "$userdir/archive";
- if (-e $userarch) {
- $rval = ArchiveAdd($archive_idx, $userarch, $view, 1, 1);
+ if (-e $userdir) {
+ $rval = ArchiveAdd($archive_idx, "$userdir/.", $view, 1, 1);
return $rval
if ($rval != 0);
}
-
- # Another special directory ... specifically for templates.
- my $userstore = "$userdir/datastore";
-
- if (-e $userstore) {
- $rval = ArchiveAdd($archive_idx, $userstore, $view, 1, 1);
-
- return $rval
- if ($rval != 0);
- }
-
return 0;
}
diff --git a/tbsetup/swapexp.in b/tbsetup/swapexp.in
index d24054062d..8b0a778bde 100644
--- a/tbsetup/swapexp.in
+++ b/tbsetup/swapexp.in
@@ -224,7 +224,7 @@ if ($eid =~ /^([-\w\.]+)$/) {
else {
tbdie("Tainted argument $eid!");
}
-my $repfile = "$eid.report";
+my $repfile = "report";
my $tempnsfile;
my $modnsfile;
my $nsfile;
@@ -916,7 +916,7 @@ if ($inout eq "out") {
}
#
- # Add the special per-experiment archive directory.
+ # Add the experiment directory.
#
if (libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) < 0) {
fatal({type => 'secondary', severity => SEV_SECONDARY,
@@ -1116,6 +1116,10 @@ elsif ($inout eq "modify") {
if (system("/bin/mv", "$modnsfile", "$nsfile")) {
fatal("Could not mv $modnsfile to $nsfile");
}
+ unlink("nsfile.ns");
+ if (system("/bin/cp -p", "$nsfile", "nsfile.ns")) {
+ fatal("Could not cp $nsfile to nsfile.ns");
+ }
}
$experiment->ClearBackupState();
@@ -1147,50 +1151,7 @@ $experiment->SaveLogFiles();
#
$experiment->CopyLogFiles();
-#
-# Copy out some files to the archive directory so that they get saved later.
-#
-$archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
-
-if (! -e "$archivedir") {
- mkdir("$archivedir", 0775)
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'mkdir', undef, $archivedir]},
- "Failed to mkdir $archivedir");
-}
-if (! -e "$archivedir/tbdata") {
- mkdir("$archivedir/tbdata", 0775)
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'mkdir', undef, "$archivedir/tbdata"]},
- "Failed to mkdir $archivedir/tbdata");
-}
-if (! -e "$archivedir/nsdata") {
- mkdir("$archivedir/nsdata", 0775)
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'mkdir', undef, "$archivedir/nsdata"]},
- "Failed to mkdir $archivedir/nsdata");
-}
-
-system("cp -fp $workdir/$eid.ns $archivedir/nsdata/nsfile.ns") == 0
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'copy', "$workdir/$eid.ns", "$archivedir/nsdata/nsfile.ns"]},
- "Failed to copy nsfile to $archivedir/tbdata");
-
-if (defined($logname)) {
- system("cp -fp $logname $archivedir/tbdata/log.$tag") == 0
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'copy', $logname, "$archivedir/tbdata/log.$tag"]},
- "Failed to copy logfile to $archivedir/tbdata");
-}
-
-if (-e $repfile) {
- system("cp -fp $workdir/$repfile $archivedir/tbdata/tbreport.$tag") == 0
- or fatal({type => 'primary', severity => SEV_ERROR,
- error => ['archive_op_failed', 'copy', "$workdir/$repfile", "$archivedir/tbdata/tbreport.$tag"]},
- "Failed to copy nsfile to $archivedir/tbdata");
-}
-
-# And tell the archive library about the above files.
+# And tell the archive library to add the experiment directory.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal("Failed to add user archive files to the archive!");
diff --git a/tbsetup/template_analyze.in b/tbsetup/template_analyze.in
index 90f38d1e4d..7b16fb1d27 100644
--- a/tbsetup/template_analyze.in
+++ b/tbsetup/template_analyze.in
@@ -182,7 +182,7 @@ foreach my $name (keys(%runlist)) {
my $rowref = $runlist{$name};
my $idx = $rowref->{"idx"};
my $dir = "$checkout/run${idx}";
- my $subdir = "archive/dbdata";
+ my $subdir = "dbdata";
my $tag = $rowref->{"ending_archive_tag"};
my $dbdump = "$dir/dbdump.gz";
diff --git a/tbsetup/template_commit.in b/tbsetup/template_commit.in
index e348bd76e4..c8c90d2e6d 100644
--- a/tbsetup/template_commit.in
+++ b/tbsetup/template_commit.in
@@ -167,13 +167,13 @@ sub CommitFromTemplate()
my $tid = $template->tid();
my $gid = $template->gid();
my $userdir = $template->path();
- my $nsfile = "$userdir/archive/nsdata/nsfile.ns";
+ my $nsfile = "$userdir/tbdata/nsfile.ns";
my $optarg = ($quiet ? "-q" : "");
#
# The NS file is taken from the template.
#
- fatal(1, "There is no NS file in $userdir/archive/nsdata!")
+ fatal(1, "There is no NS file in $userdir/tbdata!")
if (! -e $nsfile);
#
@@ -204,12 +204,12 @@ sub CommitFromInstance()
}
my $userdir = $instance->path();
- my $nsfile = "$userdir/archive/nsdata/nsfile.ns";
+ my $nsfile = "$userdir/tbdata/nsfile.ns";
#
# The NS file is taken from the instance.
#
- fatal(1, "There is no NS file in $userdir/archive/nsdata!")
+ fatal(1, "There is no NS file in $userdir/tbdata!")
if (! -e $nsfile);
#
diff --git a/tbsetup/template_create.in b/tbsetup/template_create.in
index 8d9e319138..a7d783bd74 100644
--- a/tbsetup/template_create.in
+++ b/tbsetup/template_create.in
@@ -367,12 +367,13 @@ if ($modify) {
# and tell the archive library about the above files.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal(-1, "Failed to add datastore files to the archive!");
-
- # and then do a savepoint prior to the commit below.
- print "Doing another savepoint\n";
- libArchive::TBExperimentArchiveSavePoint($pid, $eid, "CopyDataStore") >= 0
- or fatal(-1, "Failed to do a savepoint on the experiment archive!");
}
+# Do a savepoint prior to the commit below. As noted above, this is
+# not done in batchexp.
+print "Doing a savepoint\n";
+libArchive::TBExperimentArchiveSavePoint($pid, $eid, "CopyDataStore") >= 0
+ or fatal(-1, "Failed to do a savepoint on the experiment archive!");
+
print "Doing final commit\n";
libArchive::TBCommitExperimentArchive($pid, $eid, "TemplateCreate")
>= 0 or fatal(-1, "Failed to commit experiment archive!");
diff --git a/tbsetup/template_exprun.in b/tbsetup/template_exprun.in
index f41ebc38d1..57129fedb4 100644
--- a/tbsetup/template_exprun.in
+++ b/tbsetup/template_exprun.in
@@ -357,7 +357,7 @@ $SIG{TERM} = \&sighandler;
# If not in batch mode, go into the background. Parent exits.
#
if (! ($debug || $foreground)) {
- $logname = TBExptCreateLogFile($pid, $eid, "newrun");
+ $logname = TBExptCreateLogFile($pid, $eid, "run");
TBExptSetLogFile($pid, $eid, $logname);
TBExptOpenLogFile($pid, $eid);
@@ -463,7 +463,7 @@ if (defined($instance->runidx())) {
#
my $this_runid = $thisrun->runid();
- system("$archcontrol -d -t ${action}run_${this_runid} commit $pid $eid");
+ system("$archcontrol -d -t stoprun_${this_runid} commit $pid $eid");
# This has to be done after the archive commit, so we can find the tag.
$instance->FinalizeCurrentRun() == 0
@@ -521,8 +521,8 @@ if ($doswapmod) {
# Now do the swapmod, using the original NS file for now. The environ
# variables will be passed to the NS reparse by parse-ns wrapper script.
#
- my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
- my $nsfile = "$archivedir/nsdata/nsfile.ns";
+ my $userdir = $instance->path();
+ my $nsfile = "$userdir/tbdata/nsfile.ns";
my @arguments = ($swapexp, "-q", "-x", "-s", "modify",
$pid, $eid, $nsfile);
@@ -545,7 +545,7 @@ if ($doswapmod) {
#
if (defined($paramfile)) {
my $source = $instance->workdir() . "/$PARAMS";
- my $target = $instance->path() . "/archive/$PARAMS";
+ my $target = $instance->path() . "/$PARAMS";
unlink($target)
if (-e $target);
@@ -597,14 +597,17 @@ if (keys(%deadnodes)) {
SENDMAIL($dbuid, $subject, $message, $TBOPS, "CC: $TBOPS");
}
-# Stop the web interface from spewing.
-TBExptCloseLogFile($pid, $eid)
- if (defined($logname));
-
# log file gets copied out to the user directory.
$experiment->CopyLogFiles()
if (defined($logname));
+# We lose the log info for this commit.
+system("$archcontrol -d -t startrun_${runid} commit $pid $eid");
+
+# Stop the web interface from spewing.
+TBExptCloseLogFile($pid, $eid)
+ if (defined($logname));
+
exit(0);
#
diff --git a/tbsetup/template_instantiate.in b/tbsetup/template_instantiate.in
index 21dd34b25a..d55dfdb270 100644
--- a/tbsetup/template_instantiate.in
+++ b/tbsetup/template_instantiate.in
@@ -353,7 +353,7 @@ if (! ($foreground || $batchmode)) {
}
if (my $childpid = AuditStart(LIBAUDIT_DAEMON, $logname,
- LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) {
+ LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) {
#
# Parent exits normally, unless in waitmode. We have to set
# justexit to make sure the END block below does not run.
@@ -522,7 +522,7 @@ my $userdir = $experiment->UserDir();
# Now we stash the parameter file.
#
if ($paramfile) {
- system("/bin/cp", $paramfile, $userdir . "/archive/parameters.xml") == 0
+ system("/bin/cp", $paramfile, $userdir . "/parameters.xml") == 0
or tbdie("Could not copy out $paramfile");
}
--
GitLab