Commit b898a8cc authored by Leigh Stoller's avatar Leigh Stoller

As discussed in meetings and email ... this commit changes what is

archived.  Rather then a special "archive" directory in the experiment
directory, we know archive the entire experiment directory.

This change should be backwards compatable, but let me know if not.

Note that the nsdata directory is gone; the nsfile comes from the
tbdata, but I know place a copy in nsfile.ns so that the name is well
known.
parent efa81ba4
......@@ -2655,8 +2655,7 @@ sub LogHole($)
my $pid = $self->pid();
my $eid = $self->eid();
my $userdir = TBExptUserDir($pid, $eid);
my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
my $logdir = "$archivedir/logs";
my $logdir = "$userdir/logs";
if (! -e $logdir) {
mkdir($logdir, 0775)
......@@ -2670,13 +2669,6 @@ sub LogHole($)
" LOGHOLE_ARGS='-c -s'") == 0
or return -1;
#
# Copy the experiment logs directory into the archive.
#
Template::mysystem("cd $userdir; ".
" $RSYNC -rtgoDlz --delete logs $archivedir") == 0
or return -1;
return 0;
}
......@@ -2716,11 +2708,11 @@ sub DumpDB($)
my $pid = $self->pid();
my $eid = $self->eid();
my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
my $userdir = TBExptUserDir($pid, $eid);
my $dumpdir = "$userdir/dbdata";
# The file is a gzipped.
my $dumpdir = "$archivedir/dbdata";
my $dumpfile = "$dumpdir/dbdump.gz";
my $dumpfile= "$dumpdir/dbdump.gz";
if (! -e $dumpdir) {
mkdir($dumpdir, 0775)
......
......@@ -250,9 +250,6 @@ elsif ($action eq "addtoarchive") {
my $userdir = TBExptUserDir($pid, $eid);
my $errors = 0;
# This is special by convention;
my $subdir = "$userdir/archive";
usage()
if (! @ARGV);
......
......@@ -218,7 +218,7 @@ if (!defined($tempnsfile) && !defined($copyarg) && !TBAdmin($dbuid)) {
tbdie("Only admins can create experiments with no NS file");
}
my $nsfile = "$eid.ns";
my $repfile = "$eid.report";
my $repfile = "report";
# Defaults for the DB and for the email message.
$noswap_reason = "None Given"
......@@ -448,6 +448,22 @@ if (system("/bin/cp", "$tempnsfile", "$nsfile")) {
}
chmod(0664, "$nsfile");
# Future; do not name the ns as above, but use generic name.
if (system("/bin/cp", "$tempnsfile", "nsfile.ns")) {
fatal({type => 'primary', severity => SEV_ERROR,
error => ['copy_ns_file_failed', $tempnsfile, "nsfile.ns"]},
"Could not copy $tempnsfile to $workdir/$nsfile");
}
# We created this file below so kill it.
if ($copyarg) {
# This is tainted for reasons I do not understand.
if ($tempnsfile =~ /^([-\w\.\/]+)$/) {
$tempnsfile = $1;
}
unlink($tempnsfile);
}
#
# Run parse in impotent mode on the NS file. This has no effect but
# will display any errors.
......@@ -662,32 +678,7 @@ $experiment->SaveLogFiles();
#
$experiment->CopyLogFiles();
#
# Copy out some files to the archive directory so that they get saved later.
#
my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
if (! -e "$archivedir/tbdata") {
mkdir("$archivedir/tbdata", 0775)
or fatal("Failed to mkdir $archivedir/tbdata");
}
if (! -e "$archivedir/nsdata") {
mkdir("$archivedir/nsdata", 0775)
or fatal("Failed to mkdir $archivedir/nsdata");
}
system("cp -fp $workdir/$nsfile $archivedir/nsdata/nsfile.ns") == 0
or fatal("Failed to copy nsfile to $archivedir/tbdata");
if (defined($logname)) {
system("cp -fp $logname $archivedir/tbdata/log.batchexp") == 0
or fatal("Failed to copy logfile to $archivedir/tbdata");
}
system("cp -fp $workdir/$repfile $archivedir/tbdata/tbreport.batchexp") == 0
or fatal("Failed to copy nsfile to $archivedir/tbdata");
# And tell the archive library about the above files.
# Tell the archive library to add all files to the archive.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal("Failed to add user archive files to the archive!");
......@@ -1220,56 +1211,36 @@ sub CopyInArchive()
return 0
if (!defined($copyarg));
my $dstdir = "$userdir/archive/nsdata";
$tempnsfile = "$dstdir/nsfile.ns";
$tempnsfile = "/tmp/$$.ns";
if ($copyfrom eq "exp") {
#
# Copy in from a current experiment. This is probably a bad thing
# to do, but hey.
# Grab a copy from the DB since we save all current NS files there.
#
my $srcdir = $copy_experiment->UserDir() . "/archive/nsdata";
if (-e $srcdir) {
print "Copying experiment archive from ${srcdir}\n";
system("$RSYNC -rtgoDlz --delete ${srcdir}/ $dstdir");
if ($?) {
fatal("CopyInArchive: Failed to copy ${srcdir} to $dstdir");
}
}
else {
#
# This is a really old experiment with no archive yet.
# Get the old experiment nsfile from the db.
#
my $query_result =
DBQueryFatal("select nsfile from nsfiles ".
"where pid='$copypid' and eid='$copyeid'");
tbdie("No such experiment in DB for $copypid/$copyeid\n")
if (!$query_result->numrows);
my ($nsfile) = $query_result->fetchrow_array();
tbdie("No nsfile in DB for $copypid/$copyeid\n")
if (!defined($nsfile) || $nsfile eq "");
if (! -e "$dstdir") {
mkdir("$dstdir", 0775)
or fatal("Failed to mkdir $dstdir for copy");
}
open(NS, "> $tempnsfile")
or tbdie("Could not write ns code to $tempnsfile!\n");
$nsfile =~ s/\r//g;
print NS $nsfile;
print NS "\n";
close(NS);
chmod(0664, "$tempnsfile");
}
my $query_result =
DBQueryFatal("select nsfile from nsfiles ".
"where pid='$copypid' and eid='$copyeid'");
tbdie("No such experiment in DB for $copypid/$copyeid\n")
if (!$query_result->numrows);
my ($nsfile) = $query_result->fetchrow_array();
tbdie("No nsfile in DB for $copypid/$copyeid\n")
if (!defined($nsfile) || $nsfile eq "");
open(NS, "> $tempnsfile")
or tbdie("Could not write ns code to $tempnsfile!\n");
$nsfile =~ s/\r//g;
print NS $nsfile;
print NS "\n";
close(NS);
chmod(0664, "$tempnsfile");
}
else {
#
# Ask for a checkout of the archive.
#
my $subdir = "/archive/nsdata";
my $dstdir = "$userdir/nsdata";
my $subdir = "/exp/tbdata";
my $oldns = "$dstdir/${copyeid}.ns";
print "Checking out experiment archive to $dstdir\n";
libArchive::TBCheckoutExperimentArchivebyExptIDX($copyidx,
......@@ -1277,6 +1248,31 @@ sub CopyInArchive()
$copytag,
$subdir)
== 0 or fatal("CopyInArchive: Checking out archive");
#
# An old template will have the nsfile in a different place.
#
if (! -e $oldns) {
$subdir = "/archive/nsdata";
$oldns = "$dstdir/archive/nsdata/nsfile.ns";
print "Checking out experiment archive to $dstdir\n";
libArchive::TBCheckoutExperimentArchivebyExptIDX($copyidx,
$dstdir,
$copytag,
$subdir)
== 0 or fatal("CopyInArchive: Checking out archive");
}
fatal("Missing NS file: $oldns")
if (! -e $oldns);
#
# Now copy the old NS file out and delete the rest of it.
#
system("/bin/cp", "$oldns", "$tempnsfile") == 0
or fatal("Failed to copy $oldns to $tempnsfile");
system("/bin/rm -rf $dstdir");
}
}
......@@ -1321,4 +1317,5 @@ END {
$cleaning = 1;
cleanup();
$? = $saved_exitcode;
}
......@@ -207,7 +207,7 @@ $cmd .= " -d"
if ($debug);
$cmd .= " -a"
if ($agent);
$cmd .= " -t " . TBExptUserDir($pid, $eid) . "/archive/events"
$cmd .= " -t " . TBExptUserDir($pid, $eid) . "/logs/events"
if (Template::Instance->LookupByExptidx($exptidx));
$cmd .= " -l " . TBExptUserDir($pid, $eid) . "/logs/event-sched.log";
$cmd .= " -k " . TBDB_EVENTKEY($pid, $eid);
......
......@@ -366,7 +366,7 @@ sub ArchiveAdd($$;$$$)
#
# What does this do?
# Basically, we copy the last part (directory) to / of the checkin.
# eg: cp /proj/pid/exp/eid/archive/... /archive of the checkins.
# eg: cp /proj/pid/exp/eid... /exp of the checkins.
# This avoids pid/eid tokens in the archive.
#
# Last part of path must be a directory.
......@@ -375,11 +375,9 @@ sub ArchiveAdd($$;$$$)
print STDERR "ArchiveAdd: Must be a directory: $pathname\n";
return -1;
}
my ($filename,$directory,undef) = fileparse($pathname);
$rootdir = $filename;
$sourcedir = $directory;
$sourcefile = $filename . "/";
$rootdir = "exp";
$sourcedir = $pathname;
$sourcefile = "./";
}
elsif ($pathname =~ /^[\/]+(\w+)\/([-\w\/\.\+\@,~]+)$/) {
$rootdir = $1;
......@@ -1500,7 +1498,7 @@ sub TBUserFileArchiveDirectory($$)
}
#
# Add files the user explicitly wants archived
# Add all files from the experiment directory to the archive.
#
sub TBExperimentArchiveAddUserFiles($$)
{
......@@ -1517,25 +1515,13 @@ sub TBExperimentArchiveAddUserFiles($$)
if ($rval < 0);
my $userdir = TBExptUserDir($pid, $eid);
my $userarch = "$userdir/archive";
if (-e $userarch) {
$rval = ArchiveAdd($archive_idx, $userarch, $view, 1, 1);
if (-e $userdir) {
$rval = ArchiveAdd($archive_idx, "$userdir/.", $view, 1, 1);
return $rval
if ($rval != 0);
}
# Another special directory ... specifically for templates.
my $userstore = "$userdir/datastore";
if (-e $userstore) {
$rval = ArchiveAdd($archive_idx, $userstore, $view, 1, 1);
return $rval
if ($rval != 0);
}
return 0;
}
......
......@@ -224,7 +224,7 @@ if ($eid =~ /^([-\w\.]+)$/) {
else {
tbdie("Tainted argument $eid!");
}
my $repfile = "$eid.report";
my $repfile = "report";
my $tempnsfile;
my $modnsfile;
my $nsfile;
......@@ -916,7 +916,7 @@ if ($inout eq "out") {
}
#
# Add the special per-experiment archive directory.
# Add the experiment directory.
#
if (libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) < 0) {
fatal({type => 'secondary', severity => SEV_SECONDARY,
......@@ -1116,6 +1116,10 @@ elsif ($inout eq "modify") {
if (system("/bin/mv", "$modnsfile", "$nsfile")) {
fatal("Could not mv $modnsfile to $nsfile");
}
unlink("nsfile.ns");
if (system("/bin/cp -p", "$nsfile", "nsfile.ns")) {
fatal("Could not cp $nsfile to nsfile.ns");
}
}
$experiment->ClearBackupState();
......@@ -1147,50 +1151,7 @@ $experiment->SaveLogFiles();
#
$experiment->CopyLogFiles();
#
# Copy out some files to the archive directory so that they get saved later.
#
$archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
if (! -e "$archivedir") {
mkdir("$archivedir", 0775)
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'mkdir', undef, $archivedir]},
"Failed to mkdir $archivedir");
}
if (! -e "$archivedir/tbdata") {
mkdir("$archivedir/tbdata", 0775)
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'mkdir', undef, "$archivedir/tbdata"]},
"Failed to mkdir $archivedir/tbdata");
}
if (! -e "$archivedir/nsdata") {
mkdir("$archivedir/nsdata", 0775)
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'mkdir', undef, "$archivedir/nsdata"]},
"Failed to mkdir $archivedir/nsdata");
}
system("cp -fp $workdir/$eid.ns $archivedir/nsdata/nsfile.ns") == 0
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'copy', "$workdir/$eid.ns", "$archivedir/nsdata/nsfile.ns"]},
"Failed to copy nsfile to $archivedir/tbdata");
if (defined($logname)) {
system("cp -fp $logname $archivedir/tbdata/log.$tag") == 0
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'copy', $logname, "$archivedir/tbdata/log.$tag"]},
"Failed to copy logfile to $archivedir/tbdata");
}
if (-e $repfile) {
system("cp -fp $workdir/$repfile $archivedir/tbdata/tbreport.$tag") == 0
or fatal({type => 'primary', severity => SEV_ERROR,
error => ['archive_op_failed', 'copy', "$workdir/$repfile", "$archivedir/tbdata/tbreport.$tag"]},
"Failed to copy nsfile to $archivedir/tbdata");
}
# And tell the archive library about the above files.
# And tell the archive library to add the experiment directory.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal("Failed to add user archive files to the archive!");
......
......@@ -182,7 +182,7 @@ foreach my $name (keys(%runlist)) {
my $rowref = $runlist{$name};
my $idx = $rowref->{"idx"};
my $dir = "$checkout/run${idx}";
my $subdir = "archive/dbdata";
my $subdir = "dbdata";
my $tag = $rowref->{"ending_archive_tag"};
my $dbdump = "$dir/dbdump.gz";
......
......@@ -167,13 +167,13 @@ sub CommitFromTemplate()
my $tid = $template->tid();
my $gid = $template->gid();
my $userdir = $template->path();
my $nsfile = "$userdir/archive/nsdata/nsfile.ns";
my $nsfile = "$userdir/tbdata/nsfile.ns";
my $optarg = ($quiet ? "-q" : "");
#
# The NS file is taken from the template.
#
fatal(1, "There is no NS file in $userdir/archive/nsdata!")
fatal(1, "There is no NS file in $userdir/tbdata!")
if (! -e $nsfile);
#
......@@ -204,12 +204,12 @@ sub CommitFromInstance()
}
my $userdir = $instance->path();
my $nsfile = "$userdir/archive/nsdata/nsfile.ns";
my $nsfile = "$userdir/tbdata/nsfile.ns";
#
# The NS file is taken from the instance.
#
fatal(1, "There is no NS file in $userdir/archive/nsdata!")
fatal(1, "There is no NS file in $userdir/tbdata!")
if (! -e $nsfile);
#
......
......@@ -367,12 +367,13 @@ if ($modify) {
# and tell the archive library about the above files.
libArchive::TBExperimentArchiveAddUserFiles($pid, $eid) == 0
or fatal(-1, "Failed to add datastore files to the archive!");
# and then do a savepoint prior to the commit below.
print "Doing another savepoint\n";
libArchive::TBExperimentArchiveSavePoint($pid, $eid, "CopyDataStore") >= 0
or fatal(-1, "Failed to do a savepoint on the experiment archive!");
}
# Do a savepoint prior to the commit below. As noted above, this is
# not done in batchexp.
print "Doing a savepoint\n";
libArchive::TBExperimentArchiveSavePoint($pid, $eid, "CopyDataStore") >= 0
or fatal(-1, "Failed to do a savepoint on the experiment archive!");
print "Doing final commit\n";
libArchive::TBCommitExperimentArchive($pid, $eid, "TemplateCreate")
>= 0 or fatal(-1, "Failed to commit experiment archive!");
......
......@@ -357,7 +357,7 @@ $SIG{TERM} = \&sighandler;
# If not in batch mode, go into the background. Parent exits.
#
if (! ($debug || $foreground)) {
$logname = TBExptCreateLogFile($pid, $eid, "newrun");
$logname = TBExptCreateLogFile($pid, $eid, "run");
TBExptSetLogFile($pid, $eid, $logname);
TBExptOpenLogFile($pid, $eid);
......@@ -463,7 +463,7 @@ if (defined($instance->runidx())) {
#
my $this_runid = $thisrun->runid();
system("$archcontrol -d -t ${action}run_${this_runid} commit $pid $eid");
system("$archcontrol -d -t stoprun_${this_runid} commit $pid $eid");
# This has to be done after the archive commit, so we can find the tag.
$instance->FinalizeCurrentRun() == 0
......@@ -521,8 +521,8 @@ if ($doswapmod) {
# Now do the swapmod, using the original NS file for now. The environ
# variables will be passed to the NS reparse by parse-ns wrapper script.
#
my $archivedir = libArchive::TBUserFileArchiveDirectory($pid, $eid);
my $nsfile = "$archivedir/nsdata/nsfile.ns";
my $userdir = $instance->path();
my $nsfile = "$userdir/tbdata/nsfile.ns";
my @arguments = ($swapexp, "-q", "-x", "-s", "modify",
$pid, $eid, $nsfile);
......@@ -545,7 +545,7 @@ if ($doswapmod) {
#
if (defined($paramfile)) {
my $source = $instance->workdir() . "/$PARAMS";
my $target = $instance->path() . "/archive/$PARAMS";
my $target = $instance->path() . "/$PARAMS";
unlink($target)
if (-e $target);
......@@ -597,14 +597,17 @@ if (keys(%deadnodes)) {
SENDMAIL($dbuid, $subject, $message, $TBOPS, "CC: $TBOPS");
}
# Stop the web interface from spewing.
TBExptCloseLogFile($pid, $eid)
if (defined($logname));
# log file gets copied out to the user directory.
$experiment->CopyLogFiles()
if (defined($logname));
# We lose the log info for this commit.
system("$archcontrol -d -t startrun_${runid} commit $pid $eid");
# Stop the web interface from spewing.
TBExptCloseLogFile($pid, $eid)
if (defined($logname));
exit(0);
#
......
......@@ -353,7 +353,7 @@ if (! ($foreground || $batchmode)) {
}
if (my $childpid = AuditStart(LIBAUDIT_DAEMON, $logname,
LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) {
LIBAUDIT_LOGONLY|LIBAUDIT_NODELETE|LIBAUDIT_FANCY)) {
#
# Parent exits normally, unless in waitmode. We have to set
# justexit to make sure the END block below does not run.
......@@ -522,7 +522,7 @@ my $userdir = $experiment->UserDir();
# Now we stash the parameter file.
#
if ($paramfile) {
system("/bin/cp", $paramfile, $userdir . "/archive/parameters.xml") == 0
system("/bin/cp", $paramfile, $userdir . "/parameters.xml") == 0
or tbdie("Could not copy out $paramfile");
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment