Commit f8333ef2 authored by Leigh Stoller's avatar Leigh Stoller

* Add new tables to store NS files (and any files they source) in the

  DB alongside the resource records. Previously, we stored only the
  nsfiles for current experiments, and purged them when the experiment
  was terminated. The new approach saves them forever using the resource
  record ID. Note that we do not store copies of NS files, but reference
  them indirectly instead so that we can MD5 them and avoid the dups.

  I put a "compressed" bit into the table cause at some point we will
  start compressing the data before storing them into the DB. Or maybe
  we bag this and start using GFS!

  Also note that this addresses the problem of losing the NS file
  history when using swapmod, since the NS file is overwritten.

* Add a pmapping table stores the nodes (and their types) used by an
  experiment. This data is also saved forever, alongside the resource
  records, so that we can more accurately replay an experiment. As Rob
  points out, the node names can also be used in conjunction with the
  ptop files that are saved, to get a 100% accurate remap of resources.
parent d42dddc8
......@@ -45,6 +45,7 @@ my $TBSWAP = "$TB/bin/tbswap";
my $TBREPORT = "$TB/bin/tbreport";
my $TBEND = "$TB/bin/tbend";
my $DU = "/usr/bin/du";
my $MD5 = "/sbin/md5";
# Swap Actions
$EXPT_PRELOAD = TBDB_STATS_PRELOAD();
......@@ -225,6 +226,7 @@ sub thumbnail($) { return resources($_[0], 'thumbnail'); }
sub swapin_time($) { return resources($_[0], 'swapin_time'); }
sub swapout_time($) { return resources($_[0], 'swapout_time'); }
sub lastidx($) { return resources($_[0], 'lastidx'); }
sub input_data_idx($) { return resources($_[0], 'input_data_idx'); }
#
# Lookup an experiment given an experiment index.
......@@ -544,10 +546,16 @@ sub Delete($;$)
if (! $purge);
#
# Now we can clean up the stats records.
# Now we can clean up the stats and resource records.
#
my $rsrcidx = $self->rsrcidx();
$self->DeleteInputFiles();
DBQueryWarn("DELETE from experiment_pmapping ".
"WHERE rsrcidx=$rsrcidx")
if (defined($rsrcidx) && $rsrcidx);
DBQueryWarn("DELETE from experiment_resources ".
"WHERE idx=$rsrcidx")
if (defined($rsrcidx) && $rsrcidx);
......@@ -562,6 +570,188 @@ sub Delete($;$)
return 0;
}
#
# Add an input file to the template. The point of this is to reduce
# duplication by taking an md5 of the input file, and sharing that
# record/file.
#
sub AddInputFile($$;$)
{
my ($self, $inputfile, $isnsfile) = @_;
my $input_data_idx;
my $isnew = 0;
# Must be a real reference.
return -1
if (! ref($self));
$isnsfile = 0
if (! defined($isnsfile));
return -1
if (! -r $inputfile);
my $data_string = `cat $inputfile`;
return -1
if ($?);
my $exptidx = $self->idx();
my $rsrcidx = $self->rsrcidx();
if ($data_string) {
# As you can see, we md5 the raw data.
$data_string = DBQuoteSpecial($data_string);
if (length($data_string) >= DBLIMIT_NSFILESIZE()) {
tberror("Input file is too big (> " . DBLIMIT_NSFILESIZE() . ")!");
return -1;
}
#
# Grab an MD5 of the file to see if we already have a copy of it.
# Avoids needless duplication.
#
my $md5 = `$MD5 -q $inputfile`;
chomp($md5);
DBQueryWarn("lock tables experiment_input_data write, ".
" experiment_inputs write, ".
" experiment_resources write")
or return -1;
my $query_result =
DBQueryWarn("select idx from experiment_input_data ".
"where md5='$md5'");
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
if ($query_result->numrows) {
($input_data_idx) = $query_result->fetchrow_array();
$isnew = 0;
}
else {
$query_result =
DBQueryWarn("insert into experiment_input_data ".
"(idx, md5, input) ".
"values (NULL, '$md5', $data_string)");
if (!$query_result) {
DBQueryWarn("unlock tables");
return -1;
}
$input_data_idx = $query_result->insertid;
$isnew = 1;
}
if (! DBQueryWarn("insert into experiment_inputs ".
" (rsrcidx, exptidx, input_data_idx) values ".
" ($rsrcidx, $exptidx, '$input_data_idx')")) {
DBQueryWarn("delete from experiment_input_data ".
"where idx='$input_data_idx'")
if ($isnew);
DBQueryWarn("unlock tables");
return -1;
}
if ($isnsfile &&
$self->TableUpdate("experiment_resources",
"input_data_idx='$input_data_idx'") != 0) {
DBQueryWarn("unlock tables");
return -1;
}
DBQueryWarn("unlock tables");
}
return 0;
}
#
# Delete the input files, but only if not in use.
#
sub DeleteInputFiles($)
{
my ($self) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $rsrcidx = $self->rsrcidx();
my $nsidx = $self->input_data_idx();
DBQueryWarn("lock tables experiment_input_data write, ".
" experiment_resources write, ".
" experiment_inputs write")
or return -1;
#
# Get all input files for this rsrc record.
#
my $query_result =
DBQueryWarn("select input_data_idx from experiment_inputs ".
"where rsrcidx='$rsrcidx'");
goto bad
if (! $query_result);
goto done
if (! $query_result->numrows);
while (my ($input_data_idx) = $query_result->fetchrow_array()) {
#
# Delete but only if not in use.
#
my $query_result =
DBQueryWarn("select count(rsrcidx) from experiment_inputs ".
"where input_data_idx='$input_data_idx' and ".
" rsrcidx!='$rsrcidx'");
goto bad
if (! $query_result);
next
if ($query_result->numrows);
DBQueryWarn("delete from experiment_input_data ".
"where idx='$input_data_idx'")
or goto bad;
DBQueryWarn("delete from experiment_inputs ".
"where input_data_idx='$input_data_idx'")
or goto bad;
if (defined($nsidx) && $nsidx == $input_data_idx) {
DBQueryWarn("update experiment_resources set input_data_idx=NULL ".
"where rsrcidx='$rsrcidx'")
or goto bad;
}
}
done:
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return 1;
}
#
# Grab an input file.
#
sub GetInputFile($$$)
{
my ($self, $idx, $pref) = @_;
# Must be a real reference.
return -1
if (! ref($self));
my $query_result =
DBQueryWarn("select input from experiment_input_data ".
"where idx='$idx'");
return -1
if (! $query_result || !$query_result->numrows);
my ($nsfile) = $query_result->fetchrow_array();
$$pref = $nsfile;
return 0;
}
#
# Refresh a class instance by reloading from the DB.
#
......@@ -1053,7 +1243,7 @@ sub CreateLogFile($$$)
}
#
# Set the experiments nsfiles table entry.
# Set the experiments NS file using AddInputFile() above
#
sub SetNSFile($$)
{
......@@ -1063,26 +1253,25 @@ sub SetNSFile($$)
return -1
if (! ref($self));
my $nsfile_string = `cat $nsfile`;
return 0
if (!$nsfile_string);
my $pid = $self->pid();
my $eid = $self->eid();
my $idx = $self->idx();
$nsfile_string = DBQuoteSpecial($nsfile_string);
return $self->AddInputFile($nsfile, 1);
}
if (length($nsfile_string) >= DBLIMIT_NSFILESIZE()) {
print "NS file is way too big!\n";
return -1;
}
sub GetNSFile($$)
{
my ($self, $pref) = @_;
# Must be a real reference.
return -1
if (!DBQueryWarn("delete from nsfiles where exptidx='$idx'") ||
!DBQueryWarn("insert into nsfiles (exptidx, pid, eid, nsfile) ".
"values ($idx, '$pid', '$eid', $nsfile_string)"));
if (! ref($self));
return 0;
# In case there is no NS file stored.
$$pref = undef;
my $input_data_idx = $self->input_data_idx();
return 0
if (!defined($input_data_idx));
return $self->GetInputFile($input_data_idx, $pref);
}
#
......@@ -1222,18 +1411,23 @@ sub PreSwap($$$)
#
# In SWAPIN, copy over the thumbnail. This is temporary; I think
# the thumbnail is going to end up going someplace else.
# For swapmod, its gonna get overwritten in tbprerun.
# Ditto above for input_data_idx.
#
my $thumbdata = (defined($self->thumbnail()) ?
DBQuoteSpecial($self->thumbnail()) : "NULL");
my $input_data_idx = (defined($self->input_data_idx()) ?
$self->input_data_idx() : "NULL");
my $byswapmod = ($which eq $EXPT_SWAPMOD ? 1 : 0);
my $byswapin = ($which eq $EXPT_SWAPIN ? 1 : 0);
my $query_result =
DBQueryWarn("insert into experiment_resources ".
" (idx, uid_idx, tstamp, exptidx, lastidx, ".
" byswapmod, byswapin, thumbnail) ".
" byswapmod, byswapin, input_data_idx, thumbnail) ".
"values (0, '$uid_idx', now(), $exptidx, $rsrcidx,".
" $byswapmod, $byswapin, $thumbdata)");
" $byswapmod, $byswapin, ".
" $input_data_idx, $thumbdata)");
return -1
if (! $query_result ||
! $query_result->insertid);
......@@ -1508,11 +1702,15 @@ sub PostSwap($$$$)
#
if ($which eq $EXPT_START ||
$which eq $EXPT_SWAPIN ||
$which eq $EXPT_SWAPMOD) {
($which eq $EXPT_SWAPMOD &&
$self->state() eq libdb::EXPTSTATE_ACTIVE())) {
my $query_result =
DBQueryWarn("select r.node_id from reserved as r ".
DBQueryWarn("select r.node_id,n.type,r.erole, ".
" r.vname,n.phys_nodeid ".
" from reserved as r ".
"left join nodes as n on r.node_id=n.node_id ".
"where r.exptidx='$exptidx' and n.role='testnode'");
"where r.exptidx='$exptidx' and ".
" (n.role='testnode' or n.role='virtnode')");
return -1
if (! $query_result);
......@@ -1522,6 +1720,20 @@ sub PostSwap($$$$)
DBQueryWarn("update experiment_resources set pnodes=$pnodes ".
"where idx=$rsrcidx")
or return -1;
# Generate the pmapping insert.
my @mappings = ();
while (my ($node_id,$type,$erole,$vname,$physnode) =
$query_result->fetchrow_array()) {
push(@mappings,
"($rsrcidx, '$vname', '$physnode', '$type', '$erole')");
}
if (@mappings) {
DBQueryWarn("insert into experiment_pmapping values ".
join(",", @mappings))
or return -1;
}
}
#
......
......@@ -3480,8 +3480,7 @@ sub TBExptContainsNodeCT($$$)
"virt_firewalls",
"firewall_rules",
"virt_tiptunnels",
"ipsubnets",
"nsfiles");
"ipsubnets");
@physicalTables = ("delays",
"vlans",
......
......@@ -464,13 +464,8 @@ sub readXML($$$$$) {
defined($rowhash{'pathname'})) {
my $pathname = $rowhash{'pathname'};
# libArchive checks the paths to make sure they are
# from the allowed places.
if (libArchive::TBExperimentArchiveAddFile($pid, $eid,
$pathname)
< 0) {
fatal("Failed to add $pathname to the archive!");
}
$experiment->AddInputFile($pathname) == 0
or fatal("Failed to add input file $pathname!");
}
}
next;
......
......@@ -506,6 +506,48 @@ CREATE TABLE `eventlist` (
KEY `vnode` (`vnode`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `experiment_input_data`
--
DROP TABLE IF EXISTS `experiment_input_data`;
CREATE TABLE `experiment_input_data` (
`idx` int(10) unsigned NOT NULL auto_increment,
`md5` varchar(32) NOT NULL default '',
`compressed` tinyint(1) unsigned default '0',
`input` mediumblob,
PRIMARY KEY (`idx`),
UNIQUE KEY `md5` (`md5`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `experiment_template_inputs`
--
DROP TABLE IF EXISTS `experiment_inputs`;
CREATE TABLE `experiment_inputs` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`exptidx` int(10) unsigned NOT NULL default '0',
`input_data_idx` int(10) unsigned NOT NULL default '0',
PRIMARY KEY (`rsrcidx`,`input_data_idx`),
KEY `rsrcidx` (`rsrcidx`),
KEY `exptidx` (`exptidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `experiment_pmapping`
--
DROP TABLE IF EXISTS `experiment_pmapping`;
CREATE TABLE `experiment_pmapping` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`vname` varchar(32) default NULL,
`node_id` varchar(32) NOT NULL default '',
`node_type` varchar(30) NOT NULL default '',
`node_erole` varchar(30) NOT NULL default '',
PRIMARY KEY (`rsrcidx`,`vname`,`node_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `experiment_resources`
--
......@@ -541,10 +583,12 @@ CREATE TABLE `experiment_resources` (
`delay_capacity` tinyint(3) unsigned default NULL,
`batchmode` tinyint(1) unsigned default '0',
`archive_tag` varchar(64) default NULL,
`input_data_idx` int(10) unsigned default NULL,
`thumbnail` mediumblob,
PRIMARY KEY (`idx`),
KEY `exptidx` (`exptidx`),
KEY `lastidx` (`lastidx`)
KEY `inputdata` (`input_data_idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......
......@@ -4114,3 +4114,42 @@ last_net_act,last_cpu_act,last_ext_act);
alter table virt_vtypes change name
`name` varchar(32) NOT NULL default '';
4.128: More historical data for experiments.
CREATE TABLE `experiment_input_data` (
`idx` int(10) unsigned NOT NULL auto_increment,
`md5` varchar(32) NOT NULL default '',
`compressed` tinyint(1) unsigned default '0',
`input` mediumblob,
PRIMARY KEY (`idx`),
UNIQUE KEY `md5` (`md5`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `experiment_pmapping`;
CREATE TABLE `experiment_pmapping` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`vname` varchar(32) default NULL,
`node_id` varchar(32) NOT NULL default '',
`node_type` varchar(30) NOT NULL default '',
`node_erole` varchar(30) NOT NULL default '',
PRIMARY KEY (`rsrcidx`,`vname`,`node_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
CREATE TABLE `experiment_inputs` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`exptidx` int(10) unsigned NOT NULL default '0',
`input_data_idx` int(10) unsigned NOT NULL default '0',
PRIMARY KEY (`rsrcidx`,`input_data_idx`),
KEY `rsrcidx` (`rsrcidx`),
KEY `exptidx` (`exptidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
alter table experiment_resources add
`input_data_idx` int(10) unsigned default NULL after archive_tag;
And then run this script:
./nsfiles.pl
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use lib "/usr/testbed/lib";
use libdb;
use libtestbed;
use Experiment;
my $tmpfile = "/tmp/nsfile.$$";
#
# Untaint the path
#
$ENV{'PATH'} = '/bin:/usr/bin:/usr/sbin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
my $query_result =
DBQueryFatal("select * from nsfiles");
while (my ($pid, $eid, $exptidx, $nsfile) = $query_result->fetchrow_array()) {
my $experiment = Experiment->Lookup($exptidx);
if (!defined($experiment)) {
die("Could not lookup experiment object for $exptidx\n");
}
open(NSFILE, ">$tmpfile")
or die("Could not open $tmpfile for writing!\n");
print NSFILE $nsfile;
close(NSFILE);
$experiment->SetNSFile($tmpfile) == 0
or die("Could not add $tmpfile to $experiment!\n");
}
......@@ -1212,13 +1212,10 @@ sub CopyInArchive()
#
# Grab a copy from the DB since we save all current NS files there.
#
my $query_result =
DBQueryFatal("select nsfile from nsfiles ".
"where pid='$copypid' and eid='$copyeid'");
tbdie("No such experiment in DB for $copypid/$copyeid\n")
if (!$query_result->numrows);
my ($nsfile) = $query_result->fetchrow_array();
tbdie("No nsfile in DB for $copypid/$copyeid\n")
my $nsfile;
$copy_experiment->GetNSFile(\$nsfile) == 0
or tbdie("Could not get NS file for $copy_experiment");
tbdie("No nsfile in DB for $copy_experiment")
if (!defined($nsfile) || $nsfile eq "");
open(NS, "> $tempnsfile")
......
......@@ -14,6 +14,7 @@ use Getopt::Std;
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use Experiment;
#
# Do things necessary for setting up inner elab experiment.
......@@ -94,7 +95,8 @@ my $dbuid;
my $user_name;
my $user_email;
my $query_result;
my $exptinfo;
my $inner_experiment;
my $inner_nsfile;
#
# Parse command arguments. Once we return from getopts, all that should
......@@ -215,30 +217,18 @@ if ($fwboot) {
# from the DB and save it.
#
if (defined($elabinelab_eid)) {
$query_result =
DBQueryFatal("select nsfile from nsfiles ".
"where pid='$pid' and eid='$elabinelab_eid'");
$inner_experiment = Experiment->Lookup($pid, $elabinelab_eid);
die("*** $0:\n".
" No such experiment in DB for $pid/$elabinelab_eid\n")
if (!$query_result->numrows);
my ($nsfile) = $query_result->fetchrow_array();
if (!defined($inner_experiment));
$inner_experiment->GetNSFile(\$inner_nsfile) == 0 or
die("*** $0:\n".
" No nsfile in DB for $pid/$elabinelab_eid\n")
if (!defined($nsfile) || $nsfile eq "");
$query_result =
DBQueryFatal("select * from experiments ".
"where pid='$pid' and eid='$elabinelab_eid'");
" Could not get NS file for $inner_experiment\n");
die("*** $0:\n".
" No such experiment in DB for $pid/$elabinelab_eid\n")
if (!$query_result->numrows);
$exptinfo = $query_result->fetchrow_hashref();
$exptinfo->{"nsfile"} = $nsfile;
" No nsfile in DB for $inner_experiment")
if (!defined($inner_nsfile) || $inner_nsfile eq "");
}
#
......@@ -486,7 +476,7 @@ if (defined($elabinelab_eid)) {
open(NS, "> /tmp/$$.ns")
or die("*** $0:\n".
" Could not write ns code to tmp file!\n");
print NS $exptinfo->{"nsfile"};
print NS $inner_nsfile;
print NS "\n";
close(NS);
......
# -*- tcl -*-
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2006 University of Utah and the Flux Group.
# Copyright (c) 2000-2007 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -681,10 +681,6 @@ Simulator instproc run {} {
$self spitxml_data "eventlist" $fields $values
}
foreach sourcefile $sourcefile_list {
$self spitxml_data "external_sourcefiles" [list "pathname" ] [list $sourcefile ]
}
foreach name [array names parameter_list] {
set default_value $parameter_list($name)
set description $parameter_descriptions($name)
......
......@@ -193,7 +193,8 @@ class Experiment
return ExperimentStats::Lookup($this->idx());
}
function GetResources() {
return ExperimentResources::Lookup($this->idx());
$stats = $this->GetStats();
return ExperimentResources::Lookup($stats->rsrcidx());
}
# accessors
......@@ -357,18 +358,18 @@ class Experiment
# Return the stored NS file for an experiment.
function NSFile() {
$pid = $this->pid();
$eid = $this->eid();
$resources = $this->GetResources();
$input_data_idx = $resources->input_data_idx();
$query_result =
DBQueryFatal("select nsfile from nsfiles ".
"where pid='$pid' and eid='$eid'");
DBQueryFatal("select input from experiment_input_data ".
"where idx='$input_data_idx'");
if (! mysql_num_rows($query_result)) {
return null;
}
$row = mysql_fetch_array($query_result);
return $row["nsfile"];
return $row["input"];
}
#
......@@ -1096,10 +1097,11 @@ class ExperimentStats
#
function AccessCheck ($user, $access_type) {
global $TBDB_TRUST_USER;
$pid_idx = $this->pid_idx();
if (! ($project = Project::Lookup($this->pid()))) {
if (! ($project = Project::Lookup($pid_idx))) {
TBERROR("ExperimentStats::AccessCheck: ".
"Cannot map $pid to its object", 1);
"Cannot map project $pid_idx to its object", 1);
}
return $project->AccessCheck($user, $TBDB_TRUST_USER);
}
......@@ -1112,14 +1114,12 @@ class ExperimentResources
#
# Constructor by lookup on unique index for current resources
#
function ExperimentResources($exptidx) {
$safe_exptidx = addslashes($exptidx);
function ExperimentResources($rsrcidx) {
$safe_rsrcidx = addslashes($rsrcidx);
$query_result =
DBQueryWarn("select r.* from experiment_stats as s ".
"left join experiment_resources as r on ".
" s.rsrcidx=r.idx ".
"where s.exptidx='$safe_exptidx'");
DBQueryWarn("select r.* from experiment_resources as r ".
"where r.idx='$safe_rsrcidx'");
if (!$query_result || !mysql_num_rows($query_result)) {
$this->resources = null;
......@@ -1133,9 +1133,9 @@ class ExperimentResources
return !is_null($this->resources);
}
# Lookup by exptidx.
function Lookup($exptidx) {
$foo = new ExperimentResources($exptidx);
# Lookup by resource record number
function Lookup($rsrcidx) {
$foo = new ExperimentResources($rsrcidx);
if ($foo->IsValid())
return $foo;
......@@ -1151,6 +1151,39 @@ class ExperimentResources
function exptidx() { return $this->field('exptidx'); }
function lastidx() { return $this->field('lastidx'); }
function wirelesslans() { return $this->field('wirelesslans'); }
function input_data_idx() { return $this->field('input_data_idx'); }
function GetStats() {
return ExperimentStats::Lookup($this->exptidx());
}
#
# Project level check via the stats record.
#
function AccessCheck ($user, $access_type) {
$experiment_stats = $this->GetStats();
return $experiment_stats->AccessCheck($user, $access_type);
}
# Return the stored NS file this resource record
function NSFile() {
$input_data_idx = $this->input_data_idx();
if (! $input_data_idx) {
return null;
}
$query_result =
DBQueryFatal("select input from experiment_input_data ".
"where idx='$input_data_idx'");
if (! mysql_num_rows($query_result)) {
return null;
}
$row = mysql_fetch_array($query_result);
return $row["input"];
}
}
#
......
......@@ -206,15 +206,7 @@ else if (strcmp($mode, "clear") == 0) {
#
set_time_limit(0);
$query_result = DBQueryFatal("SELECT nsfile from nsfiles ".
"where pid='$pid' and eid='$eid'");
if (mysql_num_rows($query_result)) {
$row = mysql_fetch_array($query_result);
$nsdata = $row["nsfile"];
}
else {
$nsdata = ""; # XXX what to do...
}