Add database/filesystem backend and HTTP transport for the blob store.

sql/database-create.sql

@@ -108,6 +108,29 @@ CREATE TABLE `archives` (
   PRIMARY KEY  (`idx`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
+--
+-- Table structure for table `blob_files`
+--
+
+DROP TABLE IF EXISTS `blob_files`;
+CREATE TABLE `blob_files` (
+  `filename` varchar(255) NOT NULL,
+  `hash` varchar(64) default NULL,
+  `hash_mtime` datetime default NULL,
+  PRIMARY KEY  (`filename`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+--
+-- Table structure for table `blobs`
+--
+
+DROP TABLE IF EXISTS `blobs`;
+CREATE TABLE `blobs` (
+  `uuid` varchar(40) NOT NULL,
+  `filename` tinytext,
+  PRIMARY KEY  (`uuid`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
 --
 -- Table structure for table `bridges`
 --
@@ -444,6 +467,7 @@ CREATE TABLE `deltas` (
 -- Table structure for table `elabinelab_attributes`
 --
 
+DROP TABLE IF EXISTS `elabinelab_attributes`;
 CREATE TABLE `elabinelab_attributes` (
   `pid` varchar(12) NOT NULL default '',
   `eid` varchar(32) NOT NULL default '',
@@ -628,7 +652,7 @@ DROP TABLE IF EXISTS `event_triggertypes`;
 CREATE TABLE `event_triggertypes` (
   `idx` smallint(5) unsigned NOT NULL,
   `type` tinytext NOT NULL,
-  PRIMARY KEY (`idx`)
+  PRIMARY KEY  (`idx`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
@@ -669,8 +693,8 @@ CREATE TABLE `experiment_blobs` (
   `path` varchar(255) NOT NULL default '',
   `action` varchar(255) NOT NULL default '',
   PRIMARY KEY  (`idx`),
-  UNIQUE KEY `exptidx` (`exptidx`, `path`, `action`)
-) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1;
+  UNIQUE KEY `exptidx` (`exptidx`,`path`,`action`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
 -- Table structure for table `experiment_features`
@@ -683,7 +707,7 @@ CREATE TABLE `experiment_features` (
   `exptidx` int(11) NOT NULL default '0',
   `pid` varchar(12) NOT NULL default '',
   `eid` varchar(32) NOT NULL default '',
-  PRIMARY KEY (`feature`,`exptidx`)
+  PRIMARY KEY  (`feature`,`exptidx`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
@@ -1226,6 +1250,7 @@ CREATE TABLE `experiments` (
   PRIMARY KEY  (`idx`),
   UNIQUE KEY `pideid` (`pid`,`eid`),
   UNIQUE KEY `pididxeid` (`pid_idx`,`eid`),
+  UNIQUE KEY `keyhash` (`keyhash`),
   KEY `batchmode` (`batchmode`),
   KEY `state` (`state`),
   KEY `eid_uuid` (`eid_uuid`)
@@ -1319,22 +1344,6 @@ CREATE TABLE `foreign_keys` (
   PRIMARY KEY  (`table1`,`column1`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
---
--- Table structure for table `fs_resources`
---
-
-DROP TABLE IF EXISTS `fs_resources`;
-CREATE TABLE `fs_resources` (
-  `rsrcidx` int(10) unsigned NOT NULL default '0',
-  `fileidx` int(11) unsigned NOT NULL default '0',
-  `exptidx` int(10) unsigned NOT NULL default '0',
-  `type` enum('r','w','rw','l') default 'r',
-  `size` int(11) unsigned default '0',
-  PRIMARY KEY  (`rsrcidx`,`fileidx`),
-  KEY `rsrcidx` (`rsrcidx`),
-  KEY `fileidx` (`fileidx`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
 --
 -- Table structure for table `frisbee_blobs`
 --
@@ -1347,11 +1356,27 @@ CREATE TABLE `frisbee_blobs` (
   `load_address` text,
   `frisbee_pid` int(11) default '0',
   `load_busy` tinyint(4) NOT NULL default '0',
-  PRIMARY KEY (`idx`),
+  PRIMARY KEY  (`idx`),
   UNIQUE KEY `path` (`path`),
   UNIQUE KEY `imageid` (`imageid`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
+--
+-- Table structure for table `fs_resources`
+--
+
+DROP TABLE IF EXISTS `fs_resources`;
+CREATE TABLE `fs_resources` (
+  `rsrcidx` int(10) unsigned NOT NULL default '0',
+  `fileidx` int(11) unsigned NOT NULL default '0',
+  `exptidx` int(10) unsigned NOT NULL default '0',
+  `type` enum('r','w','rw','l') default 'r',
+  `size` int(11) unsigned default '0',
+  PRIMARY KEY  (`rsrcidx`,`fileidx`),
+  KEY `rsrcidx` (`rsrcidx`),
+  KEY `fileidx` (`fileidx`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
 --
 -- Table structure for table `global_policies`
 --
@@ -1552,25 +1577,10 @@ CREATE TABLE `image_history` (
   PRIMARY KEY  (`history_id`),
   KEY `node_id` (`node_id`,`history_id`),
   KEY `stamp` (`stamp`),
-  KEY `rsrcidx` (`rsrcidx`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
---
--- Table structure for table `subboss_images`
---
-
-DROP TABLE IF EXISTS `subboss_images`;
-CREATE TABLE `subboss_images` (
-  `subboss_id` varchar(32) NOT NULL default '',
-  `imageid` int(8) unsigned NOT NULL default '0',
-  `load_address` text,
-  `frisbee_pid` int(11) default '0',
-  `load_busy` tinyint(4) NOT NULL default '0',
-  `sync` tinyint(4) NOT NULL default '0',
-  PRIMARY KEY  (`subboss_id`,`imageid`)
+  KEY `rsrcidx` (`rsrcidx`),
+  KEY `node_history_id` (`node_history_id`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
-
 --
 -- Table structure for table `images`
 --
@@ -2209,10 +2219,10 @@ CREATE TABLE `node_hostkeys` (
   `sshrsa_v1` mediumtext,
   `sshrsa_v2` mediumtext,
   `sshdsa_v2` mediumtext,
+  `sfshostid` varchar(128) default NULL,
   `tpmblob` mediumtext,
   `tpmx509` mediumtext,
   `tpmidentity` mediumtext,
-  `sfshostid` varchar(128) default NULL,
   PRIMARY KEY  (`node_id`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
@@ -2492,6 +2502,19 @@ CREATE TABLE `nonces` (
   PRIMARY KEY  (`node_id`,`purpose`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
+--
+-- Table structure for table `nonlocal_user_bindings`
+--
+
+DROP TABLE IF EXISTS `nonlocal_user_bindings`;
+CREATE TABLE `nonlocal_user_bindings` (
+  `uid` varchar(8) NOT NULL default '',
+  `uid_idx` mediumint(8) unsigned NOT NULL default '0',
+  `exptidx` int(11) NOT NULL default '0',
+  PRIMARY KEY  (`uid_idx`,`exptidx`),
+  KEY `uid` (`uid`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
 --
 -- Table structure for table `nonlocal_user_pubkeys`
 --
@@ -2508,19 +2531,6 @@ CREATE TABLE `nonlocal_user_pubkeys` (
   KEY `uid` (`uid`,`idx`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
---
--- Table structure for table `nonlocal_user_bindings`
---
-
-DROP TABLE IF EXISTS `nonlocal_user_bindings`;
-CREATE TABLE `nonlocal_user_bindings` (
-  `uid` varchar(8) NOT NULL default '',
-  `uid_idx` mediumint(8) unsigned NOT NULL default '0',
-  `exptidx` int(11) NOT NULL default '0',
-   PRIMARY KEY  (`uid_idx`,`exptidx`),
-   KEY `uid` (`uid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
 --
 -- Table structure for table `nonlocal_users`
 --
@@ -2533,9 +2543,9 @@ CREATE TABLE `nonlocal_users` (
   `created` datetime default NULL,
   `name` tinytext,
   `email` tinytext,
-   PRIMARY KEY  (`uid_idx`),
-   KEY `uid` (`uid`),
-   UNIQUE KEY `uid_uuid` (`uid_uuid`)
+  PRIMARY KEY  (`uid_idx`),
+  UNIQUE KEY `uid_uuid` (`uid_uuid`),
+  KEY `uid` (`uid`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
@@ -3345,6 +3355,21 @@ CREATE TABLE `state_triggers` (
   PRIMARY KEY  (`node_id`,`op_mode`,`state`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
+--
+-- Table structure for table `subboss_images`
+--
+
+DROP TABLE IF EXISTS `subboss_images`;
+CREATE TABLE `subboss_images` (
+  `subboss_id` varchar(32) NOT NULL default '',
+  `imageid` int(8) unsigned NOT NULL default '0',
+  `load_address` text,
+  `frisbee_pid` int(11) default '0',
+  `load_busy` tinyint(4) NOT NULL default '0',
+  `sync` tinyint(4) NOT NULL default '0',
+  PRIMARY KEY  (`subboss_id`,`imageid`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
 --
 -- Table structure for table `subbosses`
 --
@@ -3358,20 +3383,7 @@ CREATE TABLE `subbosses` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
--- Table structure for table `switch_paths`
---
-
-DROP TABLE IF EXISTS `switch_paths`;
-CREATE TABLE `switch_paths` (
-  `pid` varchar(12) default NULL,
-  `eid` varchar(32) default NULL,
-  `vname` varchar(32) default NULL,
-  `node_id1` varchar(32) default NULL,
-  `node_id2` varchar(32) default NULL
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
---
--- Table structure for table `openvpn_config`
+-- Table structure for table `sw_configfiles`
 --
 
 DROP TABLE IF EXISTS `sw_configfiles`;
@@ -3382,7 +3394,20 @@ CREATE TABLE `sw_configfiles` (
   `file` varchar(4) NOT NULL,
   `data` text,
   `swid` varchar(20) NOT NULL,
-   PRIMARY KEY(`id`)
+  PRIMARY KEY  (`id`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+--
+-- Table structure for table `switch_paths`
+--
+
+DROP TABLE IF EXISTS `switch_paths`;
+CREATE TABLE `switch_paths` (
+  `pid` varchar(12) default NULL,
+  `eid` varchar(32) default NULL,
+  `vname` varchar(32) default NULL,
+  `node_id1` varchar(32) default NULL,
+  `node_id2` varchar(32) default NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
@@ -3435,6 +3460,7 @@ CREATE TABLE `table_regex` (
 -- Table structure for table `template_stamps`
 --
 
+DROP TABLE IF EXISTS `template_stamps`;
 CREATE TABLE `template_stamps` (
   `guid` varchar(16) NOT NULL default '',
   `vers` smallint(5) unsigned NOT NULL default '0',
@@ -3588,8 +3614,8 @@ CREATE TABLE `unixgroup_membership` (
 DROP TABLE IF EXISTS `user_features`;
 CREATE TABLE `user_features` (
   `feature` varchar(64) NOT NULL default '',
-  `uid_idx` mediumint(8) unsigned NOT NULL default '0',
   `added` datetime NOT NULL,
+  `uid_idx` mediumint(8) unsigned NOT NULL default '0',
   `uid` varchar(8) NOT NULL default '',
   PRIMARY KEY  (`feature`,`uid_idx`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
@@ -3747,10 +3773,10 @@ CREATE TABLE `users` (
   `wikionly` tinyint(1) default '0',
   `mailman_password` tinytext,
   PRIMARY KEY  (`uid_idx`),
-  KEY `uid` (`uid`),
   KEY `unix_uid` (`unix_uid`),
   KEY `status` (`status`),
-  KEY `uid_uuid` (`uid_uuid`)
+  KEY `uid_uuid` (`uid_uuid`),
+  KEY `uid` (`uid`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
 
 --
@@ -3953,8 +3979,8 @@ CREATE TABLE `virt_lans` (
   `ip` varchar(15) NOT NULL default '',
   `delay` float(10,2) default '0.00',
   `bandwidth` int(10) unsigned default NULL,
-  `est_bandwidth` int(10) unsigned default NULL,
   `backfill` int(10) unsigned default '0',
+  `est_bandwidth` int(10) unsigned default NULL,
   `lossrate` float(10,8) default NULL,
   `q_limit` int(11) default '0',
   `q_maxthresh` int(11) default '0',
@@ -3973,8 +3999,8 @@ CREATE TABLE `virt_lans` (
   `mask` varchar(15) default '255.255.255.0',
   `rdelay` float(10,2) default NULL,
   `rbandwidth` int(10) unsigned default NULL,
-  `rest_bandwidth` int(10) unsigned default NULL,
   `rbackfill` int(10) unsigned default '0',
+  `rest_bandwidth` int(10) unsigned default NULL,
   `rlossrate` float(10,8) default NULL,
   `cost` float NOT NULL default '1',
   `widearea` tinyint(4) default '0',
@@ -4538,5 +4564,3 @@ CREATE TABLE `wires` (
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
-
-

sql/updates/4/229

+#
+# Add extenal references slots to nodes table, as for geni.
+#
+use strict;
+use libdb;
+
+sub DoUpdate($$$)
+{
+    my ($dbhandle, $dbname, $version) = @_;
+
+    if( !DBTableExists( "blobs" ) ) {
+	DBQueryFatal( "CREATE TABLE blobs ( " .
+		      "uuid VARCHAR( 40 ) NOT NULL, " .
+		      "filename TINYTEXT, " .
+		      "PRIMARY KEY( uuid ) );" );
+    }
+
+    if( !DBTableExists( "blob_files" ) ) {
+	DBQueryFatal( "CREATE TABLE blob_files ( " .
+		      "filename VARCHAR( 255 ) NOT NULL, " .
+		      "hash VARCHAR( 64 ) DEFAULT NULL, " .
+		      "hash_mtime DATETIME DEFAULT NULL, " .
+		      "PRIMARY KEY( filename ) );" );
+    }
+
+    if( !DBKeyExists( "experiments", "keyhash" ) ) {
+	DBQueryFatal( "ALTER TABLE experiments " .
+		      "ADD UNIQUE KEY keyhash ( keyhash );" );
+    }
+
+    return 0;
+}
+
+1;

utils/GNUmakefile.in

@@ -17,7 +17,7 @@ SUBDIRS		= nsgen
 BIN_SCRIPTS	= delay_config sshtb create_image node_admin link_config \
                   setdest loghole webcopy linkmon_ctl snmp-if-deref.sh \
 		  template_record spewevents \
-		  wbts_dump
+		  wbts_dump mkblob
 SBIN_SCRIPTS	= vlandiff vlansync withadminprivs export_tables cvsupd.pl \
                   eventping grantnodetype import_commitlog daemon_wrapper \
 		  opsreboot deletenode node_statewait grabwebcams \
@@ -26,7 +26,7 @@ SBIN_SCRIPTS	= vlandiff vlansync withadminprivs export_tables cvsupd.pl \
                   wanodecheckin wanodecreate spewimage \
                   anonsendmail epmodeset fixexpinfo node_traffic \
 		  dumpdescriptor subboss_tftpboot_sync testbed-control \
-		  archive-expinfo grantfeature emulabfeature
+		  archive-expinfo grantfeature emulabfeature addblob readblob
 
 WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \
                   webwanodecheckin webspewimage
@@ -40,7 +40,7 @@ CTRLSBIN_SCRIPTS= opsdb_control.proxy daemon_wrapper
 # These scripts installed setuid, with sudo. 
 SETUID_BIN_SCRIPTS   = 
 SETUID_SBIN_SCRIPTS  = grabwebcams checkquota spewconlog opsdb_control suchown \
-                       anonsendmail
+                       anonsendmail readblob
 SETUID_LIBX_SCRIPTS  = xlogin
 
 #

utils/addblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+

utils/mkblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+use English;
+use strict;
+
+#
+# Configure variables
+#
+my $TB			= "@prefix@";
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+
+#
+# Handle command-line options.
+#
+sub usage() {
+    print STDERR "Usage: $0 <filename>\n";
+    exit( 1 );
+}
+
+usage() unless @ARGV == 1;
+my ( $filename ) = @ARGV;
+#
+# Must taint check!
+#
+if ($filename =~ /^([-\w #%&*+,.\/:;=?@\[\\\]^{|}]+)$/) {
+    $filename = $1;
+}
+else {
+    print STDERR "Bad character in filename\n";
+    exit( 1 );
+}
+
+# We could use MySQL's UUID() function, but if we call it in the INSERT it
+# becomes a pain to retrieve it.  So we do the job ourselves.
+my $uuid = `@UUIDGEN@`;
+chomp $uuid;
+
+my $result = DBQueryWarn( "INSERT INTO blobs SET uuid='$uuid', " .
+			  "filename='$filename';" );
+unless( $result ) {
+    print STDERR "Could not insert record.\n";
+    exit( 1 );
+}
+
+print "$uuid\n";
+
+exit( 0 );

utils/readblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+use Cwd;
+use Digest::SHA1;
+use English;
+use Getopt::Std;
+use POSIX qw( getuid setuid );
+use strict;
+
+#
+# Configure variables
+#
+my $TB			= "@prefix@";
+my $FSDIR_PROJ		= "@FSDIR_PROJ@";
+my $FSDIR_GROUPS	= "@FSDIR_GROUPS@";
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+if ($EUID != 0) {
+    # We don't want to run this script unless its the real version.
+    die("*** $0:\n".
+	"    Must be root! Maybe its a development version?\n");
+}
+
+# This script is setuid, so please do not run it as root. Hard to track
+# what has happened.
+if ($UID == 0) {
+    die("*** $0:\n".
+	"    Please do not run this as root! Its already setuid!\n");
+}
+
+# Temporarily drop privileges as soon as possible.
+$EUID = $UID;
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+
+#
+# Handle command-line options.
+#
+sub usage() {
+    print STDERR "Usage: $0 [-h hash] [-q] <key> <blob>\n";
+    exit( 1 );
+}
+my $hash = 0;
+my $query = 0;
+my %options = ();
+if (! getopts("h:q", \%options)) {
+    usage();
+}
+if ($options{q}) {
+    $query = 1;
+}
+if ($options{'h'}) {
+    $hash = $options{'h'};
+}
+
+usage() unless @ARGV == 2;
+my ( $key, $blob ) = @ARGV;
+#
+# Must taint check!
+#
+if ($key =~ /^([-\w]+)$/) {
+    $key = $1;
+}
+else {
+    print STDERR "Bad data in argument: $key\n";
+    exit( 1 );
+}
+   
+if ($blob =~ /^([-\w]+)$/) {
+    $blob = $1;
+}
+else {
+    print STDERR "Bad data in argument: $blob\n";
+    exit( 1 );
+}
+   
+my $result = DBQueryWarn( "SELECT groups.unix_gid FROM experiments, groups " .
+			  "WHERE experiments.keyhash='$key' AND " .
+			  "experiments.gid_idx=groups.gid_idx;" );
+
+if( !$result || $result->numrows != 1 ) {
+    print STDERR "Could not resolve key\n";
+    exit( 1 );
+}
+
+my ( $unix_gid ) = $result->fetchrow_array();
+
+# Temporarily reacquire privileges.
+$EUID = 0;
+
+# Add the supplementary group ID.
+$EGID = $EGID . " " . $unix_gid;
+
+# Now we can permanently drop all privileges.
+setuid( getuid() );
+
+$result = DBQueryWarn( "SELECT filename FROM blobs WHERE uuid='$blob';" );
+
+unless( $result && $result->numrows == 1 ) {
+    print STDERR "could not resolve $blob\n";
+    exit( 1 );
+}
+
+my ( $filename ) = $result->fetchrow_array();
+
+$filename = Cwd::realpath( $filename );
+
+# FIXME verify that $filename lives in a legitimate part of the filesystem
+
+my $mtime = ( stat( $filename ) )[ 9 ];
+
+if( !defined( $mtime ) ) {
+    print STDERR $filename . ": " . $ERRNO . "\n";
+    exit( 1 );
+}
+
+$result = DBQueryWarn( "SELECT hash, UNIX_TIMESTAMP( hash_mtime ) FROM " .
+			  "blob_files WHERE filename='$filename';" );
+if( $result && $result->numrows == 1 ) {
+    my ( $file_hash, $file_mtime ) = $result->fetchrow_array();
+
+    # Don't trust any existing hash unless the mtimes match _exactly_.  Even
+    # if our mtime appears more recent than the file's, that is an
+    # excellent indication that our hash cannot be trusted (e.g. the
+    # file might have been restored from backup).
+    exit( 2 ) if( lc( $hash ) eq lc( $file_hash ) && $mtime == $file_mtime );
+}
+
+exit( 0 ) if $query;
+
+if( !open( FILE, $filename ) ) {
+    print STDERR $filename . ": " . $ERRNO . "\n";