Commit bf20b3e5 authored by Gary Wong's avatar Gary Wong
Browse files

Add database/filesystem backend and HTTP transport for the blob store.

parent dca12fb0
......@@ -108,6 +108,29 @@ CREATE TABLE `archives` (
PRIMARY KEY (`idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `blob_files`
--
DROP TABLE IF EXISTS `blob_files`;
CREATE TABLE `blob_files` (
`filename` varchar(255) NOT NULL,
`hash` varchar(64) default NULL,
`hash_mtime` datetime default NULL,
PRIMARY KEY (`filename`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `blobs`
--
DROP TABLE IF EXISTS `blobs`;
CREATE TABLE `blobs` (
`uuid` varchar(40) NOT NULL,
`filename` tinytext,
PRIMARY KEY (`uuid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `bridges`
--
......@@ -444,6 +467,7 @@ CREATE TABLE `deltas` (
-- Table structure for table `elabinelab_attributes`
--
DROP TABLE IF EXISTS `elabinelab_attributes`;
CREATE TABLE `elabinelab_attributes` (
`pid` varchar(12) NOT NULL default '',
`eid` varchar(32) NOT NULL default '',
......@@ -628,7 +652,7 @@ DROP TABLE IF EXISTS `event_triggertypes`;
CREATE TABLE `event_triggertypes` (
`idx` smallint(5) unsigned NOT NULL,
`type` tinytext NOT NULL,
PRIMARY KEY (`idx`)
PRIMARY KEY (`idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......@@ -669,8 +693,8 @@ CREATE TABLE `experiment_blobs` (
`path` varchar(255) NOT NULL default '',
`action` varchar(255) NOT NULL default '',
PRIMARY KEY (`idx`),
UNIQUE KEY `exptidx` (`exptidx`, `path`, `action`)
) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=latin1;
UNIQUE KEY `exptidx` (`exptidx`,`path`,`action`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `experiment_features`
......@@ -683,7 +707,7 @@ CREATE TABLE `experiment_features` (
`exptidx` int(11) NOT NULL default '0',
`pid` varchar(12) NOT NULL default '',
`eid` varchar(32) NOT NULL default '',
PRIMARY KEY (`feature`,`exptidx`)
PRIMARY KEY (`feature`,`exptidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......@@ -1226,6 +1250,7 @@ CREATE TABLE `experiments` (
PRIMARY KEY (`idx`),
UNIQUE KEY `pideid` (`pid`,`eid`),
UNIQUE KEY `pididxeid` (`pid_idx`,`eid`),
UNIQUE KEY `keyhash` (`keyhash`),
KEY `batchmode` (`batchmode`),
KEY `state` (`state`),
KEY `eid_uuid` (`eid_uuid`)
......@@ -1319,22 +1344,6 @@ CREATE TABLE `foreign_keys` (
PRIMARY KEY (`table1`,`column1`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `fs_resources`
--
DROP TABLE IF EXISTS `fs_resources`;
CREATE TABLE `fs_resources` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`fileidx` int(11) unsigned NOT NULL default '0',
`exptidx` int(10) unsigned NOT NULL default '0',
`type` enum('r','w','rw','l') default 'r',
`size` int(11) unsigned default '0',
PRIMARY KEY (`rsrcidx`,`fileidx`),
KEY `rsrcidx` (`rsrcidx`),
KEY `fileidx` (`fileidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `frisbee_blobs`
--
......@@ -1347,11 +1356,27 @@ CREATE TABLE `frisbee_blobs` (
`load_address` text,
`frisbee_pid` int(11) default '0',
`load_busy` tinyint(4) NOT NULL default '0',
PRIMARY KEY (`idx`),
PRIMARY KEY (`idx`),
UNIQUE KEY `path` (`path`),
UNIQUE KEY `imageid` (`imageid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `fs_resources`
--
DROP TABLE IF EXISTS `fs_resources`;
CREATE TABLE `fs_resources` (
`rsrcidx` int(10) unsigned NOT NULL default '0',
`fileidx` int(11) unsigned NOT NULL default '0',
`exptidx` int(10) unsigned NOT NULL default '0',
`type` enum('r','w','rw','l') default 'r',
`size` int(11) unsigned default '0',
PRIMARY KEY (`rsrcidx`,`fileidx`),
KEY `rsrcidx` (`rsrcidx`),
KEY `fileidx` (`fileidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `global_policies`
--
......@@ -1552,25 +1577,10 @@ CREATE TABLE `image_history` (
PRIMARY KEY (`history_id`),
KEY `node_id` (`node_id`,`history_id`),
KEY `stamp` (`stamp`),
KEY `rsrcidx` (`rsrcidx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `subboss_images`
--
DROP TABLE IF EXISTS `subboss_images`;
CREATE TABLE `subboss_images` (
`subboss_id` varchar(32) NOT NULL default '',
`imageid` int(8) unsigned NOT NULL default '0',
`load_address` text,
`frisbee_pid` int(11) default '0',
`load_busy` tinyint(4) NOT NULL default '0',
`sync` tinyint(4) NOT NULL default '0',
PRIMARY KEY (`subboss_id`,`imageid`)
KEY `rsrcidx` (`rsrcidx`),
KEY `node_history_id` (`node_history_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `images`
--
......@@ -2209,10 +2219,10 @@ CREATE TABLE `node_hostkeys` (
`sshrsa_v1` mediumtext,
`sshrsa_v2` mediumtext,
`sshdsa_v2` mediumtext,
`sfshostid` varchar(128) default NULL,
`tpmblob` mediumtext,
`tpmx509` mediumtext,
`tpmidentity` mediumtext,
`sfshostid` varchar(128) default NULL,
PRIMARY KEY (`node_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
......@@ -2492,6 +2502,19 @@ CREATE TABLE `nonces` (
PRIMARY KEY (`node_id`,`purpose`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `nonlocal_user_bindings`
--
DROP TABLE IF EXISTS `nonlocal_user_bindings`;
CREATE TABLE `nonlocal_user_bindings` (
`uid` varchar(8) NOT NULL default '',
`uid_idx` mediumint(8) unsigned NOT NULL default '0',
`exptidx` int(11) NOT NULL default '0',
PRIMARY KEY (`uid_idx`,`exptidx`),
KEY `uid` (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `nonlocal_user_pubkeys`
--
......@@ -2508,19 +2531,6 @@ CREATE TABLE `nonlocal_user_pubkeys` (
KEY `uid` (`uid`,`idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `nonlocal_user_bindings`
--
DROP TABLE IF EXISTS `nonlocal_user_bindings`;
CREATE TABLE `nonlocal_user_bindings` (
`uid` varchar(8) NOT NULL default '',
`uid_idx` mediumint(8) unsigned NOT NULL default '0',
`exptidx` int(11) NOT NULL default '0',
PRIMARY KEY (`uid_idx`,`exptidx`),
KEY `uid` (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `nonlocal_users`
--
......@@ -2533,9 +2543,9 @@ CREATE TABLE `nonlocal_users` (
`created` datetime default NULL,
`name` tinytext,
`email` tinytext,
PRIMARY KEY (`uid_idx`),
KEY `uid` (`uid`),
UNIQUE KEY `uid_uuid` (`uid_uuid`)
PRIMARY KEY (`uid_idx`),
UNIQUE KEY `uid_uuid` (`uid_uuid`),
KEY `uid` (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......@@ -3345,6 +3355,21 @@ CREATE TABLE `state_triggers` (
PRIMARY KEY (`node_id`,`op_mode`,`state`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `subboss_images`
--
DROP TABLE IF EXISTS `subboss_images`;
CREATE TABLE `subboss_images` (
`subboss_id` varchar(32) NOT NULL default '',
`imageid` int(8) unsigned NOT NULL default '0',
`load_address` text,
`frisbee_pid` int(11) default '0',
`load_busy` tinyint(4) NOT NULL default '0',
`sync` tinyint(4) NOT NULL default '0',
PRIMARY KEY (`subboss_id`,`imageid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `subbosses`
--
......@@ -3358,20 +3383,7 @@ CREATE TABLE `subbosses` (
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `switch_paths`
--
DROP TABLE IF EXISTS `switch_paths`;
CREATE TABLE `switch_paths` (
`pid` varchar(12) default NULL,
`eid` varchar(32) default NULL,
`vname` varchar(32) default NULL,
`node_id1` varchar(32) default NULL,
`node_id2` varchar(32) default NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `openvpn_config`
-- Table structure for table `sw_configfiles`
--
DROP TABLE IF EXISTS `sw_configfiles`;
......@@ -3382,7 +3394,20 @@ CREATE TABLE `sw_configfiles` (
`file` varchar(4) NOT NULL,
`data` text,
`swid` varchar(20) NOT NULL,
PRIMARY KEY(`id`)
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table `switch_paths`
--
DROP TABLE IF EXISTS `switch_paths`;
CREATE TABLE `switch_paths` (
`pid` varchar(12) default NULL,
`eid` varchar(32) default NULL,
`vname` varchar(32) default NULL,
`node_id1` varchar(32) default NULL,
`node_id2` varchar(32) default NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......@@ -3435,6 +3460,7 @@ CREATE TABLE `table_regex` (
-- Table structure for table `template_stamps`
--
DROP TABLE IF EXISTS `template_stamps`;
CREATE TABLE `template_stamps` (
`guid` varchar(16) NOT NULL default '',
`vers` smallint(5) unsigned NOT NULL default '0',
......@@ -3588,8 +3614,8 @@ CREATE TABLE `unixgroup_membership` (
DROP TABLE IF EXISTS `user_features`;
CREATE TABLE `user_features` (
`feature` varchar(64) NOT NULL default '',
`uid_idx` mediumint(8) unsigned NOT NULL default '0',
`added` datetime NOT NULL,
`uid_idx` mediumint(8) unsigned NOT NULL default '0',
`uid` varchar(8) NOT NULL default '',
PRIMARY KEY (`feature`,`uid_idx`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
......@@ -3747,10 +3773,10 @@ CREATE TABLE `users` (
`wikionly` tinyint(1) default '0',
`mailman_password` tinytext,
PRIMARY KEY (`uid_idx`),
KEY `uid` (`uid`),
KEY `unix_uid` (`unix_uid`),
KEY `status` (`status`),
KEY `uid_uuid` (`uid_uuid`)
KEY `uid_uuid` (`uid_uuid`),
KEY `uid` (`uid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
......@@ -3953,8 +3979,8 @@ CREATE TABLE `virt_lans` (
`ip` varchar(15) NOT NULL default '',
`delay` float(10,2) default '0.00',
`bandwidth` int(10) unsigned default NULL,
`est_bandwidth` int(10) unsigned default NULL,
`backfill` int(10) unsigned default '0',
`est_bandwidth` int(10) unsigned default NULL,
`lossrate` float(10,8) default NULL,
`q_limit` int(11) default '0',
`q_maxthresh` int(11) default '0',
......@@ -3973,8 +3999,8 @@ CREATE TABLE `virt_lans` (
`mask` varchar(15) default '255.255.255.0',
`rdelay` float(10,2) default NULL,
`rbandwidth` int(10) unsigned default NULL,
`rest_bandwidth` int(10) unsigned default NULL,
`rbackfill` int(10) unsigned default '0',
`rest_bandwidth` int(10) unsigned default NULL,
`rlossrate` float(10,8) default NULL,
`cost` float NOT NULL default '1',
`widearea` tinyint(4) default '0',
......@@ -4538,5 +4564,3 @@ CREATE TABLE `wires` (
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
#
# Add extenal references slots to nodes table, as for geni.
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if( !DBTableExists( "blobs" ) ) {
DBQueryFatal( "CREATE TABLE blobs ( " .
"uuid VARCHAR( 40 ) NOT NULL, " .
"filename TINYTEXT, " .
"PRIMARY KEY( uuid ) );" );
}
if( !DBTableExists( "blob_files" ) ) {
DBQueryFatal( "CREATE TABLE blob_files ( " .
"filename VARCHAR( 255 ) NOT NULL, " .
"hash VARCHAR( 64 ) DEFAULT NULL, " .
"hash_mtime DATETIME DEFAULT NULL, " .
"PRIMARY KEY( filename ) );" );
}
if( !DBKeyExists( "experiments", "keyhash" ) ) {
DBQueryFatal( "ALTER TABLE experiments " .
"ADD UNIQUE KEY keyhash ( keyhash );" );
}
return 0;
}
1;
......@@ -17,7 +17,7 @@ SUBDIRS = nsgen
BIN_SCRIPTS = delay_config sshtb create_image node_admin link_config \
setdest loghole webcopy linkmon_ctl snmp-if-deref.sh \
template_record spewevents \
wbts_dump
wbts_dump mkblob
SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
eventping grantnodetype import_commitlog daemon_wrapper \
opsreboot deletenode node_statewait grabwebcams \
......@@ -26,7 +26,7 @@ SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
wanodecheckin wanodecreate spewimage \
anonsendmail epmodeset fixexpinfo node_traffic \
dumpdescriptor subboss_tftpboot_sync testbed-control \
archive-expinfo grantfeature emulabfeature
archive-expinfo grantfeature emulabfeature addblob readblob
WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \
webwanodecheckin webspewimage
......@@ -40,7 +40,7 @@ CTRLSBIN_SCRIPTS= opsdb_control.proxy daemon_wrapper
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS =
SETUID_SBIN_SCRIPTS = grabwebcams checkquota spewconlog opsdb_control suchown \
anonsendmail
anonsendmail readblob
SETUID_LIBX_SCRIPTS = xlogin
#
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use strict;
#
# Configure variables
#
my $TB = "@prefix@";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
#
# Handle command-line options.
#
sub usage() {
print STDERR "Usage: $0 <filename>\n";
exit( 1 );
}
usage() unless @ARGV == 1;
my ( $filename ) = @ARGV;
#
# Must taint check!
#
if ($filename =~ /^([-\w #%&*+,.\/:;=?@\[\\\]^{|}]+)$/) {
$filename = $1;
}
else {
print STDERR "Bad character in filename\n";
exit( 1 );
}
# We could use MySQL's UUID() function, but if we call it in the INSERT it
# becomes a pain to retrieve it. So we do the job ourselves.
my $uuid = `@UUIDGEN@`;
chomp $uuid;
my $result = DBQueryWarn( "INSERT INTO blobs SET uuid='$uuid', " .
"filename='$filename';" );
unless( $result ) {
print STDERR "Could not insert record.\n";
exit( 1 );
}
print "$uuid\n";
exit( 0 );
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
use Cwd;
use Digest::SHA1;
use English;
use Getopt::Std;
use POSIX qw( getuid setuid );
use strict;
#
# Configure variables
#
my $TB = "@prefix@";
my $FSDIR_PROJ = "@FSDIR_PROJ@";
my $FSDIR_GROUPS = "@FSDIR_GROUPS@";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
if ($EUID != 0) {
# We don't want to run this script unless its the real version.
die("*** $0:\n".
" Must be root! Maybe its a development version?\n");
}
# This script is setuid, so please do not run it as root. Hard to track
# what has happened.
if ($UID == 0) {
die("*** $0:\n".
" Please do not run this as root! Its already setuid!\n");
}
# Temporarily drop privileges as soon as possible.
$EUID = $UID;
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
#
# Handle command-line options.
#
sub usage() {
print STDERR "Usage: $0 [-h hash] [-q] <key> <blob>\n";
exit( 1 );
}
my $hash = 0;
my $query = 0;
my %options = ();
if (! getopts("h:q", \%options)) {
usage();
}
if ($options{q}) {
$query = 1;
}
if ($options{'h'}) {
$hash = $options{'h'};
}
usage() unless @ARGV == 2;
my ( $key, $blob ) = @ARGV;
#
# Must taint check!
#
if ($key =~ /^([-\w]+)$/) {
$key = $1;
}
else {
print STDERR "Bad data in argument: $key\n";
exit( 1 );
}
if ($blob =~ /^([-\w]+)$/) {
$blob = $1;
}
else {
print STDERR "Bad data in argument: $blob\n";
exit( 1 );
}
my $result = DBQueryWarn( "SELECT groups.unix_gid FROM experiments, groups " .
"WHERE experiments.keyhash='$key' AND " .
"experiments.gid_idx=groups.gid_idx;" );
if( !$result || $result->numrows != 1 ) {
print STDERR "Could not resolve key\n";
exit( 1 );
}
my ( $unix_gid ) = $result->fetchrow_array();
# Temporarily reacquire privileges.
$EUID = 0;
# Add the supplementary group ID.
$EGID = $EGID . " " . $unix_gid;
# Now we can permanently drop all privileges.
setuid( getuid() );
$result = DBQueryWarn( "SELECT filename FROM blobs WHERE uuid='$blob';" );
unless( $result && $result->numrows == 1 ) {
print STDERR "could not resolve $blob\n";
exit( 1 );
}
my ( $filename ) = $result->fetchrow_array();
$filename = Cwd::realpath( $filename );
# FIXME verify that $filename lives in a legitimate part of the filesystem
my $mtime = ( stat( $filename ) )[ 9 ];
if( !defined( $mtime ) ) {
print STDERR $filename . ": " . $ERRNO . "\n";
exit( 1 );
}
$result = DBQueryWarn( "SELECT hash, UNIX_TIMESTAMP( hash_mtime ) FROM " .
"blob_files WHERE filename='$filename';" );
if( $result && $result->numrows == 1 ) {
my ( $file_hash, $file_mtime ) = $result->fetchrow_array();
# Don't trust any existing hash unless the mtimes match _exactly_. Even
# if our mtime appears more recent than the file's, that is an
# excellent indication that our hash cannot be trusted (e.g. the
# file might have been restored from backup).
exit( 2 ) if( lc( $hash ) eq lc( $file_hash ) && $mtime == $file_mtime );
}
exit( 0 ) if $query;
if( !open( FILE, $filename ) ) {
print STDERR $filename . ": " . $ERRNO . "\n";