Add database/filesystem backend and HTTP transport for the blob store.

sql/updates/4/229

+#
+# Add extenal references slots to nodes table, as for geni.
+#
+use strict;
+use libdb;
+
+sub DoUpdate($$$)
+{
+    my ($dbhandle, $dbname, $version) = @_;
+
+    if( !DBTableExists( "blobs" ) ) {
+	DBQueryFatal( "CREATE TABLE blobs ( " .
+		      "uuid VARCHAR( 40 ) NOT NULL, " .
+		      "filename TINYTEXT, " .
+		      "PRIMARY KEY( uuid ) );" );
+    }
+
+    if( !DBTableExists( "blob_files" ) ) {
+	DBQueryFatal( "CREATE TABLE blob_files ( " .
+		      "filename VARCHAR( 255 ) NOT NULL, " .
+		      "hash VARCHAR( 64 ) DEFAULT NULL, " .
+		      "hash_mtime DATETIME DEFAULT NULL, " .
+		      "PRIMARY KEY( filename ) );" );
+    }
+
+    if( !DBKeyExists( "experiments", "keyhash" ) ) {
+	DBQueryFatal( "ALTER TABLE experiments " .
+		      "ADD UNIQUE KEY keyhash ( keyhash );" );
+    }
+
+    return 0;
+}
+
+1;

utils/GNUmakefile.in

@@ -17,7 +17,7 @@ SUBDIRS		= nsgen
 BIN_SCRIPTS	= delay_config sshtb create_image node_admin link_config \
                   setdest loghole webcopy linkmon_ctl snmp-if-deref.sh \
 		  template_record spewevents \
-		  wbts_dump
+		  wbts_dump mkblob
 SBIN_SCRIPTS	= vlandiff vlansync withadminprivs export_tables cvsupd.pl \
                   eventping grantnodetype import_commitlog daemon_wrapper \
 		  opsreboot deletenode node_statewait grabwebcams \
@@ -26,7 +26,7 @@ SBIN_SCRIPTS	= vlandiff vlansync withadminprivs export_tables cvsupd.pl \
                   wanodecheckin wanodecreate spewimage \
                   anonsendmail epmodeset fixexpinfo node_traffic \
 		  dumpdescriptor subboss_tftpboot_sync testbed-control \
-		  archive-expinfo grantfeature emulabfeature
+		  archive-expinfo grantfeature emulabfeature addblob readblob
 
 WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \
                   webwanodecheckin webspewimage
@@ -40,7 +40,7 @@ CTRLSBIN_SCRIPTS= opsdb_control.proxy daemon_wrapper
 # These scripts installed setuid, with sudo. 
 SETUID_BIN_SCRIPTS   = 
 SETUID_SBIN_SCRIPTS  = grabwebcams checkquota spewconlog opsdb_control suchown \
-                       anonsendmail
+                       anonsendmail readblob
 SETUID_LIBX_SCRIPTS  = xlogin
 
 #

utils/addblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+

utils/mkblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+use English;
+use strict;
+
+#
+# Configure variables
+#
+my $TB			= "@prefix@";
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+
+#
+# Handle command-line options.
+#
+sub usage() {
+    print STDERR "Usage: $0 <filename>\n";
+    exit( 1 );
+}
+
+usage() unless @ARGV == 1;
+my ( $filename ) = @ARGV;
+#
+# Must taint check!
+#
+if ($filename =~ /^([-\w #%&*+,.\/:;=?@\[\\\]^{|}]+)$/) {
+    $filename = $1;
+}
+else {
+    print STDERR "Bad character in filename\n";
+    exit( 1 );
+}
+
+# We could use MySQL's UUID() function, but if we call it in the INSERT it
+# becomes a pain to retrieve it.  So we do the job ourselves.
+my $uuid = `@UUIDGEN@`;
+chomp $uuid;
+
+my $result = DBQueryWarn( "INSERT INTO blobs SET uuid='$uuid', " .
+			  "filename='$filename';" );
+unless( $result ) {
+    print STDERR "Could not insert record.\n";
+    exit( 1 );
+}
+
+print "$uuid\n";
+
+exit( 0 );

utils/readblob.in

+#!/usr/bin/perl -w
+
+#
+# EMULAB-COPYRIGHT
+# Copyright (c) 2010 University of Utah and the Flux Group.
+# All rights reserved.
+#
+
+use Cwd;
+use Digest::SHA1;
+use English;
+use Getopt::Std;
+use POSIX qw( getuid setuid );
+use strict;
+
+#
+# Configure variables
+#
+my $TB			= "@prefix@";
+my $FSDIR_PROJ		= "@FSDIR_PROJ@";
+my $FSDIR_GROUPS	= "@FSDIR_GROUPS@";
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+if ($EUID != 0) {
+    # We don't want to run this script unless its the real version.
+    die("*** $0:\n".
+	"    Must be root! Maybe its a development version?\n");
+}
+
+# This script is setuid, so please do not run it as root. Hard to track
+# what has happened.
+if ($UID == 0) {
+    die("*** $0:\n".
+	"    Please do not run this as root! Its already setuid!\n");
+}
+
+# Temporarily drop privileges as soon as possible.
+$EUID = $UID;
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+
+#
+# Handle command-line options.
+#
+sub usage() {
+    print STDERR "Usage: $0 [-h hash] [-q] <key> <blob>\n";
+    exit( 1 );
+}
+my $hash = 0;
+my $query = 0;
+my %options = ();
+if (! getopts("h:q", \%options)) {
+    usage();
+}
+if ($options{q}) {
+    $query = 1;
+}
+if ($options{'h'}) {
+    $hash = $options{'h'};
+}
+
+usage() unless @ARGV == 2;
+my ( $key, $blob ) = @ARGV;
+#
+# Must taint check!
+#
+if ($key =~ /^([-\w]+)$/) {
+    $key = $1;
+}
+else {
+    print STDERR "Bad data in argument: $key\n";
+    exit( 1 );
+}
+   
+if ($blob =~ /^([-\w]+)$/) {
+    $blob = $1;
+}
+else {
+    print STDERR "Bad data in argument: $blob\n";
+    exit( 1 );
+}
+   
+my $result = DBQueryWarn( "SELECT groups.unix_gid FROM experiments, groups " .
+			  "WHERE experiments.keyhash='$key' AND " .
+			  "experiments.gid_idx=groups.gid_idx;" );
+
+if( !$result || $result->numrows != 1 ) {
+    print STDERR "Could not resolve key\n";
+    exit( 1 );
+}
+
+my ( $unix_gid ) = $result->fetchrow_array();
+
+# Temporarily reacquire privileges.
+$EUID = 0;
+
+# Add the supplementary group ID.
+$EGID = $EGID . " " . $unix_gid;
+
+# Now we can permanently drop all privileges.
+setuid( getuid() );
+
+$result = DBQueryWarn( "SELECT filename FROM blobs WHERE uuid='$blob';" );
+
+unless( $result && $result->numrows == 1 ) {
+    print STDERR "could not resolve $blob\n";
+    exit( 1 );
+}
+
+my ( $filename ) = $result->fetchrow_array();
+
+$filename = Cwd::realpath( $filename );
+
+# FIXME verify that $filename lives in a legitimate part of the filesystem
+
+my $mtime = ( stat( $filename ) )[ 9 ];
+
+if( !defined( $mtime ) ) {
+    print STDERR $filename . ": " . $ERRNO . "\n";
+    exit( 1 );
+}
+
+$result = DBQueryWarn( "SELECT hash, UNIX_TIMESTAMP( hash_mtime ) FROM " .
+			  "blob_files WHERE filename='$filename';" );
+if( $result && $result->numrows == 1 ) {
+    my ( $file_hash, $file_mtime ) = $result->fetchrow_array();
+
+    # Don't trust any existing hash unless the mtimes match _exactly_.  Even
+    # if our mtime appears more recent than the file's, that is an
+    # excellent indication that our hash cannot be trusted (e.g. the
+    # file might have been restored from backup).
+    exit( 2 ) if( lc( $hash ) eq lc( $file_hash ) && $mtime == $file_mtime );
+}
+
+exit( 0 ) if $query;
+
+if( !open( FILE, $filename ) ) {
+    print STDERR $filename . ": " . $ERRNO . "\n";
+    exit( 1 );    
+}
+
+my $sha1 = Digest::SHA1->new;
+
+my $buffer;
+my $len;
+while( ( $len = sysread( FILE, $buffer, 0x10000 ) ) > 0 ) {
+    if( syswrite( STDOUT, $buffer, $len ) < $len ) {
+	print STDERR "error writing output\n";
+	exit( 1 );
+    }
+
+    $sha1->add( $buffer );
+}
+
+if( !defined( $len ) ) {
+    print STDERR $filename . ": " . $ERRNO . "\n";
+    exit( 1 );    
+} else {
+    my $newhash = $sha1->hexdigest;
+    DBQueryWarn( "REPLACE INTO blob_files SET filename='$filename', " .
+		 "hash='$newhash', hash_mtime=FROM_UNIXTIME($mtime);" );
+
+    exit( 0 );
+}

www/GNUmakefile.in

@@ -28,7 +28,7 @@ SUBDIRS = garcia-telemetry tutorial
 # is changed.
 # 
 all: defs.php3 dbdefs.php3 swish.conf websearch htmlinstall xmlrpc.php3 \
-	xmlrpcpipe.php3 all-subdirs
+	xmlrpcpipe.php3 blob/read.php3 all-subdirs
 
 include $(TESTBED_SRCDIR)/GNUmakerules
 
@@ -126,6 +126,8 @@ UMFILES += $(wildcard $(SRCDIR)/usermap/*.js)
 
 JSFILES += $(wildcard $(SRCDIR)/js/*.js)
 
+BLOBFILES += $(wildcard blob/*.php3)
+
 # need to make it *.gz; with simply "*", 
 # we end up sucking over "CVS"
 DOWNLOADFILES = $(wildcard $(SRCDIR)/downloads/*.gz)
@@ -182,6 +184,7 @@ ALLUM     = $(notdir $(UMFILES))
 ALLJS     = $(notdir $(JSFILES))
 ALLROBO   = $(notdir $(ROBOTRACKFILES))
 ALLWISTATS = $(notdir $(WIRELESSSTATSFILES))
+ALLBLOB   = $(notdir $(BLOBFILES))
 
 INSTALLFILES = $(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
 	$(addprefix $(INSTALL_WWWDIR)/, $(ALLFILES)) \
@@ -200,6 +203,7 @@ INSTALLFILES = $(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
 	$(addprefix $(INSTALL_WWWDIR)/robotrack/, $(ALLROBO)) \
 	$(addprefix $(INSTALL_WWWDIR)/wireless-stats/, $(ALLWISTATS)) \
 	$(addprefix $(INSTALL_WWWDIR)/autostatus-icons/, $(ALLICONS)) \
+	$(addprefix $(INSTALL_WWWDIR)/blob/, $(ALLBLOB)) \
 	$(addprefix $(INSTALL_LIBEXECDIR)/, websearch) \
 	$(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
 	$(addprefix $(INSTALL_WWWDIR)/cvsweb/, $(ALLCVSWEB)) \

www/blob/read.php3.in

+<?php
+
+if( preg_match( "(/(\w+)/([\w-]+)$)D",
+    $_SERVER[ 'PATH_INFO' ], $matches ) == 1 ) {
+    $key = $matches[ 1 ];
+    $blob = $matches[ 2 ];
+    $hash = "";
+} else {
+    header( "HTTP/1.0 404 Not found" );
+
+    echo( "<html><head><title>Not found</title></head>\n" );
+    echo( "<body><p>The URL given was not a valid blob spefication.</p></body></html>\n" );
+    return;
+}
+
+if( preg_match( "/(\w+)/D", $_GET[ 'hash' ], $matches ) == 1 ) {
+    $hash = "-h " . $matches[ 1 ];
+}
+
+header( "Content-type: application/octet-stream" );
+passthru( "@prefix@/sbin/readblob $hash $key $blob", $retval );
+
+if( $retval == 2 ) {
+    header( "HTTP/1.0 304 Not modified" );
+} else if( $retval > 0 ) {
+    header( "HTTP/1.0 403 Forbidden" );
+    header( "Content-type: text/html" );
+
+    echo( "<html><head><title>Forbidden</title></head>\n" );
+    echo( "<body><p>The blob specified could not be accessed.</p></body></html>\n" );
+}
+
+return;
+?>
+<?php
+
+if( preg_match( "(/(\w+)/([\w-]+)$)D",
+    $_SERVER[ 'PATH_INFO' ], $matches ) == 1 ) {
+    $key = $matches[ 1 ];
+    $blob = $matches[ 2 ];
+    $hash = "";
+} else {
+    header( "HTTP/1.0 404 Not found" );
+
+    echo( "<html><head><title>Not found</title></head>\n" );
+    echo( "<body><p>The URL given was not a valid blob spefication.</p></body></html>\n" );
+    return;
+}
+
+if( preg_match( "/(\w+)/D", $_GET[ 'hash' ], $matches ) == 1 ) {
+    $hash = "-h " . $matches[ 1 ];
+}
+
+header( "Content-type: application/octet-stream" );
+passthru( "@prefix@/sbin/readblob $hash $key $blob", $retval );
+
+if( $retval == 2 ) {
+    header( "HTTP/1.0 304 Not modified" );
+} else if( $retval > 0 ) {
+    header( "HTTP/1.0 403 Forbidden" );
+    header( "Content-type: text/html" );
+
+    echo( "<html><head><title>Forbidden</title></head>\n" );
+    echo( "<body><p>The blob specified could not be accessed.</p></body></html>\n" );
+}
+
+return;
+?>

xmlrpc/GNUmakefile.in

@@ -35,7 +35,7 @@ SYMLINKS        = node_admin node_reboot os_load create_image node_list \
 		  modexp expinfo node_avail tbuisp expwait template_commit \
 		  template_export template_swapin template_swapout \
 		  template_stoprun template_instantiate template_startrun \
-		  template_checkout node_avail_list
+		  template_checkout node_avail_list mkblob
 
 #
 # Force dependencies on the scripts so that they will be rerun through

xmlrpc/emulabserver.py.in

@@ -330,6 +330,7 @@ class EmulabServer:
         self.instances["node"]       = node(self);
         self.instances["elabinelab"] = elabinelab(self);
         self.instances["subboss"] = subboss(self);
+        self.instances["blob"]       = blob(self);
         return
 
     def __getattr__(self, name):
@@ -5796,6 +5797,44 @@ class template:
         return EmulabResponse(RESPONSE_SUCCESS, output=output)
     pass
 
+#
+# This class implements the server side of the XMLRPC interface to blobs.
+#
+class blob:
+    ##
+    # Initialize the object.
+    #
+    def __init__(self, server):
+        self.server   = server
+        self.readonly = server.readonly
+        self.uid      = server.uid
+        self.uid_idx  = server.uid_idx
+        self.debug    = server.debug
+        self.VERSION  = VERSION
+        return
+
+    def mkblob( self, version, argdict ):
+        if version != self.VERSION:
+            return EmulabResponse(RESPONSE_BADVERSION,
+                                  output="Client version mismatch!")
+
+        try:
+            checknologins()
+            pass
+        except NoLoginsError, e:
+            return EmulabResponse(RESPONSE_REFUSED, output=str(e))
+        
+        argerror = CheckRequiredArgs(argdict, ("filename",))
+        if (argerror):
+            return argerror
+
+        (exitval, output) = runcommand( TBDIR + "/bin/mkblob " + argdict[ "filename" ] )
+
+        if exitval:
+            return EmulabResponse( RESPONSE_ERROR, exitval >> 8, output=output )
+        else:
+            return EmulabResponse( RESPONSE_SUCCESS, value=0, output=output )
+
 #
 # Utility functions
 #

xmlrpc/script_wrapper.py.in

@@ -149,6 +149,8 @@ API = {
                             "help" : "Modify resources for run" },
     "template_stoprun"  : { "func" : "template_stoprun",
                             "help" : "Stop current experiment run" },
+    "mkblob"            : { "func" : "mkblob",
+                            "help" : "Create a new blob in the blob store" }
 };
 
 #
@@ -2601,6 +2603,43 @@ class template_stoprun:
         return
     pass
 
+#
+# mkblob
+#
+class mkblob:
+    def __init__(self, argv=None):
+        self.argv = argv;
+        return
+
+    def apply(self):
+        try:
+            opts, req_args = getopt.getopt(self.argv, "h", [ "help" ]);
+            pass
+        except getopt.error, e:
+            print e.args[0]
+            self.usage();
+            return -1;
+
+        for opt, val in opts:
+            if opt in ("-h", "--help"):
+                self.usage()
+                return 0
+
+        # Do this after so --help is seen.
+        if len(req_args) != 1:
+            self.usage();
+            return -1;
+
+        rval,response = do_method( "blob", "mkblob",
+                                   { "filename" : req_args[ 0 ] } );
+        return rval;
+
+    def usage(self):
+        print "mkblob [options] filename"
+        wrapperoptions();
+        return
+    pass
+
 #
 # Infer template guid from path
 #