Commit bf20b3e5 authored by Gary Wong's avatar Gary Wong

Add database/filesystem backend and HTTP transport for the blob store.

parent dca12fb0
This diff is collapsed.
#
# Add extenal references slots to nodes table, as for geni.
#
use strict;
use libdb;
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
if( !DBTableExists( "blobs" ) ) {
DBQueryFatal( "CREATE TABLE blobs ( " .
"uuid VARCHAR( 40 ) NOT NULL, " .
"filename TINYTEXT, " .
"PRIMARY KEY( uuid ) );" );
}
if( !DBTableExists( "blob_files" ) ) {
DBQueryFatal( "CREATE TABLE blob_files ( " .
"filename VARCHAR( 255 ) NOT NULL, " .
"hash VARCHAR( 64 ) DEFAULT NULL, " .
"hash_mtime DATETIME DEFAULT NULL, " .
"PRIMARY KEY( filename ) );" );
}
if( !DBKeyExists( "experiments", "keyhash" ) ) {
DBQueryFatal( "ALTER TABLE experiments " .
"ADD UNIQUE KEY keyhash ( keyhash );" );
}
return 0;
}
1;
......@@ -17,7 +17,7 @@ SUBDIRS = nsgen
BIN_SCRIPTS = delay_config sshtb create_image node_admin link_config \
setdest loghole webcopy linkmon_ctl snmp-if-deref.sh \
template_record spewevents \
wbts_dump
wbts_dump mkblob
SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
eventping grantnodetype import_commitlog daemon_wrapper \
opsreboot deletenode node_statewait grabwebcams \
......@@ -26,7 +26,7 @@ SBIN_SCRIPTS = vlandiff vlansync withadminprivs export_tables cvsupd.pl \
wanodecheckin wanodecreate spewimage \
anonsendmail epmodeset fixexpinfo node_traffic \
dumpdescriptor subboss_tftpboot_sync testbed-control \
archive-expinfo grantfeature emulabfeature
archive-expinfo grantfeature emulabfeature addblob readblob
WEB_SBIN_SCRIPTS= webnewnode webdeletenode webspewconlog webarchive_list \
webwanodecheckin webspewimage
......@@ -40,7 +40,7 @@ CTRLSBIN_SCRIPTS= opsdb_control.proxy daemon_wrapper
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS =
SETUID_SBIN_SCRIPTS = grabwebcams checkquota spewconlog opsdb_control suchown \
anonsendmail
anonsendmail readblob
SETUID_LIBX_SCRIPTS = xlogin
#
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use strict;
#
# Configure variables
#
my $TB = "@prefix@";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
#
# Handle command-line options.
#
sub usage() {
print STDERR "Usage: $0 <filename>\n";
exit( 1 );
}
usage() unless @ARGV == 1;
my ( $filename ) = @ARGV;
#
# Must taint check!
#
if ($filename =~ /^([-\w #%&*+,.\/:;=?@\[\\\]^{|}]+)$/) {
$filename = $1;
}
else {
print STDERR "Bad character in filename\n";
exit( 1 );
}
# We could use MySQL's UUID() function, but if we call it in the INSERT it
# becomes a pain to retrieve it. So we do the job ourselves.
my $uuid = `@UUIDGEN@`;
chomp $uuid;
my $result = DBQueryWarn( "INSERT INTO blobs SET uuid='$uuid', " .
"filename='$filename';" );
unless( $result ) {
print STDERR "Could not insert record.\n";
exit( 1 );
}
print "$uuid\n";
exit( 0 );
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2010 University of Utah and the Flux Group.
# All rights reserved.
#
use Cwd;
use Digest::SHA1;
use English;
use Getopt::Std;
use POSIX qw( getuid setuid );
use strict;
#
# Configure variables
#
my $TB = "@prefix@";
my $FSDIR_PROJ = "@FSDIR_PROJ@";
my $FSDIR_GROUPS = "@FSDIR_GROUPS@";
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
if ($EUID != 0) {
# We don't want to run this script unless its the real version.
die("*** $0:\n".
" Must be root! Maybe its a development version?\n");
}
# This script is setuid, so please do not run it as root. Hard to track
# what has happened.
if ($UID == 0) {
die("*** $0:\n".
" Please do not run this as root! Its already setuid!\n");
}
# Temporarily drop privileges as soon as possible.
$EUID = $UID;
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
#
# Handle command-line options.
#
sub usage() {
print STDERR "Usage: $0 [-h hash] [-q] <key> <blob>\n";
exit( 1 );
}
my $hash = 0;
my $query = 0;
my %options = ();
if (! getopts("h:q", \%options)) {
usage();
}
if ($options{q}) {
$query = 1;
}
if ($options{'h'}) {
$hash = $options{'h'};
}
usage() unless @ARGV == 2;
my ( $key, $blob ) = @ARGV;
#
# Must taint check!
#
if ($key =~ /^([-\w]+)$/) {
$key = $1;
}
else {
print STDERR "Bad data in argument: $key\n";
exit( 1 );
}
if ($blob =~ /^([-\w]+)$/) {
$blob = $1;
}
else {
print STDERR "Bad data in argument: $blob\n";
exit( 1 );
}
my $result = DBQueryWarn( "SELECT groups.unix_gid FROM experiments, groups " .
"WHERE experiments.keyhash='$key' AND " .
"experiments.gid_idx=groups.gid_idx;" );
if( !$result || $result->numrows != 1 ) {
print STDERR "Could not resolve key\n";
exit( 1 );
}
my ( $unix_gid ) = $result->fetchrow_array();
# Temporarily reacquire privileges.
$EUID = 0;
# Add the supplementary group ID.
$EGID = $EGID . " " . $unix_gid;
# Now we can permanently drop all privileges.
setuid( getuid() );
$result = DBQueryWarn( "SELECT filename FROM blobs WHERE uuid='$blob';" );
unless( $result && $result->numrows == 1 ) {
print STDERR "could not resolve $blob\n";
exit( 1 );
}
my ( $filename ) = $result->fetchrow_array();
$filename = Cwd::realpath( $filename );
# FIXME verify that $filename lives in a legitimate part of the filesystem
my $mtime = ( stat( $filename ) )[ 9 ];
if( !defined( $mtime ) ) {
print STDERR $filename . ": " . $ERRNO . "\n";
exit( 1 );
}
$result = DBQueryWarn( "SELECT hash, UNIX_TIMESTAMP( hash_mtime ) FROM " .
"blob_files WHERE filename='$filename';" );
if( $result && $result->numrows == 1 ) {
my ( $file_hash, $file_mtime ) = $result->fetchrow_array();
# Don't trust any existing hash unless the mtimes match _exactly_. Even
# if our mtime appears more recent than the file's, that is an
# excellent indication that our hash cannot be trusted (e.g. the
# file might have been restored from backup).
exit( 2 ) if( lc( $hash ) eq lc( $file_hash ) && $mtime == $file_mtime );
}
exit( 0 ) if $query;
if( !open( FILE, $filename ) ) {
print STDERR $filename . ": " . $ERRNO . "\n";
exit( 1 );
}
my $sha1 = Digest::SHA1->new;
my $buffer;
my $len;
while( ( $len = sysread( FILE, $buffer, 0x10000 ) ) > 0 ) {
if( syswrite( STDOUT, $buffer, $len ) < $len ) {
print STDERR "error writing output\n";
exit( 1 );
}
$sha1->add( $buffer );
}
if( !defined( $len ) ) {
print STDERR $filename . ": " . $ERRNO . "\n";
exit( 1 );
} else {
my $newhash = $sha1->hexdigest;
DBQueryWarn( "REPLACE INTO blob_files SET filename='$filename', " .
"hash='$newhash', hash_mtime=FROM_UNIXTIME($mtime);" );
exit( 0 );
}
......@@ -28,7 +28,7 @@ SUBDIRS = garcia-telemetry tutorial
# is changed.
#
all: defs.php3 dbdefs.php3 swish.conf websearch htmlinstall xmlrpc.php3 \
xmlrpcpipe.php3 all-subdirs
xmlrpcpipe.php3 blob/read.php3 all-subdirs
include $(TESTBED_SRCDIR)/GNUmakerules
......@@ -126,6 +126,8 @@ UMFILES += $(wildcard $(SRCDIR)/usermap/*.js)
JSFILES += $(wildcard $(SRCDIR)/js/*.js)
BLOBFILES += $(wildcard blob/*.php3)
# need to make it *.gz; with simply "*",
# we end up sucking over "CVS"
DOWNLOADFILES = $(wildcard $(SRCDIR)/downloads/*.gz)
......@@ -182,6 +184,7 @@ ALLUM = $(notdir $(UMFILES))
ALLJS = $(notdir $(JSFILES))
ALLROBO = $(notdir $(ROBOTRACKFILES))
ALLWISTATS = $(notdir $(WIRELESSSTATSFILES))
ALLBLOB = $(notdir $(BLOBFILES))
INSTALLFILES = $(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
$(addprefix $(INSTALL_WWWDIR)/, $(ALLFILES)) \
......@@ -200,6 +203,7 @@ INSTALLFILES = $(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
$(addprefix $(INSTALL_WWWDIR)/robotrack/, $(ALLROBO)) \
$(addprefix $(INSTALL_WWWDIR)/wireless-stats/, $(ALLWISTATS)) \
$(addprefix $(INSTALL_WWWDIR)/autostatus-icons/, $(ALLICONS)) \
$(addprefix $(INSTALL_WWWDIR)/blob/, $(ALLBLOB)) \
$(addprefix $(INSTALL_LIBEXECDIR)/, websearch) \
$(addprefix $(INSTALL_SBINDIR)/, htmlinstall) \
$(addprefix $(INSTALL_WWWDIR)/cvsweb/, $(ALLCVSWEB)) \
......
<?php
if( preg_match( "(/(\w+)/([\w-]+)$)D",
$_SERVER[ 'PATH_INFO' ], $matches ) == 1 ) {
$key = $matches[ 1 ];
$blob = $matches[ 2 ];
$hash = "";
} else {
header( "HTTP/1.0 404 Not found" );
echo( "<html><head><title>Not found</title></head>\n" );
echo( "<body><p>The URL given was not a valid blob spefication.</p></body></html>\n" );
return;
}
if( preg_match( "/(\w+)/D", $_GET[ 'hash' ], $matches ) == 1 ) {
$hash = "-h " . $matches[ 1 ];
}
header( "Content-type: application/octet-stream" );
passthru( "@prefix@/sbin/readblob $hash $key $blob", $retval );
if( $retval == 2 ) {
header( "HTTP/1.0 304 Not modified" );
} else if( $retval > 0 ) {
header( "HTTP/1.0 403 Forbidden" );
header( "Content-type: text/html" );
echo( "<html><head><title>Forbidden</title></head>\n" );
echo( "<body><p>The blob specified could not be accessed.</p></body></html>\n" );
}
return;
?>
<?php
if( preg_match( "(/(\w+)/([\w-]+)$)D",
$_SERVER[ 'PATH_INFO' ], $matches ) == 1 ) {
$key = $matches[ 1 ];
$blob = $matches[ 2 ];
$hash = "";
} else {
header( "HTTP/1.0 404 Not found" );
echo( "<html><head><title>Not found</title></head>\n" );
echo( "<body><p>The URL given was not a valid blob spefication.</p></body></html>\n" );
return;
}
if( preg_match( "/(\w+)/D", $_GET[ 'hash' ], $matches ) == 1 ) {
$hash = "-h " . $matches[ 1 ];
}
header( "Content-type: application/octet-stream" );
passthru( "@prefix@/sbin/readblob $hash $key $blob", $retval );
if( $retval == 2 ) {
header( "HTTP/1.0 304 Not modified" );
} else if( $retval > 0 ) {
header( "HTTP/1.0 403 Forbidden" );
header( "Content-type: text/html" );
echo( "<html><head><title>Forbidden</title></head>\n" );
echo( "<body><p>The blob specified could not be accessed.</p></body></html>\n" );
}
return;
?>
......@@ -35,7 +35,7 @@ SYMLINKS = node_admin node_reboot os_load create_image node_list \
modexp expinfo node_avail tbuisp expwait template_commit \
template_export template_swapin template_swapout \
template_stoprun template_instantiate template_startrun \
template_checkout node_avail_list
template_checkout node_avail_list mkblob
#
# Force dependencies on the scripts so that they will be rerun through
......
......@@ -330,6 +330,7 @@ class EmulabServer:
self.instances["node"] = node(self);
self.instances["elabinelab"] = elabinelab(self);
self.instances["subboss"] = subboss(self);
self.instances["blob"] = blob(self);
return
def __getattr__(self, name):
......@@ -5796,6 +5797,44 @@ class template:
return EmulabResponse(RESPONSE_SUCCESS, output=output)
pass
#
# This class implements the server side of the XMLRPC interface to blobs.
#
class blob:
##
# Initialize the object.
#
def __init__(self, server):
self.server = server
self.readonly = server.readonly
self.uid = server.uid
self.uid_idx = server.uid_idx
self.debug = server.debug
self.VERSION = VERSION
return
def mkblob( self, version, argdict ):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!")
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
argerror = CheckRequiredArgs(argdict, ("filename",))
if (argerror):
return argerror
(exitval, output) = runcommand( TBDIR + "/bin/mkblob " + argdict[ "filename" ] )
if exitval:
return EmulabResponse( RESPONSE_ERROR, exitval >> 8, output=output )
else:
return EmulabResponse( RESPONSE_SUCCESS, value=0, output=output )
#
# Utility functions
#
......
......@@ -149,6 +149,8 @@ API = {
"help" : "Modify resources for run" },
"template_stoprun" : { "func" : "template_stoprun",
"help" : "Stop current experiment run" },
"mkblob" : { "func" : "mkblob",
"help" : "Create a new blob in the blob store" }
};
#
......@@ -2601,6 +2603,43 @@ class template_stoprun:
return
pass
#
# mkblob
#
class mkblob:
def __init__(self, argv=None):
self.argv = argv;
return
def apply(self):
try:
opts, req_args = getopt.getopt(self.argv, "h", [ "help" ]);
pass
except getopt.error, e:
print e.args[0]
self.usage();
return -1;
for opt, val in opts:
if opt in ("-h", "--help"):
self.usage()
return 0
# Do this after so --help is seen.
if len(req_args) != 1:
self.usage();
return -1;
rval,response = do_method( "blob", "mkblob",
{ "filename" : req_args[ 0 ] } );
return rval;
def usage(self):
print "mkblob [options] filename"
wrapperoptions();
return
pass
#
# Infer template guid from path
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment