Skip to content

GitLab

Commit afc987a4 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

Merge remote-tracking branch 'central/imageversion'

parents 2d43ff3f 06591f8a
Hide whitespace changes
Inline Side-by-side
backend/newimageid_ez.in
View file @ afc987a4
......@@ -34,15 +34,16 @@ use Cwd qw(realpath);
#
sub usage()
{
print("Usage: newimageid [-v] [-a] [-s] <xmlfile>\n");
print("Usage: newimageid [-v] [-a] [-s] [-t target] <xmlfile>\n");
exit(-1);
}
my $optlist = "dvfas";
my $optlist = "dvfast:";
my $debug = 0;
my $force = 0;
my $verify = 0; # Check data and return status only.
my $allpc = 0; # insert mappings for all pc types.
my $skipadmin = 0; # Skip SLOT_ADMINONLY checks.
my $target;
#
# Configure variables
......@@ -52,6 +53,8 @@ my $TBOPS = "@TBOPSEMAIL@";
my $TBAUDIT = "@TBAUDITEMAIL@";
my $TBGROUP_DIR = "@GROUPSROOT_DIR@";
my $TBPROJ_DIR = "@PROJROOT_DIR@";
my $CREATEIMAGE = "$TB/bin/create_image";
my $CLONEIMAGE = "$TB/sbin/clone_image";
#
# Untaint the path
......@@ -107,6 +110,20 @@ if (defined($options{"s"})) {
if (defined($options{"a"})) {
$allpc = 1;
}
if (defined($options{"t"})) {
$target = $options{"t"};
# Might be an EC2 target. Also need to untaint for below.
if ($target =~ /^([-\w\@\.\+]+@[-\w\@\.\+]+)$/) {
$target = $1;
}
else {
my $node = Node->Lookup($target);
if (!defined($node)) {
fatal("No such node!");
}
$target = $node->node_id();
}
}
if (@ARGV != 1) {
usage();
}
......@@ -727,7 +744,8 @@ UserError($usrerr)
fatal("Could not create new Image!")
if (!defined($new_image));
my $imageid = $new_image->imageid();
my $imageid = $new_image->imageid();
my $imagepid = $new_image->pid();
#
# Insert a submap entry.
......@@ -748,6 +766,27 @@ if (defined($parentos)) {
}
}
#
# If a target was specified, fire off image creation. It will return
# quickly enough (going into the background) that we can wait here for
# it.
#
if (defined($target)) {
#
# Use clone_image for a node, create_image for an EC2 target. We use
# clone cause it will do the provenance, so we do not have to duplicate
# that stuff here. Then it just calls create_image.
#
if ($target =~ /^.*@.*$/) {
system("$CREATEIMAGE -p $imagepid $imageid '$target'"); }
else {
# Must be an imagename, not and imageid.
system("$CLONEIMAGE $imagename '$target'");
}
if ($?) {
fatal("Could not image capture from $target");
}
}
# The web interface requires this line to be printed.
print "IMAGE $imagename/$imageid has been created\n";
......
clientside/lib/tmcd/tmcd.h
View file @ afc987a4
......@@ -46,4 +46,4 @@
* it in clientside/tmcc/common/libsetup.pm!
*/
#define DEFAULT_VERSION 2
#define CURRENT_VERSION 38
#define CURRENT_VERSION 39
clientside/os/GNUmakefile.in
View file @ afc987a4
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -75,7 +75,10 @@ endif
$(MAKE) -C genhostsfile client-install
ifeq ($(SYSTEM),FreeBSD)
$(MAKE) -C growdisk client-install
$(INSTALL_PROGRAM) $(SRCDIR)/create-image $(LBINDIR)/
$(INSTALL_PROGRAM) $(SRCDIR)/create-swapimage $(LBINDIR)/
endif
$(INSTALL_PROGRAM) $(SRCDIR)/create-versioned-image $(LBINDIR)/
mfs:
$(MAKE) -C growdisk client
......@@ -83,6 +86,9 @@ mfs:
$(MAKE) -C growdisk client
$(MAKE) -C imagezip client
$(MAKE) -C frisbee.redux client
$(INSTALL_PROGRAM) $(SRCDIR)/create-image $(LBINDIR)/
$(INSTALL_PROGRAM) $(SRCDIR)/create-versioned-image $(LBINDIR)/
$(INSTALL_PROGRAM) $(SRCDIR)/create-swapimage $(LBINDIR)/
subboss: subboss-subdirs
......
os/create-image → clientside/os/create-image
View file @ afc987a4
#!/usr/bin/perl -wT
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
......@@ -53,6 +53,7 @@ delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
my $sudo = "";
my $zipper = "/usr/local/bin/imagezip";
my $uploader = "/usr/local/bin/frisupload";
my $xenscript = "/usr/local/bin/create-xen-image";
my $slice = "";
my $device;
my $filename;
......@@ -69,6 +70,19 @@ if ($EUID != 0) {
}
}
#
# A newer server side is going to invoke this script for XEN nodes, to be
# backwards compatible with older XEN client sides that had its own version
# of create-image. It is now called create-xen-image, so call that script,
# which conveniently is argument compatible with this script. This test for
# the file is kinda bogus, but this script does not include libsetup, which
# hides that. Not sure why we do not include libsetup (ask Mike).
#
if ($^O eq 'linux' && -e "/etc/emulab/genvmtype") {
exec $xenscript, @ARGV;
die("Could not exec $xenscript");
}
# Frisbee master server params
my $iserver = "boss"; # XXX
my $imageid;
......
os/create-versioned-image → clientside/os/create-versioned-image
View file @ afc987a4
#!/usr/bin/perl -wT
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
......@@ -88,17 +88,23 @@ use Getopt::Std;
# Partition on DISK from which to load image. Used for imagezip -s option.
# If not set or set to zero, then it is a whole-disk image.
#
# IZOPTS=<string>
# Additional options for imagezip.
#
# PROXY=<vnodeid>
# The proxy argument for use on XEN, when acting on behalf of a container.
#
sub usage()
{
print STDERR
"Usage:\n".
"create-versioned-image [-nv] -f param-file\n".
"create-versioned-image [-nvx] -f param-file\n".
" or\n".
"create-versioned-image [-nv] KEY=VALUE ...\n";
"create-versioned-image [-nv] [-x vnode_id] KEY=VALUE ...\n";
exit(-1);
}
my $optlist = "f:nv";
my $optlist = "f:nvx:";
#
# Turn off line buffering on output
......@@ -114,25 +120,14 @@ delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# No configure vars.
#
my $sudo;
my $sudo = "";
my $zipper = "/usr/local/bin/imagezip";
my $uploader = "/usr/local/bin/frisupload";
my $frisbee = "/usr/local/bin/frisbee";
my $localdir = "/local";
my $impotent = 0;
my $verbose = 0;
#
# We do not create relocations (-N) in the resulting image right now for a
# couple of reasons. One is that we never did relocation support for GRUB
# partition boot code, so modern Linux images would not have relocations
# anyway. For FreeBSD this does mean that we cannot relocate them (we use
# a relocation for correct disklabel construction), but we never really
# took advantage of this anyway. The second reason is that ranges with
# relocations are always considered different for hash comparisons, so an
# otherwise empty FreeBSD delta image would have 64K of data in it.
#
my $zipperopts = "-N";
my $isxen = 0;
sub process_image($);
sub mysystem($);
......@@ -148,6 +143,13 @@ sub map_diskname($)
my ($dev) = @_;
my ($dtype, $dunit);
#
# When called on XEN, the diskname is correct, and in fact we will
# just mess it up.
#
return $dev
if ($isxen);
# strip off /dev/ if it is there
$dev =~ s/^\/dev\///;
......@@ -182,13 +184,7 @@ sub map_diskname($)
}
}
$dev = "/dev/$dtype$dunit";
if (-r "$dev") {
print STDERR "Could not read $dev\n";
return undef;
}
return $dev;
return "/dev/$dtype$dunit";
}
sub parse_params(@)
......@@ -246,6 +242,25 @@ sub parse_params(@)
$iinfo{$iid}{'nsigfile'} = $val;
next;
}
if ($key eq "izopts") {
#
# No spaces in string, so options are encoded; e.g.:
# -N -z 9 -d -a SHA1
# would be encoded as:
# N,z=9,d,a=SHA1
# We unencode them here.
#
my $optstr = "";
foreach my $opt (split(',', $val)) {
$optstr .= " -" . join(' ', split('=', $opt));
}
$iinfo{$iid}{'izopts'} = $optstr;
next;
}
if ($key eq "proxy") {
$iinfo{$iid}{'proxy'} = $val;
next;
}
} else {
print STDERR "Bogus parameter: '$kv'\n";
$errors++;
......@@ -259,11 +274,16 @@ sub parse_params(@)
}
}
for my $path (qw#/usr/local/bin /usr/bin#) {
#
# If we are running as a user, then we will need sudo
#
if ($EUID != 0) {
for my $path (qw#/usr/local/bin /usr/bin#) {
if (-e "$path/sudo") {
$sudo = "$path/sudo";
last;
$sudo = "$path/sudo";
last;
}
}
}
#
......@@ -280,6 +300,10 @@ if (defined($options{"n"})) {
if (defined($options{"v"})) {
$verbose = 1;
}
if (defined($options{"x"})) {
$isxen = 1;
$localdir = "/capture/" . $options{"x"} . "/frisbee";
}
if (defined($options{"f"})) {
my $pfile = $options{"f"};
if ($pfile =~ /^(\/tmp\/[-\w\.]+)$/) {
......@@ -335,7 +359,7 @@ foreach my $iid (sort keys %iinfo) {
# For method=frisbee, the actual files will be here.
# For method=file, a symlink to the actual file will be here.
#
if (mysystem("$sudo mkdir -p $localdir")) {
if (! -e $localdir && mysystem("$sudo mkdir -p $localdir")) {
print STDERR "Could not create $localdir\n";
exit(1);
}
......@@ -351,12 +375,18 @@ if (mysystem("$sudo mkdir -p $localdir")) {
# memory filesystem is too much for our older machines--let's go
# with 64MB for now.
#
my $MEMFS_SIZE = (64 * 1024 * 1024);
my $MEMFS_SIZE = "64m";
if ($dofrisbee) {
if ($^O eq 'linux') {
die "No can do Linux right now!\n";
if (!$isxen &&
mysystem("$sudo mount -t tmpfs -o size=$MEMFS_SIZE tmpfs $localdir")) {
print STDERR "Could not create $MEMFS_SIZE byte local MFS\n";
# XXX try NFS instead
exit(1);
}
} else {
if (mysystem("$sudo mdconfig -a -t swap -s 64m -u 4") ||
if (mysystem("$sudo mdconfig -a -t swap -s $MEMFS_SIZE -u 4") ||
mysystem("$sudo newfs -b 8192 -i 25000 -o space /dev/md4") ||
mysystem("$sudo mount /dev/md4 $localdir")) {
print STDERR "Could not create $MEMFS_SIZE byte local MFS\n";
......@@ -384,7 +414,9 @@ foreach my $iid (sort keys %iinfo) {
#
if ($dofrisbee) {
if ($^O eq 'linux') {
print STDERR "No can do Linux right now!\n";
if (!$isxen && mysystem("$sudo umount $localdir")) {
print STDERR "WARNING: could not destroy local MFS\n";
}
} else {
if (mysystem("$sudo umount $localdir") ||
mysystem("$sudo mdconfig -d -u 4")) {
......@@ -527,25 +559,28 @@ sub process_image($)
# Fire off the command:
#
# file:
# imagezip [-s $part] [-H $sigfile] [-U $nsigfile] $disk $ifile
# imagezip $izopts [-s $part] [-H $sigfile] [-U $nsigfile] $disk $ifile
#
# frisbee:
# imagezip [-s $part] [-H $sigfile] [-U $localdir/$nsigfile] $disk - | \
# imagezip $izopts [-s $part] [-H $sigfile] [-U $localdir/$nsigfile] $disk - | \
# frisupload -S $server -F $ifile -
# [ cat $localdir/$nsigfile | frisupload -S $server -F $nsigfile ]
#
my $cmd = "$zipper $zipperopts";
my $cmd = "$sudo $zipper";
if ($verbose) {
$cmd .= " -o";
}
if (exists($iinfo{$iid}{'izopts'})) {
$cmd .= $iinfo{$iid}{'izopts'};
}
if ($iinfo{$iid}{'part'} != 0) {
$cmd .= " -s " . $iinfo{$iid}{'part'};
}
if (exists($iinfo{$iid}{'sigfile'})) {
$cmd .= " -H /local/sigfile";
$cmd .= " -H $localdir/sigfile";
}
if (exists($iinfo{$iid}{'nsigfile'})) {
$cmd .= " -U /local/nsigfile";
$cmd .= " -U $localdir/nsigfile";
}
$cmd .= " " . $iinfo{$iid}{'disk'};
......@@ -554,14 +589,32 @@ sub process_image($)
$cmd .= " $image";
} elsif ($iinfo{$iid}{'method'} eq "frisbee") {
my $srv = $iinfo{$iid}{'server'};
$cmd .= " - | $uploader -S $srv -F $image -";
# use basic shell sleezy trick to capture exit status from imagezip
$cmd = "( $cmd - || echo \$? > $localdir/imagezip.stat )";
$cmd .= " | $uploader -S $srv -F $image";
if (exists($iinfo{$iid}{'proxy'})) {
$cmd .= " -P " . $iinfo{$iid}{'proxy'};
}
$cmd .= " - ";
}
if (mysystem("$sudo $cmd")) {
if (mysystem("$cmd") || -e "$localdir/imagezip.stat") {
my $stat = sprintf("0x%04x", $?);
my $izstat = 0;
if (-e "$localdir/imagezip.stat") {
$izstat = `cat $localdir/imagezip.stat`;
chomp($izstat);
}
$izstat = sprintf("0x%04x", $izstat);
print STDERR "*** Failed to create image!\n";
print STDERR " command: '$sudo $cmd'\n";
print STDERR " status: $stat\n";
print STDERR " command: '$cmd'\n";
print STDERR " status: $stat\n";
print STDERR " izstatus: $izstat\n"
if ($izstat);
exit(3);
}
......@@ -576,7 +629,7 @@ sub process_image($)
}
}
mysystem("$sudo rm /local/*");
mysystem("$sudo rm $localdir/*");
}
sub mysystem($)
......
clientside/os/frisbee.redux/GNUmakefile.in
View file @ afc987a4
#
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -280,8 +280,8 @@ subboss-install: subboss
client: frisbee frisupload
client-install: client
$(INSTALL_PROGRAM) frisbee $(DESTDIR)$(CLIENT_BINDIR)
$(INSTALL_PROGRAM) frisupload $(DESTDIR)$(CLIENT_BINDIR)
$(INSTALL_PROGRAM) frisbee $(DESTDIR)/usr/local/bin
$(INSTALL_PROGRAM) frisupload $(DESTDIR)/usr/local/bin
clean:
/bin/rm -f *.o *.a *.debug
......
clientside/os/frisbee.redux/config_emulab.c
View file @ afc987a4
......@@ -98,6 +98,26 @@ static int INELABINELAB = ELABINELAB;
static int INELABINELAB = 0;
#endif
/*
* An Emulab image ID string looks like:
* [<pid>]<imagename>[<vers>][<meta>]
* where:
* <pid> is the project
* <imagename> is the image identifier string
* <vers> is an image version number (not yet implemented)
* <meta> is a string indicating that this is not the actual image,
* rather it is metadata file associated with the image.
* By convention, the string is the filename extension used
* for the metadata file in question. Currently, the only
* metadata string is 'sig' indicating that this is an image
* signature file.
* Each of these fields has a separator character distinguishing the
* start of the field. These are defined here.
*/
#define IID_SEP_NAME '/'
#define IID_SEP_VERS ':'
#define IID_SEP_META ','
static uint64_t put_maxsize = 10000000000ULL; /* zero means no limit */
static uint32_t put_maxwait = 2000; /* zero means no limit */
static uint32_t put_maxiwait = 120; /* zero means no limit */
......@@ -986,6 +1006,86 @@ can_access(int imageidx, struct emulab_ha_extra_info *ei, int upload)
return 0;
}
/*
* Parse an Emulab image ID string:
*
* [<pid>]<imagename>[<vers>][<meta>]
*
* into its component parts. Returned components are malloced strings and
* need to be freed by the caller.
*
* Note that right now there are no errors. Even with a malformed string,
* we will return 'emulab-ops' as 'pid' and the given string as 'imagename'.
*/
static void
parse_imageid(char *str, char **pidp, char **namep, char **versp, char **metap)
{
char *ipid, *iname, *ivers, *imeta;
ipid = mystrdup(str);
iname = index(ipid, IID_SEP_NAME);
if (iname == NULL) {
iname = ipid;
ipid = mystrdup("emulab-ops");
} else {
*iname = '\0';
iname = mystrdup(iname+1);
}
ivers = index(iname, IID_SEP_VERS);
if (ivers) {
char *eptr;
/* If we can't convert to a number, consider it part of name */
if (strtol(ivers+1, &eptr, 10) == 0 && eptr == ivers+1) {
ivers = NULL;
} else {
*ivers = '\0';
ivers = mystrdup(ivers+1);
}
}
imeta = index(ivers ? ivers : iname, IID_SEP_META);
if (imeta != NULL) {
*imeta = '\0';
imeta = mystrdup(imeta+1);
}
*pidp = ipid;
*namep = iname;
*versp = ivers;
*metap = imeta;
}
static char *
build_imageid(char *ipid, char *iname, char *ivers, char *imeta)
{
char *iid;
int len;
len = strlen(ipid) + strlen(iname) + 2;
if (ivers)
len += strlen(ivers) + 1;
if (imeta)
len += strlen(imeta) + 1;
iid = mymalloc(len);
strcpy(iid, ipid);
len = strlen(ipid);
iid[len++] = IID_SEP_NAME;
strcpy(&iid[len], iname);
len += strlen(iname);
if (ivers) {
iid[len++] = IID_SEP_VERS;
strcpy(&iid[len], ivers);
len += strlen(ivers);
}
if (imeta) {
iid[len++] = IID_SEP_META;
strcpy(&iid[len], imeta);
len += strlen(imeta);
}
return iid;
}
/*
* Find all images (imageid==NULL) or a specific image (imageid!=NULL)
* that a particular node can access for GET/PUT. At any time, a node is
......@@ -1036,7 +1136,8 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
MYSQL_ROW row;
char *node, *proxy, *role = NULL;
int i, j, nrows;
char *wantpid = NULL, *wantname = NULL;
char *wantpid = NULL, *wantname = NULL, *wantvers = NULL;
char *wantmeta = NULL;
struct config_host_authinfo *get = NULL, *put = NULL;
struct emulab_ha_extra_info *ei = NULL;
int imageidx = 0, ngids, igids[2];
......@@ -1307,15 +1408,8 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
* pid and imagename from the ID.
*/
if (imageid != NULL) {
wantpid = mystrdup(imageid);
wantname = index(wantpid, '/');
if (wantname == NULL) {
wantname = wantpid;
wantpid = mystrdup("emulab-ops");
} else {
*wantname = '\0';
wantname = mystrdup(wantname+1);
}
parse_imageid(imageid,
&wantpid, &wantname, &wantvers, &wantmeta);
imageidx = emulab_imageid(wantpid, wantname);
if (imageidx == 0)
goto done;
......@@ -1327,37 +1421,87 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
if (imageid != NULL) {
/* Interested in a specific image */
if (can_access(imageidx, ei, 1)) {
res = mydb_query("SELECT pid,gid,imagename,"
" path,imageid"
" FROM images WHERE"
" pid='%s'"
" AND imagename='%s'",
5, wantpid, wantname);
if (wantvers) {
res = mydb_query("SELECT i.pid,i.gid,"
"i.imagename,v.path,"
"i.imageid,v.version"
" FROM images as i "
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version='%s' "
"WHERE i.pid='%s'"
" AND i.imagename='%s'",
6, wantvers, wantpid, wantname);
}
else {
res = mydb_query("SELECT i.pid,i.gid,"
"i.imagename,v.path,"
"i.imageid,v.version"
" FROM images as i "
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
"WHERE i.pid='%s'"
" AND i.imagename='%s'",
6, wantpid, wantname);
}
} else {
/*
* Pid of expt must be same as pid of image
* and gid the same or image "shared".
*/
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"