All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit 3a21f39e authored by Leigh B Stoller's avatar Leigh B Stoller

Directory based image paths.

Soon, we will have images with both full images and deltas, for the same
image version. To make this possible, the image path will now be a
directory instead of a file, and all of the versions (ndz,sig,sha1,delta)
files will reside in the directory.

A new config variable IMAGEDIRECTORIES turns this on, there is also a check
for the ImageDiretories feature. This is applied only when a brand new
image is created; a clone version of the image inherits the path it started
with. Yes, you can have a mix of directory based and file based image
descriptors.

When it is time to convert all images over, there is a script called
imagetodir that will go through all image descriptors, create the
directory, move/rename all the files, and update the descriptors.
Ultimately, we will not support file based image paths.

I also added versioning to the image metadata descriptors so that going
forward, old clients can handle a descriptor from a new server.
parent 2baf7655
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2012 University of Utah and the Flux Group. # Copyright (c) 2000-2015 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -490,7 +490,14 @@ elsif (! $isadmin) { ...@@ -490,7 +490,14 @@ elsif (! $isadmin) {
UserError("Path: Invalid Path"); UserError("Path: Invalid Path");
} }
} }
if ($newimageid_args{"path"} =~ /\/$/) {
if (-e $newimageid_args{"path"} && ! -d $newimageid_args{"path"}) {
UserError("Path: invalid path, it should be a directory");
}
}
elsif (-d $newimageid_args{"path"} =~ /\/$/) {
UserError("Path: invalid path, its a directory");
}
# #
# See what node types this image will work on. Must be at least one! # See what node types this image will work on. Must be at least one!
# #
......
...@@ -541,7 +541,12 @@ elsif (! $isadmin) { ...@@ -541,7 +541,12 @@ elsif (! $isadmin) {
UserError("Path: Invalid Path"); UserError("Path: Invalid Path");
} }
} }
if (-d $newimageid_args{"path"}) { if ($newimageid_args{"path"} =~ /\/$/) {
if (-e $newimageid_args{"path"} && ! -d $newimageid_args{"path"}) {
UserError("Path: invalid path, it should be a directory");
}
}
elsif (-d $newimageid_args{"path"} =~ /\/$/) {
UserError("Path: invalid path, its a directory"); UserError("Path: invalid path, its a directory");
} }
......
...@@ -670,6 +670,7 @@ NOVIRTNFSMOUNTS ...@@ -670,6 +670,7 @@ NOVIRTNFSMOUNTS
PROFILEVERSIONS PROFILEVERSIONS
IMAGEDELTAS IMAGEDELTAS
IMAGEPROVENANCE IMAGEPROVENANCE
IMAGEDIRECTORIES
NFSMAPTOUSER NFSMAPTOUSER
IPV6_SUBNET_PREFIX IPV6_SUBNET_PREFIX
IPV6_ENABLED IPV6_ENABLED
...@@ -5131,6 +5132,7 @@ MANAGEMENT_NETMASK="255.255.255.0" ...@@ -5131,6 +5132,7 @@ MANAGEMENT_NETMASK="255.255.255.0"
MANAGEMENT_ROUTER="10.249.249.253" MANAGEMENT_ROUTER="10.249.249.253"
NFSMAPTOUSER="root" NFSMAPTOUSER="root"
IMAGEPROVENANCE=0 IMAGEPROVENANCE=0
IMAGEDIRECTORIES=0
IMAGEDELTAS=0 IMAGEDELTAS=0
PROFILEVERSIONS=0 PROFILEVERSIONS=0
NOVIRTNFSMOUNTS=0 NOVIRTNFSMOUNTS=0
......
...@@ -297,6 +297,7 @@ AC_SUBST(IPV6_ENABLED) ...@@ -297,6 +297,7 @@ AC_SUBST(IPV6_ENABLED)
AC_SUBST(IPV6_SUBNET_PREFIX) AC_SUBST(IPV6_SUBNET_PREFIX)
AC_SUBST(NFSMAPTOUSER) AC_SUBST(NFSMAPTOUSER)
AC_SUBST(IMAGEPROVENANCE) AC_SUBST(IMAGEPROVENANCE)
AC_SUBST(IMAGEDIRECTORIES)
AC_SUBST(IMAGEDELTAS) AC_SUBST(IMAGEDELTAS)
AC_SUBST(PROFILEVERSIONS) AC_SUBST(PROFILEVERSIONS)
AC_SUBST(NOVIRTNFSMOUNTS) AC_SUBST(NOVIRTNFSMOUNTS)
...@@ -457,6 +458,7 @@ MANAGEMENT_NETMASK="255.255.255.0" ...@@ -457,6 +458,7 @@ MANAGEMENT_NETMASK="255.255.255.0"
MANAGEMENT_ROUTER="10.249.249.253" MANAGEMENT_ROUTER="10.249.249.253"
NFSMAPTOUSER="root" NFSMAPTOUSER="root"
IMAGEPROVENANCE=0 IMAGEPROVENANCE=0
IMAGEDIRECTORIES=0
IMAGEDELTAS=0 IMAGEDELTAS=0
PROFILEVERSIONS=0 PROFILEVERSIONS=0
NOVIRTNFSMOUNTS=0 NOVIRTNFSMOUNTS=0
......
...@@ -37,6 +37,7 @@ use EmulabConstants; ...@@ -37,6 +37,7 @@ use EmulabConstants;
use libtestbed; use libtestbed;
use English; use English;
use Data::Dumper; use Data::Dumper;
use File::Basename;
use overload ('""' => 'Stringify'); use overload ('""' => 'Stringify');
# Configure variables # Configure variables
...@@ -86,6 +87,7 @@ sub BlessRow($$) ...@@ -86,6 +87,7 @@ sub BlessRow($$)
my $self = {}; my $self = {};
my $imageid = $row->{"imageid"}; my $imageid = $row->{"imageid"};
$self->{'IMAGE'} = $row; $self->{'IMAGE'} = $row;
$self->{'HASH'} = {};
bless($self, $class); bless($self, $class);
return $self; return $self;
...@@ -259,8 +261,21 @@ AUTOLOAD { ...@@ -259,8 +261,21 @@ AUTOLOAD {
# A DB row proxy method call. # A DB row proxy method call.
if (exists($self->{'IMAGE'}->{$name})) { if (exists($self->{'IMAGE'}->{$name})) {
# Allow update.
if (scalar(@_) == 2) {
$self->{'IMAGE'}->{$name} = $_[1];
}
return $self->{'IMAGE'}->{$name}; return $self->{'IMAGE'}->{$name};
} }
# Or it is for a local storage slot.
if ($name =~ /^_.*$/) {
if (scalar(@_) == 2) {
return $self->{'HASH'}->{$name} = $_[1];
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
}
carp("No such slot '$name' field in class $type"); carp("No such slot '$name' field in class $type");
return undef; return undef;
} }
...@@ -270,6 +285,7 @@ sub DESTROY { ...@@ -270,6 +285,7 @@ sub DESTROY {
my $self = shift; my $self = shift;
$self->{'IMAGE'} = undef; $self->{'IMAGE'} = undef;
$self->{'HASH'} = undef;
} }
# #
...@@ -781,12 +797,14 @@ sub NewVersion($$$$) ...@@ -781,12 +797,14 @@ sub NewVersion($$$$)
# Fix up the path by appending the version number. # Fix up the path by appending the version number.
# #
my $path = $self->path(); my $path = $self->path();
if (0) {
if ($path =~ /^(.*):\d+$/) { if ($path =~ /^(.*):\d+$/) {
$path = $1 . ":${clone_vers}"; $path = $1 . ":${clone_vers}";
} }
else { else {
$path .= ":${clone_vers}"; $path .= ":${clone_vers}";
} }
}
if (!$isdataset) { if (!$isdataset) {
DBQueryWarn("update $ostablename set ". DBQueryWarn("update $ostablename set ".
...@@ -1577,22 +1595,47 @@ sub MarkUpdate($$;$) ...@@ -1577,22 +1595,47 @@ sub MarkUpdate($$;$)
# #
# Set the hash. # Set the hash.
# #
sub SetHash($$) sub SetFullHash($$)
{ {
my ($self, $hash) = @_; my ($self, $hash) = @_;
return $self->Update({"hash" => $hash}); return $self->Update({"hash" => $hash});
} }
sub SetDeltaHash($$)
{
my ($self, $hash) = @_;
return $self->Update({"deltahash" => $hash});
}
# #
# Set the size. # Set the size.
# #
sub SetSize($$) sub SetFullSize($$)
{ {
my ($self, $size) = @_; my ($self, $size) = @_;
return $self->Update({"size" => $size}); return $self->Update({"size" => $size});
} }
sub SetDeltaSize($$)
{
my ($self, $size) = @_;
return $self->Update({"deltasize" => $size});
}
sub SetUploaderPath($$)
{
my ($self, $path) = @_;
return $self->Update({"uploader_path" => $path});
}
sub ClearUploaderPath($)
{
my ($self) = @_;
return $self->Update({"uploader_path" => ''});
}
# #
# Set the sector range of an image. # Set the sector range of an image.
...@@ -2168,5 +2211,102 @@ sub LocalVersionURL($) ...@@ -2168,5 +2211,102 @@ sub LocalVersionURL($)
return "$TBBASE/image_metadata.php?uuid=$uuid"; return "$TBBASE/image_metadata.php?uuid=$uuid";
} }
#
# Path and Directory stuff.
#
# Images are stored as directories now. Inside the directory are base
# and delta images for each version, as well as sig and sha1 files.
#
sub IsDirPath($)
{
my ($self) = @_;
#
# Does the path indicate a directory or a file.
#
if (!defined($self->path())) {
print STDERR "No path is set for $self\n";
return 0;
}
return 1
if ($self->path() =~ /\/$/);
return 0;
}
sub FullImagePath($)
{
my ($self) = @_;
my $path = $self->path();
my $vers = $self->version();
my $name = $self->imagename();
if ($self->IsDirPath()) {
return $path . $name . ".ndz" . ($vers ? ":$vers" : "");
}
return $path;
}
sub DeltaImagePath($)
{
my ($self) = @_;
my $path = $self->path();
my $vers = $self->version();
my $name = $self->imagename();
if ($self->IsDirPath()) {
return $path . $name . ".ddz" . ($vers ? ":$vers" : "");
}
return $path;
}
sub FullImageFile($)
{
my ($self) = @_;
return $self->FullImagePath();
}
sub DeltaImageFile($)
{
my ($self) = @_;
return $self->DeltaImagePath();
}
sub TempImageFile($)
{
my ($self) = @_;
my $path = $self->path();
my $vers = $self->version();
my $name = $self->imagename();
if ($self->IsDirPath()) {
return $path . $name . ".ddz" . ($vers ? ":$vers" : "") . ".tmp";
}
return $path . ".tmp";
}
sub FullImageSHA1File($)
{
my ($self) = @_;
return $self->FullImagePath() . ".sha1";
}
sub DeltaImageSHA1File($)
{
my ($self) = @_;
return $self->DeltaImagePath() . ".sha1";
}
sub FullImageSigFile($)
{
my ($self) = @_;
return $self->FullImagePath() . ".sig";
}
sub DeltaImageSigFile($)
{
my ($self) = @_;
return $self->DeltaImagePath() . ".sig";
}
sub HaveFullImage($)
{
my ($self) = @_;
return $self->size() ? 1 : 0;
}
sub HaveDeltaImage($)
{
my ($self) = @_;
return $self->deltasize() ? 1 : 0;
}
# _Always_ make sure that this 1 is at the end of the file... # _Always_ make sure that this 1 is at the end of the file...
1; 1;
...@@ -2881,7 +2881,8 @@ sub ImageInfo($) ...@@ -2881,7 +2881,8 @@ sub ImageInfo($)
} }
} }
else { else {
my $path = $image->path(); my $path = ($image->HaveDeltaImage() ?
$image->DeltaImageFile() : $image->FullImageFile());
if (-r $path) { if (-r $path) {
my $size = File::stat::stat($path)->size; my $size = File::stat::stat($path)->size;
......
#!/usr/bin/perl -wT #!/usr/bin/perl -wT
# #
# Copyright (c) 2000-2014 University of Utah and the Flux Group. # Copyright (c) 2000-2015 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -68,10 +68,8 @@ my $TBLOADWAIT = (10 * 60); ...@@ -68,10 +68,8 @@ my $TBLOADWAIT = (10 * 60);
my $osselect = "$TB/bin/os_select"; my $osselect = "$TB/bin/os_select";
my $TBUISP = "$TB/bin/tbuisp"; my $TBUISP = "$TB/bin/tbuisp";
my $IMAGEINFO = "$TB/sbin/imageinfo";
# Locals # Locals
my %imageinfo = (); # Per imageid DB info.
my $debug = 0; my $debug = 0;
my %children = (); # Child pids in when asyncmode=1 my %children = (); # Child pids in when asyncmode=1
my $remote_mult = 5; # Wait lots longer for remote nodes! my $remote_mult = 5; # Wait lots longer for remote nodes!
...@@ -97,7 +95,6 @@ sub osload ($$) { ...@@ -97,7 +95,6 @@ sub osload ($$) {
my $failures = 0; my $failures = 0;
my $usedefault = 1; my $usedefault = 1;
my $mereuser = 0; my $mereuser = 0;
my $rowref;
my $this_user; my $this_user;
if (!defined($args->{'nodelist'})) { if (!defined($args->{'nodelist'})) {
...@@ -388,26 +385,30 @@ sub osload ($$) { ...@@ -388,26 +385,30 @@ sub osload ($$) {
if $debug; if $debug;
# #
# We try to avoid repeated queries to DB for info that # We can have both a full image and/or a delta image. We
# does not change by caching the image info on the first # always prefer the full image if we have it.
# use. GetImageInfo() will perform various one-time #
# checks as well. if (! ($image->HaveFullImage() || $image->HaveDeltaImage())) {
# tberror "$node: no full or delta image file!";
if (!exists($imageinfo{$image->imageid()}) &&
!GetImageInfo($image,$node)) {
goto failednode;
}
$rowref = $imageinfo{$image->imageid()};
if ($rowref eq 'BADIMAGE') {
goto failednode; goto failednode;
} }
my $isfull = $image->HaveFullImage();
my $loadpart = $rowref->{'loadpart'}; my $loadpart = $image->loadpart();
my $loadlen = $rowref->{'loadlength'}; my $loadlen = $image->loadlength();
my $imagepath = $rowref->{'path'}; my $imagepid = $image->pid();
my $imagepid = $rowref->{'pid'}; my $imagesize = ($isfull ? $image->size() : $image->deltasize());
$maxwait += $rowref->{'maxloadwait'}; #
$access_keys[$i] = $rowref->{'access_key'}; # Compute a maxwait time based on the image size plus a constant
# factor for the reboot cycle. This is used later in
# WaitTillReloadDone(). Arguably, this should be part of the
# image DB state, so we store it in the imageinfo array too.
#
# size may be > 2^31, shift is unsigned
#
my $chunks = $imagesize >> 20;
$maxwait += int((($chunks / 100.0) * 65)) + $TBLOADWAIT;
$access_keys[$i] = $image->access_key();
# #
# Set the default boot OSID. # Set the default boot OSID.
...@@ -415,8 +416,8 @@ sub osload ($$) { ...@@ -415,8 +416,8 @@ sub osload ($$) {
# last image. # last image.
# #
if ($i == $imageidxs[-1]) { if ($i == $imageidxs[-1]) {
$defosid = $rowref->{'default_osid'}; $defosid = $image->default_osid();
$defvers = $rowref->{'default_vers'}; $defvers = $image->default_vers();
my $osinfo = OSinfo->Lookup($defosid, $defvers); my $osinfo = OSinfo->Lookup($defosid, $defvers);
if (!defined($osinfo)) { if (!defined($osinfo)) {
...@@ -464,12 +465,12 @@ sub osload ($$) { ...@@ -464,12 +465,12 @@ sub osload ($$) {
# If image MBR is incompatible with what is on the disk right # If image MBR is incompatible with what is on the disk right
# now, invalidate all the existing partitions ("...UNLESS" above). # now, invalidate all the existing partitions ("...UNLESS" above).
# #
if (defined($rowref->{'mbr_version'})) { if (defined($image->mbr_version())) {
if ($rowref->{'mbr_version'} && $curmbrvers && if ($image->mbr_version() && $curmbrvers &&
$rowref->{'mbr_version'} != $curmbrvers) { $image->mbr_version() != $curmbrvers) {
%partitions = (); %partitions = ();
} }
$curmbrvers = $rowref->{'mbr_version'}; $curmbrvers = $image->mbr_version();
} }
# #
...@@ -480,8 +481,8 @@ sub osload ($$) { ...@@ -480,8 +481,8 @@ sub osload ($$) {
my $partname = "part${i}_osid"; my $partname = "part${i}_osid";
my $partvers = "part${i}_vers"; my $partvers = "part${i}_vers";
my $osid = $rowref->{$partname}; my $osid = $image->DBData()->{$partname};
my $vers = $rowref->{$partvers}; my $vers = $image->DBData()->{$partvers};
if (defined($osid)) { if (defined($osid)) {
my $osinfo = OSinfo->Lookup($osid, $vers); my $osinfo = OSinfo->Lookup($osid, $vers);
if (!defined($osinfo)) { if (!defined($osinfo)) {
...@@ -583,7 +584,7 @@ sub osload ($$) { ...@@ -583,7 +584,7 @@ sub osload ($$) {
if ($WITHPROVENANCE && $WITHDELTAS) { if ($WITHPROVENANCE && $WITHDELTAS) {
my $founddelta = 0; my $founddelta = 0;
foreach my $image (@images) { foreach my $image (@images) {
if ($image->isdelta()) { if (!$image->HaveFullImage()) {
my $pimage = $image; my $pimage = $image;
my @ilist = (); my @ilist = ();
do { do {
...@@ -594,7 +595,7 @@ sub osload ($$) { ...@@ -594,7 +595,7 @@ sub osload ($$) {
goto failednode; goto failednode;
} }
push(@ilist, $pimage); push(@ilist, $pimage);
} while ($pimage->isdelta()); } while (!$pimage->HaveFullImage());
push @allimages, reverse(@ilist); push @allimages, reverse(@ilist);
$founddelta = 1; $founddelta = 1;
} }
...@@ -655,7 +656,6 @@ sub osload ($$) { ...@@ -655,7 +656,6 @@ sub osload ($$) {
&& !defined($access_keys[$i])) { && !defined($access_keys[$i])) {
$access_keys[$i] = TBGenSecretKey(); $access_keys[$i] = TBGenSecretKey();
$rowref->{'access_key'} = $access_keys[$i];
if ($allimages[$i]->Update({'access_key' => $access_keys[$i]}) != 0) { if ($allimages[$i]->Update({'access_key' => $access_keys[$i]}) != 0) {
tberror "$node: Could not initialize image access key"; tberror "$node: Could not initialize image access key";
goto failednode; goto failednode;
...@@ -897,92 +897,17 @@ sub osload ($$) { ...@@ -897,92 +897,17 @@ sub osload ($$) {
return $failures; return $failures;
} }
# sub DumpImageInfo($)
# Fetch information for a specified image the first time it is used
# (for the indicated node). This info is cached for use by all other
# nodes that require the image. Returns 1 on success, 0 on failure.
#
sub GetImageInfo($$)
{ {
my ($image, $node) = @_; my ($image) = @_;
my $imagesize = 0;
my $imageid = $image->imageid();
my $rowref = $image->DBData();
if (!defined($rowref)) {
tberror("No DBData for $image!");
$imageinfo{$imageid} = 'BADIMAGE';
return 0;
}
$imageinfo{$imageid} = $rowref;
my $imagepath = $rowref->{'path'};
#
# Perform a few validity checks: imageid should have a file name
# and that file should exist.
#
if (!defined($imagepath)) {
tberror "No filename associated with $image!";
$imageinfo{$imageid} = 'BADIMAGE';
return 0;
}
if (! -R $imagepath) {
#
# There are two reasons why a legit image might not be readable.
# One is that we are in an elabinelab and the image has just not
# been downloaded yet. The other is that we are attempting to
# access a shared (via the grantimage mechanism) image which the
# caller cannot directly access.
#
# For either case, making a proxy query request via frisbee will
# tell us whether the image is accessible and, if so, its size.
# "imageinfo" makes that call for us.
#
my $frisimageid = $rowref->{'pid'} . "/" . $rowref->{'imagename'};
my $sizestr = `$IMAGEINFO -qs -N $node $frisimageid`;
if ($sizestr =~ /^(\d+)$/) {
$imagesize = $1;
} else {
tberror "$image: access not allowed or image does not exist.";
$imageinfo{$imageid} = 'BADIMAGE';
return 0;
}
} else {
$imagesize = stat($imagepath)->size;
}
#
# A zero-length image cannot be right and will result in much confusion
# if allowed to pass: the image load will succeed, but the disk will be
# unchanged, making it appear that os_load loaded the default image.
#
if ($imagesize == 0) {
tberror "$imagepath is empty!";
$imageinfo{$imageid} = 'BADIMAGE';
return 0;
}
#
# Compute a maxwait time based on the image size plus a constant
# factor for the reboot cycle. This is used later in
# WaitTillReloadDone(). Arguably, this should be part of the
# image DB state, so we store it in the imageinfo array too.
#
if (!defined($rowref->{'maxloadwait'})) {