Skip to content

GitLab

Commit 95ca5100 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files
Options

Another checkpoint of image versioning.

parent 5b09a48c
Hide whitespace changes
Inline Side-by-side
Showing with 198 additions and 89 deletions
db/Image.pm.in
View file @ 95ca5100
......@@ -47,6 +47,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $TBAUDIT = "@TBAUDITEMAIL@";
my $TBBASE = "@TBBASE@";
my $PGENISUPPORT = @PROTOGENI_SUPPORT@;
my $DOPROVENANCE = @IMAGEPROVENANCE@;
my $TBWWW = "@TBWWW@";
# Cache of instances to avoid regenerating them.
......@@ -90,7 +91,7 @@ sub Lookup($$;$$)
if (!defined($arg2)) {
if ($arg1 =~ /^(\d*)$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.imageid='$arg1'");
......@@ -102,7 +103,7 @@ sub Lookup($$;$$)
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*)$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.pid='$1' and i.imagename='$2'");
......@@ -114,11 +115,11 @@ sub Lookup($$;$$)
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*):(\d*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*):(\d*)$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.pid='$arg1' and i.imagename='$arg2' and ".
" v.version='$arg3' and v.deleted is null");
"where i.pid='$1' and i.imagename='$2' and ".
" v.version='$3' and v.deleted is null");
return undef
if (!$result || !$result->numrows);
......@@ -163,7 +164,7 @@ sub Lookup($$;$$)
}
elsif ($arg1 =~ /^[-\w]*$/ && $arg2 =~ /^[-\w\.\+]*$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.pid='$arg1' and i.imagename='$arg2'");
......@@ -178,7 +179,7 @@ sub Lookup($$;$$)
if ($arg1 =~ /^[-\w]*$/ &&
$arg2 =~ /^[-\w\.\+]*$/ && $arg3 =~ /^\d+$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.pid='$arg1' and i.imagename='$arg2' and ".
......@@ -361,7 +362,7 @@ sub Create($$$$$$$$)
}
# Pass-through optional slots, otherwise the DB default is used.
foreach my $key ("path", "shared", "global", "ezid", "mbr_version",
"metadata_url", "imagefile_url") {
"metadata_url", "imagefile_url", "released") {
if (exists($argref->{$key})) {
push(@arg_slots, $key);
}
......@@ -520,9 +521,9 @@ sub Create($$$$$$$$)
#
# Clone an image descriptor from the DB, bumping the version number
#
sub NewVersion($$$$$)
sub NewVersion($$$$)
{
my ($self, $creator, $baseimage, $argref, $usrerr_ref) = @_;
my ($self, $creator, $baseimage, $usrerr_ref) = @_;
my $osid = $self->imageid();
my $version = $self->version();
my $tableid = int(rand(10000000));
......@@ -618,9 +619,10 @@ sub NewVersion($$$$$)
or goto bad;
DBQueryWarn("update $imtablename set ".
" uuid=uuid(),ready=0,released=1,path='$path', ".
" uuid=uuid(),ready=0,path='$path', ".
" $part_vers, default_vers='$clone_vers', ".
" version='$clone_vers',".
" created=now(), ".
" parent_imageid='$parent_imageid',".
" parent_version='$parent_version', ".
" creator='$uid',creator_idx='$uid_idx' $creator_urn ".
......@@ -629,30 +631,21 @@ sub NewVersion($$$$$)
#
# And insert into the real table. At this point we will be
# inconsistent if we crash before the commit is done. We should use
# a transaction, but myisam tables do not do real transactions.
# inconsistent if we crash before the commit is done.
#
if (! (DBQueryWarn("insert into os_info_versions ".
"select * from $ostablename") &&
DBQueryWarn("insert into image_versions ".
"select * from $imtablename") &&
DBQueryWarn("update os_info set version='$clone_vers' ".
"where osid='$osid'") &&
DBQueryWarn("update images set version='$clone_vers' ".
"where imageid='$osid'"))) {
"select * from $imtablename"))) {
DBQueryWarn("delete from os_info_versions ".
"where osid='$osid' and vers='$clone_vers'");
DBQueryWarn("delete from image_versions ".
"where imageid='$osid' and version='$clone_vers'");
DBQueryWarn("update os_info set version='$version' ".
"where osid='$osid'");
DBQueryWarn("update images set version='$version' ".
"where imageid='$osid'");
goto bad;
}
DBQueryWarn("unlock tables");
return Image->Lookup($osid);
return Image->Lookup($osid, $clone_vers);
bad:
DBQueryWarn("unlock tables");
return undef;
......@@ -675,6 +668,76 @@ sub SetProvenance($$)
return 0;
}
#
# Highest number version, rather then what is deemed most recent by the
# images table.
#
sub LookupMostRecent($)
{
my ($self) = @_;
my $imageid = $self->imageid();
my $query_result =
DBQueryWarn("select version from image_versions ".
"where imageid='$imageid' order by version desc limit 1");
return undef
if (!$query_result);
my ($version) = $query_result->fetchrow_array();
return Image->Lookup($imageid, $version);
}
#
# Release an image; all this does is make the highest numbered version
# in the image_versions table, the default version. It does this by
# updating the version number in the images table. An unreleased version
# can be used by explicitly giving the version number, but unqualified
# use always get the version that is set in the images table.
#
sub Release($)
{
my ($self) = @_;
DBQueryWarn("lock tables images write, image_versions write, ".
" os_info write, os_info_versions write")
or return -1;
if (!$self->ready()) {
print STDERR "Image::Release: $self is not marked ready!\n";
goto bad;
}
my $imageid = $self->imageid();
my $version = $self->version();
#
# Only the "head" version can be released.
#
my $query_result =
DBQueryWarn("select max(version) from image_versions ".
"where imageid='$imageid'");
goto bad
if (!$query_result || !$query_result->numrows);
my ($head) = $query_result->fetchrow_array();
if ($head != $self->version()) {
print STDERR "Image::Release: not the head version of $self\n";
goto bad;
}
return -1
if (! (DBQueryWarn("update os_info set version='$version' ".
"where osid='$imageid'") &&
DBQueryWarn("update images set version='$version' ".
"where imageid='$imageid'") &&
DBQueryWarn("update image_versions set released=1 ".
"where imageid='$imageid' and version='$version'")));
DBQueryWarn("unlock tables");
return 0;
bad:
DBQueryWarn("unlock tables");
return -1;
}
#
# Worker class method to edit image descriptor.
# Assumes most argument checking was done elsewhere.
......@@ -875,7 +938,7 @@ sub Delete($;$)
DBQueryWarn("delete from images where imageid='$imageid'")
or goto bad;
if ($purge) {
if ($purge || !$DOPROVENANCE) {
goto bad
if (! QueryWarn("delete from image_versions ".
"where imageid='$imageid'"));
......@@ -892,7 +955,7 @@ sub Delete($;$)
or goto bad;
DBQueryWarn("delete from os_info where osid='$imageid'")
or goto bad;
if ($purge) {
if ($purge || !$DOPROVENANCE) {
DBQueryWarn("delete from os_info_versions where osid='$imageid'")
or goto bad;
}
......@@ -925,7 +988,6 @@ sub DeleteVersion($)
my ($self) = @_;
my $imageid = $self->imageid();
my $version = $self->version();
DBQueryWarn("lock tables images write, image_versions write, ".
" os_info write, os_info_versions write")
......@@ -933,14 +995,16 @@ sub DeleteVersion($)
#
# We do not allow the deletion if it is not the "head" version of the
# image. Maybe later.
# image_versions. Maybe later.
#
my $query_result =
DBQueryWarn("select version from images ".
"where imageid='$imageid' and version='$version'");
DBQueryWarn("select max(version) from image_versions ".
"where imageid='$imageid'");
goto bad
if (!$query_result);
if (!$query_result->numrows) {
if (!$query_result || !$query_result->numrows);
my ($version) = $query_result->fetchrow_array();
if ($version != $self->version()) {
print STDERR "Image::DeleteVersion: not the head version of $self\n";
goto bad;
}
......@@ -948,12 +1012,8 @@ sub DeleteVersion($)
goto bad
if (! (DBQueryWarn("delete from image_versions ".
"where imageid='$imageid' and version='$version'") &&
DBQueryWarn("update images set version=version-1 ".
"where imageid='$imageid'") &&
DBQueryWarn("delete from os_info_versions ".
"where osid='$imageid' and vers='$version'") &&
DBQueryWarn("update os_info set version=version-1 ".
"where osid='$imageid'")));
"where osid='$imageid' and vers='$version'")));
DBQueryWarn("unlock tables")
or return -1;
......@@ -1644,25 +1704,25 @@ sub SetLogFile($$)
}
# Ready bit.
sub MarkReady($)
sub SetReady($$)
{
my ($self) = @_;
my ($self, $ready) = @_;
return -1
if (! $self->Update({'ready' => 1}));
if (! $self->Update({'ready' => $ready}));
return 0;
}
sub MarkReady($) { return SetReady($_[0], 1); }
sub ClearReady($) { return SetReady($_[0], 0); }
# Released bit.
sub MarkReleased($)
# Are two images the same.
sub SameImage($$)
{
my ($self) = @_;
return -1
if (! $self->Update({'released' => 1}));
my ($this, $that) = @_;
return 0;
return (($this->imageid() == $that->imageid() &&
$this->version() == $that->version()) ? 1 : 0);
}
#
......
db/OSinfo.pm.in
View file @ 95ca5100
......@@ -132,7 +132,7 @@ sub Lookup($$;$$)
if (!defined($arg2)) {
if ($arg1 =~ /^(\d*)$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid and v.vers=o.version ".
"where o.osid='$arg1'");
......@@ -144,7 +144,7 @@ sub Lookup($$;$$)
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*)$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid and v.vers=o.version ".
"where o.pid='$1' and o.osname='$2'");
......@@ -156,11 +156,11 @@ sub Lookup($$;$$)
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*):(\d*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*):(\d*)$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid ".
"where o.pid='$arg1' and o.osname='$arg2' and ".
" v.vers='$arg3' and v.deleted is null");
"where o.pid='$1' and o.osname='$2' and ".
" v.vers='$3' and v.deleted is null");
return undef
if (!$result || !$result->numrows);
......@@ -169,7 +169,7 @@ sub Lookup($$;$$)
}
elsif ($arg1 =~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
my $result =
DBQueryWarn("select * from os_info ".
DBQueryWarn("select * from os_info_versions ".
"where uuid='$arg1' and deleted is null");
return undef
......@@ -182,7 +182,7 @@ sub Lookup($$;$$)
elsif (!defined($arg3)) {
if ($arg1 =~ /^\d+$/ && $arg2 =~ /^\d+$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid ".
"where o.osid='$arg1' and v.vers='$arg2'");
......@@ -193,7 +193,7 @@ sub Lookup($$;$$)
}
elsif ($arg1 =~ /^[-\w]*$/ && $arg2 =~ /^([-\w\.\+]*):(\d+)$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid ".
"where o.pid='$arg1' and o.osname='$1' and ".
......@@ -205,7 +205,7 @@ sub Lookup($$;$$)
}
elsif ($arg1 =~ /^[-\w]*$/ && $arg2 =~ /^[-\w\.\+]*$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid and v.vers=o.version ".
"where o.pid='$arg1' and o.osname='$arg2'");
......@@ -220,7 +220,7 @@ sub Lookup($$;$$)
if ($arg1 =~ /^[-\w]*$/ &&
$arg2 =~ /^[-\w\.\+]*$/ && $arg3 =~ /^\d+$/) {
my $result =
DBQueryWarn("select o.*,v.* from os_info as o ".
DBQueryWarn("select v.* from os_info as o ".
"left join os_info_versions as v on ".
" v.osid=o.osid ".
"where o.pid='$arg1' and o.osname='$arg2' and ".
......
utils/clone_image.in
View file @ 95ca5100
......@@ -187,15 +187,6 @@ if (defined($image)) {
fatal("Cannot clone a non-ez image");
}
#
# But we do not allow emulab-ops images to be overwritten.
# Might remove this later. Just being careful since this is going
# to be used from the ProtoGENI RPC interface.
#
if ($image->pid eq TBOPSPID() && !$this_user->IsAdmin()) {
fatal("Not allowed to snapshot a system image");
}
#
# The access check above determines if the caller has permission
# to overwrite the image file.
......@@ -207,11 +198,6 @@ if (defined($image)) {
exit(0);
}
#
# We create a new version of the image descriptor for the new
# snapshot. We mark it as not ready so that others know it is
# in transition. When we later call createimage, it will make
# sure the ready bit is clear before trying to lock it.
#
# Before we do anything destructive, we lock the image.
#
......@@ -219,23 +205,63 @@ if (defined($image)) {
fatal("Image is locked, please try again later!\n");
}
if ($DOPROVENANCE) {
#
# This will include unreleased images (in image_versions, but
# not the one pointed to by the images table).
#
$image = $image->LookupMostRecent();
if (!defined($image)) {
$image->Unlock();
fatal("Cannot lookup osinfo for $image");
}
#
# We create a new version of the image descriptor for the new
# snapshot. We mark it as not ready so that others know it is
# in transition. When we later call createimage, it will make
# sure the ready bit is clear before trying to use it.
#
my $needclone = 1;
#
# Is the ready bit set? If not, it means something went wrong with
# a previous image creation. Lets reset the provenance.
# Does the most version in the table has its ready bit not set? If so
# it means something went wrong with a previous image creation. We can
# reuse it, but reset the provenance just in case the node got reloaded.
#
if (!$image->ready()) {
my $osinfo = OSinfo->Lookup($image->imageid());
my $osinfo = OSinfo->Lookup($image->imageid(), $image->version());
if (!defined($osinfo)) {
$image->Unlock();
fatal("Cannot lookup osinfo for $image");
}
$image->SetProvenance($base_image);
$osinfo->SetProvenance($base_osinfo);
$needclone = 0;
print "Reusing image version " . $image->version() . ", ".
"since it was never marked ready.\n";
}
else {
#
# If the new image is going to based on the exact same base,
# lets not create another new version, but overwrite the current
# one. Save lots of space this way. We miss saving intermediate
# versions, but this is a typical approach to getting an image
# ready for use; change, snapshot, test, change, snapshot ...
#
if ($needclone && !$image->released() &&
($image->parent_imageid() == $base_image->imageid() &&
$image->parent_version() == $base_image->version())) {
# For create_image to be happy.
$image->ClearReady(0);
$needclone = 0;
print "Reusing image version " . $image->version() . ", ".
"since the base is the same and it was not released.\n";
}
if ($needclone) {
my $clone_error;
my $clone = $image->NewVersion($this_user,
$base_image, undef, \$clone_error);
$base_image, \$clone_error);
if (!defined($clone)) {
$image->Unlock();
fatal("Could not clone image descriptor" .
......@@ -248,14 +274,14 @@ if (defined($image)) {
# Watch for a system image that is saved elsewhere; see equiv code
# in create_image. We change the path to point over to the /proj
# directory so that we do not burn up space on boss until it is
# officially "released". We can also use this version of the image
# officially "released". We *can* use this version of the image
# by explicitly using its version number, before it is released.
#
if ($image->path() =~ /^\/usr\/testbed/) {
my $path = $PROJROOT . "/" . $image->pid() . "/images/" .
basename($image->path()) . ":" . $image->version();
if ($image->Update({"path" => $path, "released" => 0})) {
if ($image->Update({"path" => $path})) {
$image->DeleteVersion();
fatal("Could not update path and ready bit");
}
......@@ -269,8 +295,9 @@ if (defined($image)) {
if ($debug);
exit(0);
}
my $output = emutil::ExecQuiet("$CREATEIMAGE -p $pid $imagename $node_id");
my $output =
emutil::ExecQuiet("$CREATEIMAGE -p $pid ".
" $imagename:" . $image->version() . " $node_id");
if ($?) {
if ($DOPROVENANCE) {
$image->DeleteVersion()
......@@ -291,6 +318,16 @@ if (!$base_image->ezid()) {
fatal("Cannot clone a non-ez image");
}
#
# Not allowed to derive an image from one that has not been released.
# Maybe relax this in the future, but this is a good simplification for
# now.
#
if ($DOPROVENANCE && !$base_image->released()) {
fatal("Not allowed to derive a new image from unreleased ".
"base $base_image");
}
#
# To avoid confusion, we do not allow users to shadow system images
# in their own project.
......@@ -321,8 +358,6 @@ my %xmlfields =
"version" => $base_osinfo->version(),
"path" => $path,
"op_mode", => $base_osinfo->op_mode(),
"global" => (defined($global) ?
($global ? 1 : 0) : $base_osinfo->shared()),
"wholedisk", => $wholedisk,
);
$xmlfields{"reboot_waittime"} = $base_osinfo->reboot_waittime()
......@@ -330,13 +365,13 @@ $xmlfields{"reboot_waittime"} = $base_osinfo->reboot_waittime()
$xmlfields{"osfeatures"} = $base_osinfo->osfeatures()
if (defined($base_osinfo->osfeatures()) &&
$base_osinfo->osfeatures() ne "");
$xmlfields{"global"} = 1
if (defined($global) && $global);
if (defined($base_image)) {
$xmlfields{"mbr_version"} = $base_image->mbr_version();
$xmlfields{"loadpart"} = $base_image->loadpart();
$xmlfields{"noexport"} = $base_image->noexport();
$xmlfields{"global"} = (defined($global) ?
($global ? 1 : 0) : $base_image->global());
# Short form uses wholedisk instead. Should fix this.
if ($base_image->loadpart() == 0 && $base_image->loadlength() == 4) {
......
utils/create_image.in
View file @ 95ca5100
......@@ -86,12 +86,14 @@ my $TBLOGS = "@TBLOGSEMAIL@";
my $BOSSIP = "@BOSSNODE_IP@";
my $CONTROL = "@USERNODE@";
my $NONFS = @NOSHAREDFS@;
my $DOPROVENANCE= @IMAGEPROVENANCE@;
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use EmulabConstants;
use libtestbed;
use libadminmfs;
use Experiment;
......@@ -284,6 +286,7 @@ if (!defined($image)) {
" No such image descriptor $imagename in project $imagepid!\n");
}
my $imageid = $image->imageid();
my $version = $image->version();
if ($mereuser &&
! $image->AccessCheck($this_user, TB_IMAGEID_ACCESS)) {
......@@ -367,8 +370,7 @@ else {
#
my $filename = $image->path();
my $isglobal = $image->global();
# Temporary until frisbee path can handle versions?
my $usepath = $image->released() ? 0 : 1;
my $usepath = 0;
#
# Redirect pathname for global images. See equiv code in clone_image.
......@@ -394,7 +396,7 @@ if ($isglobal && ($filename =~ /^\/usr\/testbed/)) {
# Use realpath to resolve any symlinks.
#
my $translated = realpath($filename);
if ($translated =~ /^([-\w\.\/\+]+)$/) {
if ($translated =~ /^([-\w\.\/\+:]+)$/) {
$filename = $1;
}
else {
......@@ -426,7 +428,7 @@ if ($image->Lock()) {
}
$needunlock = 1;
if ($image->ready()) {
if ($DOPROVENANCE && $image->ready()) {
$image->Unlock();
die("*** $0:\n".
" $image ready flag is set, this is inconsistent!\n");
......@@ -513,7 +515,8 @@ my $loadlength;
my $command = "$createimage ";
if ($usefup) {
my $id = $usepath ? $filename : ($image->pid() . "/$imagename");
my $id = $usepath ? $filename :
$image->pid() . "/$imagename" . ($version > 0 ? ":${version}" : "");
$command .= " -S $BOSSIP -F $id";
}
......@@ -903,9 +906,20 @@ if (defined($webtask)) {
$webtask->status("ready");
$webtask->Exited(0);
}
#
# Normal images are immediately marked as "released" (and ready),
# but global system images are just marked ready, and must be explicitly
# released.
#
$image->MarkReady();
if (! ($isglobal && $pid eq TBOPSPID())) {
if ($image->Release()) {
$image->Unlock();
exit(1);
}
}
$image->Unlock();
exit 0;
exit(0);
sub cleanup ()
{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment