Commit 5acdd9ce authored by Leigh Stoller's avatar Leigh Stoller

Checkpoint image provenance/versioning.

parent 5aff2d89
......@@ -1308,36 +1308,49 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
if (imageid != NULL) {
/* Interested in a specific image */
if (can_access(imageidx, ei, 1)) {
res = mydb_query("SELECT pid,gid,imagename,"
" path,imageid"
" FROM images WHERE"
" pid='%s'"
" AND imagename='%s'",
res = mydb_query("SELECT i.pid,i.gid,"
" i.imagename,v.path,i.imageid"
" FROM images as i "
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE i.pid='%s'"
" AND i.imagename='%s'",
5, wantpid, wantname);
} else {
/*
* Pid of expt must be same as pid of image
* and gid the same or image "shared".
*/
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"
" FROM images WHERE"
" pid='%s' AND imagename='%s'"
" AND pid='%s'"
" AND (gid='%s' OR"
" (gid=pid AND shared=1))",
res = mydb_query("SELECT i.pid,i.gid,"
" i.imagename,v.path,i.imageid"
" FROM images as i "
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE i.pid='%s' "
" AND i.imagename='%s'"
" AND i.pid='%s'"
" AND (i.gid='%s' OR"
" (i.gid=i.pid AND "
" v.shared=1))",
5, wantpid, wantname,
ei->pid, ei->gid);
}
} else {
/* Find all images that this pid/gid can PUT */
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"
" FROM images"
" WHERE pid='%s'"
" AND (gid='%s' OR"
" (gid=pid AND shared=1))"
" ORDER BY pid,gid,imagename",
res = mydb_query("SELECT i.pid,i.gid,i.imagename,"
"v.path,i.imageid"
" FROM images as i"
" LEFT JOIN image_versions as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE i.pid='%s'"
" AND (i.gid='%s' OR"
" (i.gid=i.pid AND v.shared=1))"
" ORDER BY i.pid,i.gid,i.imagename",
5, ei->pid, ei->gid);
}
assert(res != NULL);
......@@ -1436,34 +1449,48 @@ emulab_get_host_authinfo(struct in_addr *req, struct in_addr *host,
if (imageid != NULL) {
/* Interested in a specific image */
if (can_access(imageidx, ei, 0)) {
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"
" FROM images WHERE"
" pid='%s'"
" AND imagename='%s'",
res = mydb_query("SELECT i.pid,i.gid,"
"i.imagename,v.path,i.imageid"
" FROM images as i "
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
"WHERE i.pid='%s'"
" AND i.imagename='%s'",
5, wantpid, wantname);
} else {
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"
" FROM images WHERE"
" pid='%s' AND imagename='%s'"
" AND (global=1"
" OR (pid='%s'"
" AND (gid='%s'"
" OR shared=1)))"
res = mydb_query("SELECT i.pid,i.gid,"
"i.imagename,v.path,i.imageid"
" FROM images as i"
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE i.pid='%s' "
" AND i.imagename='%s'"
" AND (v.global=1"
" OR (i.pid='%s'"
" AND (i.gid='%s'"
" OR v.shared=1)))"
" ORDER BY pid,gid,imagename",
5, wantpid, wantname,
ei->pid, ei->gid);
}
} else {
/* Find all images that this pid/gid can GET */
res = mydb_query("SELECT pid,gid,imagename,"
"path,imageid"
" FROM images WHERE"
" (global=1"
" OR (pid='%s'"
" AND (gid='%s' OR shared=1)))"
" ORDER BY pid,gid,imagename",
res = mydb_query("SELECT i.pid,i.gid,i.imagename,"
"v.path,i.imageid"
" FROM images as i"
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE"
" (v.global=1"
" OR (i.pid='%s'"
" AND (i.gid='%s' OR v.shared=1)))"
" ORDER BY i.pid,i.gid,i.imagename",
5, ei->pid, ei->gid);
}
assert(res != NULL);
......@@ -1610,10 +1637,14 @@ dump_image_aliases(FILE *fd)
char *lastpid;
/* First the global image alias */
res = mydb_query("SELECT pid,imagename"
" FROM images"
" WHERE global=1"
" ORDER BY pid,imagename", 2);
res = mydb_query("SELECT i.pid,i.imagename"
" FROM images as i"
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE v.global=1"
" ORDER BY i.pid,i.imagename", 2);
assert(res != NULL);
nrows = mysql_num_rows(res);
......@@ -1662,10 +1693,14 @@ dump_image_aliases(FILE *fd)
free(lastpid);
lastpid = mystrdup(row[0]);
res2 = mydb_query("SELECT imagename"
" FROM images"
" WHERE pid='%s' AND shared=1"
" ORDER BY imagename",
res2 = mydb_query("SELECT i.imagename"
" FROM images as i"
" LEFT JOIN image_versions "
" as v on "
" v.imageid=i.imageid and "
" v.version=i.version "
" WHERE i.pid='%s' AND v.shared=1"
" ORDER BY i.imagename",
1, lastpid);
assert(res2 != NULL);
......
......@@ -659,6 +659,7 @@ TBOPSEMAIL
IPV6_SUBNET_PREFIX
IPV6_ENABLED
NFSMAPTOUSER
IMAGEPROVENANCE
BROWSER_CONSOLE_ENABLE
EC2META_ENABLE
NOSITECHECKIN
......@@ -5091,6 +5092,7 @@ MANAGEMENT_NETWORK="10.249.249.0"
MANAGEMENT_NETMASK="255.255.255.0"
MANAGEMENT_ROUTER="10.249.249.253"
NFSMAPTOUSER="root"
IMAGEPROVENANCE=0
#
# XXX You really don't want to change these!
......
......@@ -291,6 +291,7 @@ AC_SUBST(BROWSER_CONSOLE_ENABLE)
AC_SUBST(IPV6_ENABLED)
AC_SUBST(IPV6_SUBNET_PREFIX)
AC_SUBST(NFSMAPTOUSER)
AC_SUBST(IMAGEPROVENANCE)
#
# Offer both versions of the email addresses that have the @ escaped
......@@ -433,6 +434,7 @@ MANAGEMENT_NETWORK="10.249.249.0"
MANAGEMENT_NETMASK="255.255.255.0"
MANAGEMENT_ROUTER="10.249.249.253"
NFSMAPTOUSER="root"
IMAGEPROVENANCE=0
#
# XXX You really don't want to change these!
......
......@@ -193,10 +193,13 @@ $EXPT_RESOURCESHOSED = 0;
# These are slots in the node table that need to be restored.
@nodetable_fields = ("def_boot_osid",
"def_boot_osid_vers",
"def_boot_path",
"def_boot_cmd_line",
"temp_boot_osid",
"temp_boot_osid_vers",
"next_boot_osid",
"next_boot_osid_vers",
"next_boot_path",
"next_boot_cmd_line",
"pxe_boot_path",
......@@ -4506,12 +4509,14 @@ sub LinkTestCapable($$)
my $idx = $self->idx();
my $query_result =
DBQueryWarn("select v.vname, FIND_IN_SET('linktest',osfeatures) ".
DBQueryWarn("select v.vname, FIND_IN_SET('linktest',ov.osfeatures) ".
" from virt_nodes as v ".
"left join reserved as r on r.pid=v.pid and ".
" r.eid=v.eid and r.vname=v.vname ".
"left join nodes as n on n.node_id=r.node_id ".
"left join os_info as o on o.osid=n.def_boot_osid ".
"left join os_info_versions as ov on ".
" ov.osid=n.def_boot_osid and ".
" ov.vers=n.def_boot_osid_vers ".
"where v.exptidx='$idx' and v.role!='bridge'");
return -1
if (!defined($query_result));
......
......@@ -63,77 +63,133 @@ sub mysystem($)
return system($command);
}
sub BlessRow($$)
{
my ($class, $row) = @_;
my $self = {};
my $imageid = $row->{"imageid"};
$self->{'IMAGE'} = $row;
bless($self, $class);
return $self;
}
#
# Lookup by idx or pid,osname, depending on the args.
# Lookup by idx or pid,imagename[:version] depending on the args. We always
# return highest numbered version on this path, if no version specified.
#
sub Lookup($$;$)
sub Lookup($$;$$)
{
my ($class, $arg1, $arg2) = @_;
my $imageid;
my ($class, $arg1, $arg2, $arg3) = @_;
#
# A single arg is either an index or a "pid,osname" or "pid/osname" string.
# A single arg is either an index or "pid,imagename[:version]" or
# "pid/imagename[:version]" string.
#
if (!defined($arg2)) {
if ($arg1 =~ /^(\d*)$/) {
$imageid = $1;
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.imageid='$arg1'");
return undef
if (! $result || !$result->numrows);
return BlessRow($class, $result->fetchrow_hashref());
}
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*)$/) {
$arg1 = $1;
$arg2 = $2;
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*)$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.pid='$1' and i.imagename='$2'");
return undef
if (! $result || !$result->numrows);
return BlessRow($class, $result->fetchrow_hashref());
}
elsif ($arg1 =~ /^([-\w]*),([-\w\.\+]*):(\d*)$/ ||
$arg1 =~ /^([-\w]*)\/([-\w\.\+]*):(\d*)$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.pid='$arg1' and i.imagename='$arg2' and ".
" v.version='$arg3' and v.deleted is null");
return undef
if (!$result || !$result->numrows);
return BlessRow($class, $result->fetchrow_hashref())
}
elsif ($arg1 =~ /^\w+\-\w+\-\w+\-\w+\-\w+$/) {
my $result =
DBQueryWarn("select imageid from images ".
"where uuid='$arg1'");
DBQueryWarn("select * from image_versions ".
"where uuid='$arg1' and deleted is null");
return undef
if (! $result || !$result->numrows);
($imageid) = $result->fetchrow_array();
return BlessRow($class, $result->fetchrow_hashref());
}
else {
return undef;
}
}
elsif (! (($arg1 =~ /^[-\w\.\+]*$/) && ($arg2 =~ /^[-\w\.\+]*$/))) {
return undef;
}
elsif (!defined($arg3)) {
if ($arg1 =~ /^\d+$/ && $arg2 =~ /^\d+$/) {
my $result =
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.imageid='$arg1' and v.version='$arg2' ".
" and v.deleted is null");
return undef
if (! $result || !$result->numrows);
#
# Two args means pid/imagename lookup instead of gid_idx.
#
if (defined($arg2)) {
my $images_result =
DBQueryWarn("select imageid from images ".
"where pid='$arg1' and imagename='$arg2'");
return BlessRow($class, $result->fetchrow_hashref());
}
elsif ($arg1 =~ /^[-\w]*$/ && $arg2 =~ /^([-\w\.\+]*):(\d+)$/) {
my $result =
DBQueryWarn("select v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.pid='$arg1' and i.imagename='$1' and ".
" v.version='$2'");
return undef
if (! $result || !$result->numrows);
return undef
if (! $images_result || !$images_result->numrows);
return BlessRow($class, $result->fetchrow_hashref());
}
elsif ($arg1 =~ /^[-\w]*$/ && $arg2 =~ /^[-\w\.\+]*$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"where i.pid='$arg1' and i.imagename='$arg2'");
return undef
if (! $result || !$result->numrows);
($imageid) = $images_result->fetchrow_array();
return BlessRow($class, $result->fetchrow_hashref());
}
return undef;
}
else {
if ($arg1 =~ /^[-\w]*$/ &&
$arg2 =~ /^[-\w\.\+]*$/ && $arg3 =~ /^\d+$/) {
my $result =
DBQueryWarn("select i.*,v.* from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid ".
"where i.pid='$arg1' and i.imagename='$arg2' and ".
" v.version='$arg3' and v.deleted is null");
return undef
if (!$result || !$result->numrows);
# Look in cache first
return $images{"$imageid"}
if (exists($images{"$imageid"}));
my $query_result =
DBQueryWarn("select * from images where imageid='$imageid'");
return undef
if (!$query_result || !$query_result->numrows);
my $self = {};
$self->{'IMAGE'} = $query_result->fetchrow_hashref();
bless($self, $class);
# Add to cache.
$images{"$imageid"} = $self;
return $self;
return BlessRow($class, $result->fetchrow_hashref());
}
}
return undef;
}
AUTOLOAD {
......@@ -170,13 +226,15 @@ sub LookupByURL($$)
my $safe_url = DBQuoteSpecial($url);
my $query_result =
DBQueryWarn("select imageid from images where metadata_url=$safe_url");
DBQueryWarn("select pid,imagename from image_versions ".
"where metadata_url=$safe_url and deleted is null");
return undef
if (!$query_result || !$query_result->numrows);
my ($imageid) = $query_result->fetchrow_array();
my ($pid,$imagename) = $query_result->fetchrow_array();
return Image->Lookup($imageid);
# Want latest version.
return Image->Lookup($pid, $imagename);
}
#
......@@ -200,12 +258,13 @@ sub ActiveImages($)
my @result = ();
my $query_result =
DBQueryWarn("select imageid from frisbee_blobs where frisbee_pid!=0");
DBQueryWarn("select imageid,imageid_version from frisbee_blobs ".
"where frisbee_pid!=0");
return undef
if (!defined($query_result));
while (my ($imageid) = $query_result->fetchrow_array()) {
my $image = Image->Lookup($imageid);
while (my ($imageid,$version) = $query_result->fetchrow_array()) {
my $image = Image->Lookup($imageid, $version);
if (!defined($image)) {
print STDERR "*** Could not find DB object for image $imageid\n";
return undef;
......@@ -232,18 +291,19 @@ sub ListAll($$$)
}
if (defined($pid) && $pid =~ /^([-\w]+)$/) {
$clause .= ($clause ? " and" : "where");
$clause .= " pid='$1'";
$clause .= " i.pid='$1'";
}
my $query_result =
DBQueryWarn("select pid,imagename from images $clause ".
"order by pid,imagename");
DBQueryWarn("select distinct i.pid,i.imagename from images as i ".
"left join image_versions as v on ".
" v.imageid=i.imageid and v.version=i.version ".
"$clause order by i.pid,i.imagename");
if ($query_result) {
while (my ($pid,$name) = $query_result->fetchrow_array()) {
push(@result, "$pid/$name");
}
}
return @result;
}
......@@ -258,9 +318,11 @@ sub Refresh($)
if (! ref($self));
my $imageid = $self->imageid();
my $version = $self->version();
my $query_result =
DBQueryWarn("select * from images where imageid=$imageid");
DBQueryWarn("select * from image_versions ".
"where imageid=$imageid and version='$version'");
return -1
if (!$query_result || !$query_result->numrows);
......@@ -314,8 +376,11 @@ sub Create($$$$$$$$)
#
# The pid/imageid has to be unique, so lock the table for the check/insert.
# We lock the os_info tables too, for the call into Lookup.
#
DBQueryWarn("lock tables images write, emulab_indicies write")
DBQueryWarn("lock tables images write, image_versions write, ".
" os_info as o read, os_info_versions as v read, ".
" emulab_indicies write")
or return undef;
my $query_result =
......@@ -333,8 +398,8 @@ sub Create($$$$$$$$)
my $safe_url = DBQuoteSpecial($metadata_url);
my $query_result =
DBQueryWarn("select imagename from images ".
"where metadata_url=$safe_url");
DBQueryWarn("select imagename from image_versions ".
"where metadata_url=$safe_url and deleted is null");
if ($query_result->numrows) {
DBQueryWarn("unlock tables");
......@@ -367,15 +432,15 @@ sub Create($$$$$$$$)
$desc = DBQuoteSpecial($argref->{'description'});
}
my $query = "insert into images set ".
join(",", map("$_='" . $argref->{$_} . "'", @arg_slots));
my $bquery = "imagename='$imagename'";
$bquery .= ",imageid='$imageid'";
$bquery .= ",uuid='$uuid'";
$bquery .= ",pid='$pid',pid_idx='$pid_idx'";
$bquery .= ",gid='$gid',gid_idx='$gid_idx'";
# Append the rest
$query .= ",imagename='$imagename'";
$query .= ",imageid='$imageid'";
$query .= ",uuid='$uuid'";
$query .= ",pid='$pid',pid_idx='$pid_idx'";
$query .= ",gid='$gid',gid_idx='$gid_idx'";
my $query = "insert into image_versions set $bquery, ".
join(",", map("$_='" . $argref->{$_} . "'", @arg_slots));
$query .= ",creator='$uid',creator_idx='$uid_idx'";
$query .= ",created=now()";
$query .= ",description=$desc";
......@@ -390,23 +455,26 @@ sub Create($$$$$$$$)
if (exists($ENV{'GENIURN'}) && $ENV{'GENIURN'} ne "") {
$query .= ",creator_urn=". DBQuoteSpecial($ENV{'GENIURN'});
}
# Create the main entry:
if (! DBQueryWarn("insert into images set $bquery")) {
DBQueryWarn("unlock tables");
tberror("Error inserting new images record for $pid/$imagename!");
return undef;
}
# And the other entry.
if (! DBQueryWarn($query)) {
DBQueryWarn("unlock tables");
tberror("Error inserting new images record for $pid/$imagename!");
return undef;
}
DBQueryWarn("unlock tables");
my $image = Image->Lookup($imageid);
# Create the osidtoimageid mapping. Admins have an option to do it or not.
my $makedefault = exists($argref->{"makedefault"}) &&
$argref->{"makedefault"} eq "1";
if (!$isadmin || $makedefault) {
# Lock tables unlocks previous locks as a side-effect.
DBQueryWarn("lock tables osidtoimageid write, images write, ".
"nodes as n write, node_type_attributes as a write");
#
# Dig out the mtypes we want to turn on. The caller has already
# sanity checked them to make sure the types actually exist, and
......@@ -446,10 +514,165 @@ sub Create($$$$$$$$)
}
}
}
return $image;
}
#
# Clone an image descriptor from the DB, bumping the version number
#
sub NewVersion($$$$$)
{
my ($self, $creator, $baseimage, $argref, $usrerr_ref) = @_;
my $osid = $self->imageid();
my $version = $self->version();
my $tableid = int(rand(10000000));
my $ostablename = "os_info_versions" . $tableid;
my $imtablename = "image_versions" . $tableid;
#
# The pid/imageid has to be unique, so lock the table for the check/insert.
#
DBQueryWarn("lock tables images write, image_versions write, ".
" os_info write, os_info_versions write, ".
" emulab_indicies write")
or return undef;
my $query_result =
DBQueryWarn("create temporary table $ostablename ".
"select * from os_info_versions ".
" where osid='$osid' and vers='$version'");
goto bad
if (!$query_result);
$query_result =
DBQueryWarn("create temporary table $imtablename ".
"select * from image_versions ".
"where imageid='$osid' and version='$version'");
goto bad
if (!$query_result);
#
# Now reset a few things in each table.
#
my $clone_vers = $self->version() + 1;
my $creator_urn = "";
#
# If this is set in the environment, we use it. This allows
# us to track actual geni users, since all of that happens
# as geniuser. We do not bother to set this for local users.
#
if (exists($ENV{'GENIURN'}) && $ENV{'GENIURN'} ne "") {
$creator_urn = ",creator_urn=". DBQuoteSpecial($ENV{'GENIURN'});
}
#
# Figure out which partition needs to be changed. EZ images only
# at this time.
#
my $part_vers = "";
for (my $i = 1; $i <= 4; $i++) {
my $func = "part${i}_osid";
if (defined($self->$func())) {
$part_vers = "part${i}_vers='${clone_vers}'";
last;
}
}
my $uid = $creator->uid();
my $uid_idx = $creator->uid_idx();
#
# The parent of this image might be this image, if taking a snapshot
# of a node running the image. But we also need to support making a
# version of the image, from a node running some other image. We still
# bump the version number, but the parent pointers are different.
#
my ($parent_imageid, $parent_version);
if ($self->imageid() == $baseimage->imageid()) {
$parent_imageid = $self->imageid();
$parent_version = $self->version();
}
else {
$parent_imageid = $baseimage->imageid();
$parent_version = $baseimage->version();
}
#
# Fix up the path by appending the version number.
#
my $path = $self->path();
if ($path =~ /^(.*):\d+$/) {
$path = $1 . ":${clone_vers}";
}
else {
$path .= ":${clone_vers}";
}
DBQueryWarn("update $ostablename set ".
" uuid=uuid(), ".
" vers='$clone_vers',".
" parent_osid='$parent_imageid',".
" parent_vers='$parent_version', ".
" creator='$uid',creator_idx='$uid_idx' ".
"where osid='$osid'")
or goto bad;
DBQueryWarn("update $imtablename set ".
" uuid=uuid(),ready=0,path='$path', ".
" $part_vers, default_vers='$clone_vers', ".
" version='$clone_vers',".
" parent_imageid='$parent_imageid',".
" parent_version='$parent_version', ".
" creator='$uid',creator_idx='$uid_idx' $creator_urn ".
"where imageid='$osid'")
or goto bad;
#
# And insert into the real table. At this point we will be
# inconsistent if we crash before the commit is done. We should use
# a transaction, but myisam tables do not do real transactions.
#
if (! (DBQueryWarn("insert into os_info_versions ".
"select * from $ostablename") &&
DBQueryWarn("insert into image_versions ".
"select * from $imtablename") &&
DBQueryWarn("update os_info set version='$clone_vers' ".
"where osid='$osid'") &&
DBQueryWarn("update images set version='$clone_vers' ".
"where imageid='$osid'"))) {
DBQueryWarn("delete from os_info_versions ".
"where osid='$osid' and vers='$clone_vers'");
DBQueryWarn("delete from image_versions ".
"where imageid='$osid' and version='$clone_vers'");
DBQueryWarn("update os_info set version='$version' ".
"where osid='$osid'");
DBQueryWarn("update images set version='$version' ".
"where imageid='$osid'");
goto bad;
}
DBQueryWarn("unlock tables");
return Image->Lookup($osid);
bad:
DBQueryWarn("unlock tables");
return undef;
}
return $image;
#
# For a newly created image, set the provenance pointers back to
# the image from which it was derived.
#
sub SetProvenance($$)
{
my ($self, $base) = @_;
my $parent_imageid = $base->imageid();
my $parent_version = $base->version();
$self->Update({"parent_imageid" => $parent_imageid,
"parent_version" => $parent_version})
== 0 or return -1;
return 0;
}
#
......@@ -524,8 +747,9 @@ sub EditImageid($$$$)