Skip to content

GitLab

emulab-devel
emulab-devel
Commit 7a82778e authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files
Options

A slew of new support for cooked mode. Swapmod now works pretty well, 

although the default is currently to do cooked mode in "basic" mode,
rather then "full" mode.
parent 99c6c7c4
Hide whitespace changes
Inline Side-by-side
Showing with 2400 additions and 1313 deletions
db/libGeni.pm.in
View file @ 7a82778e
......@@ -16,9 +16,9 @@ use vars qw(@ISA @EXPORT);
# Configure variables
my $TB = "@prefix@";
my $BOSSNODE = "@BOSSNODE@";
my $TBDOCBASE = "@TBDOCBASE@";
#my $TBDOCBASE = "@TBDOCBASE@";
my $TBDOCBASE = "http://www.emulab.net";
my $ELVIN_COMPAT= @ELVIN_COMPAT@;
my $ASSIGN = "$TB/libexec/assign";
use libdb;
......@@ -32,13 +32,17 @@ use Lan;
use GeniEmulab;
use GeniResource;
use GeniResponse;
use GeniAuthority;
use GeniXML;
use English;
use Socket;
use XML::Simple;
use XML::LibXML;
use XML::SemanticDiff;
use Data::Dumper;
use File::Temp;
use IO::File;
use POSIX qw(strftime);
sub Register($$)
{
......@@ -64,61 +68,51 @@ sub RenewSlivers($;$)
#
# Map rspec to resources using assign.
#
sub MapResources($$$$)
sub MapResources($$$$$)
{
my ($experiment, $user, $rspec, $verbose) = @_;
my %cm_urns = ();
my %fragments = ();
my %nodemap = ();
my %node_cms = ();
my ($experiment, $user, $topo, $rspecref, $verbose) = @_;
my %cmurn2res = ();
my %cmurn2nodes = ();
my %cmurn2links = ();
my %cmurn2frags = ();
my %vname2res = ();
my %vname2doc = ();
Register($experiment, $user) == 0
or return -1;
foreach my $ref (@{ $rspec->{'node'} }) {
#
# Build up rspec fragments from the "intermediate" representation
# that libvtop passed in. Ick.
#
foreach my $ref (@{ $topo->{'node'} }) {
# Skip non geni nodes; handled in libvtop
next
if (! (exists($ref->{'isgeninode'}) && $ref->{'isgeninode'}));
my $resource;
my %copy = %{ $ref };
my $copy = \%copy;
my $node_urn = $ref->{'request_urn'};
my $manager_urn = $ref->{'manager_urn'};
my $virtual_id = $ref->{'virtual_id'};
my (undef,undef,$node_id) = GeniHRN::Parse($node_urn);
my $node_urn = $copy->{'request_urn'};
my ($auth,$type,$node_id) = GeniHRN::Parse($node_urn);
my $cm = GeniHRN::Generate($auth, "authority", "cm");
$copy->{'component_manager_uuid'} = $cm;
#
# Get the resource object.
#
if (!exists($cm_urns{$cm})) {
$resource = GeniResource->Lookup($experiment->idx(), $cm);
if (!exists($cmurn2res{$manager_urn})) {
$resource = GeniResource->Lookup($experiment->idx(), $manager_urn);
if (!defined($resource)) {
$resource = GeniResource->Create($experiment, $cm);
$resource = GeniResource->Create($experiment, $manager_urn);
if (!defined($resource)) {
print STDERR "Could not create GeniResource for $cm\n";
print STDERR
"Could not create GeniResource for $manager_urn\n";
return -1;
}
}
$cm_urns{$cm} = $resource;
}
$resource = $cm_urns{$cm};
#
# request_urn means nothing to assign; kill that from the copy.
#
delete($copy->{'request_urn'});
# Ditto
delete($copy->{'tarfiles'});
#
# If already have the ticket, then leave the urn alone.
# We do not run assign again, but we need the rspec to be
# complete for loops below.
#
if (!$resource->HaveTicket()) {
if ($node_id ne "*") {
$copy->{'component_uuid'} = $node_urn;
$copy->{'component_urn'} = $node_urn;
}
$cmurn2res{$manager_urn} = $resource;
}
$resource = $cmurn2res{$manager_urn};
$vname2res{$virtual_id} = $resource;
#
# The point of this is to split the rspec apart, since at present
......@@ -126,108 +120,91 @@ sub MapResources($$$$)
# to pass in an rspec that references CMs other then the one
# advertisement being passed in.
#
# This should go away when assign is fixed.
#
if (!exists($fragments{$cm})) {
$fragments{$cm} = {
'generated_by' => 'libvtop',
'type' => 'request',
'xmlns' => 'http://www.protogeni.net/resources/rspec/0.2',
# Assign now wants this stuff. I have no clue what it means!
'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
'xsi:schemaLocation' =>
"http://www.protogeni.net/resources/rspec/0.2 ".
"http://www.protogeni.net/resources/rspec/0.2/request.xsd",
'node' => [] };
}
push(@{ $fragments{$cm}->{'node'} }, $copy);
$node_cms{$ref->{'virtual_id'}} = $cm;
$nodemap{$ref->{'virtual_id'}} = $ref;
my $fragment;
if (!exists($cmurn2frags{$manager_urn})) {
$fragment = CreateNewRspec();
return -1
if (!defined($fragment));
$cmurn2frags{$manager_urn} = $fragment;
}
$fragment = $cmurn2frags{$manager_urn};
my $rspecdoc = AddNodeToRspec($fragment, $ref);
return -1
if (!defined($rspecdoc));
$vname2doc{$virtual_id} = $rspecdoc;
}
#
# As above, need to split the interfaces into the correct fragments.
#
if (exists($rspec->{'link'})) {
foreach my $ref (@{ $rspec->{'link'} }) {
if (exists($topo->{'link'})) {
foreach my $ref (@{ $topo->{'link'} }) {
my $linkname = $ref->{'virtual_id'};
# means nothing to assign; added again below.
delete($ref->{'component_manager'});
# Skip tunnels until rspec stitching in place.
next
if (exists($ref->{'link_type'}) &&
$ref->{'link_type'} eq "tunnel");
next
if (0 && exists($ref->{'link_type'}) &&
exists($ref->{'link_type'}->{'type_name'}) &&
$ref->{'link_type'}->{'type_name'} eq "tunnel");
foreach my $ifaceref (@{ $ref->{'interface_ref'} }) {
my $virtual_node_id = $ifaceref->{'virtual_node_id'};
my $ifaceref0 = $ref->{'interface_ref'}->[0];
my $ifaceref1 = $ref->{'interface_ref'}->[1];
# Do not want to add this twice.
my $virtid0 = $ifaceref0->{'virtual_node_id'};
my $resource0 = $vname2res{$virtid0};
my $fragment0 = $cmurn2frags{$resource0->manager_urn()};
if (!exists($node_cms{$virtual_node_id}) ||
!exists($fragments{$node_cms{$virtual_node_id}})) {
print STDERR "Inconsistency in rspec link $linkname\n";
print Dumper($rspec);
return -1;
}
my $fragment = $fragments{$node_cms{$virtual_node_id}};
if (!exists($fragment->{'link'})) {
$fragment->{'link'} = [];
}
push(@{ $fragment->{'link'} }, $ref);
#
# There should be a sanity check here ... too lazy.
#
last;
my $rspecdoc = AddLinkToRspec($fragment0, $ref);
return -1
if (!defined($rspecdoc));
# Only one of these in the array, for building combined rspec.
$vname2doc{$linkname} = $rspecdoc;
my $virtid1 = $ifaceref1->{'virtual_node_id'};
my $resource1 = $vname2res{$virtid1};
my $fragment1 = $cmurn2frags{$resource1->manager_urn()};
if ($resource0->manager_urn() ne $resource1->manager_urn()) {
$rspecdoc = AddLinkToRspec($fragment1, $ref);
return -1
if (!defined($rspecdoc));
}
}
}
if ($verbose) {
print STDERR "Rspec Fragments:\n";
foreach my $fragment (values(%cmurn2frags)) {
my $rspecstr = GeniXML::Serialize($fragment, 1);
print STDERR "$rspecstr\n";
}
}
#
# Discover resources at the component and run assign.
# Discover resources in parallel and run assign, writing the solution
# to a file.
#
foreach my $cm (keys(%cm_urns)) {
my $resource = $cm_urns{$cm};
my $fragment = $fragments{$cm};
my $coderef = sub {
my ($resource, $fragment, $tmp) = @{ $_[0] };
my $advertisement;
#
# We got the ticket on a previous loop.
#
if ($resource->HaveTicket()) {
print STDERR "Already have a ticket for $resource; skipping ...\n";
next;
}
print STDERR "Asking for resource list from $resource\n";
if ($resource->Discover($user, \$advertisement)) {
print STDERR "Could not get resource list for $resource\n";
return -1;
}
my $tmp = File::Temp::mktemp("XXXXX");
my $soln = $tmp . ".soln";
my $log = $tmp . ".log";
my $ptop = $tmp . ".ptop";
my $vtop = $tmp . ".vtop";
my $reqstring =
eval { XMLout($fragment, "NoAttr" => 0, RootName => "rspec") };
if ($@) {
print STDERR "XMLout error: $@\n";
print STDERR Dumper($fragment);
return -1;
}
my $reqstring = GeniXML::Serialize($fragment);
my $vtopfh = new IO::File "> $vtop";
my $ptopfh = new IO::File "> $ptop";
if (! (defined($vtopfh) && defined($ptopfh))) {
print STDERR "Could not create temporary files for ptop/vtop\n";
return -1;
}
print $vtopfh "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
print $vtopfh $reqstring;
print $ptopfh $advertisement;
$vtopfh->close();
......@@ -242,11 +219,75 @@ sub MapResources($$$$)
if ($?) {
print STDERR "Could not map to physical resources on $resource\n";
print STDERR Dumper($fragment);
my $logstuff = `cat $log`;
print STDERR "\n" . $logstuff . "\n";
my $string = GeniXML::Serialize($fragment, 1);
print STDERR "$string\n";
return -1;
}
return 0;
};
#
# Figure out which resources still need to be mapped.
#
my @todo = ();
my @results = ();
foreach my $manager_urn (keys(%cmurn2res)) {
my $resource = $cmurn2res{$manager_urn};
my $fragment = $cmurn2frags{$manager_urn};
#
# We got the ticket on a previous loop.
#
if ($resource->HaveTicket()) {
print STDERR "Already have a ticket for $resource; skipping ...\n";
next;
}
my $tmp = File::Temp::mktemp("XXXXX");
push(@todo, [$resource, $fragment, $tmp]);
}
print STDERR "Mapping resources in parallel ...\n";
if (ParRun({'maxwaittime' => 600, 'maxchildren' => 4},
\@results, $coderef, @todo)) {
print STDERR "*** MapResources: Internal error mapping resources\n";
return -1;
}
#
# Check the exit codes.
#
my $errors = 0;
my $count = 0;
foreach my $result (@results) {
my ($resource, $fragment, $tmp) = @{ $todo[$count] };
#
# ParRun does a fork; so need to refresh the resource object
# to sync it to the DB.
#
if ($resource->Refresh()) {
print STDERR "*** MapResources: Error synchronizing $resource\n";
$errors++;
}
elsif ($result != 0) {
print STDERR "*** Error mapping resources for $resource\n";
$errors++;
}
$count++;
}
return -1
if ($errors);
#
# Since everything mapped, read the solutions and write back to the rspec
#
foreach my $ref (@todo) {
my ($resource, $fragment, $tmp) = @{ $ref };
my $soln = $tmp . ".soln";
my $solution =
eval { XMLin($soln, KeyAttr => [],
ForceArray => ["node", "link", "interface",
......@@ -265,41 +306,33 @@ sub MapResources($$$$)
my $virtual_id = $ref->{'virtual_id'};
my $node_urn = $ref->{'component_uuid'};
my $cm_urn = $ref->{'component_manager_uuid'};
my $noderef = $nodemap{$virtual_id};
my $rspecdoc = $vname2doc{$virtual_id};
#
# This writes the solution back into the original rspec.
# This writes the solution back into the fragment.
#
$noderef->{'component_uuid'} = $node_urn;
$noderef->{'component_urn'} = $node_urn;
$noderef->{'component_manager_uuid'} = $cm_urn;
if (exists($noderef->{'disk_image'})) {
my $osid = $noderef->{'disk_image'}->{'name'};
my ($auth,$type,$id) = GeniHRN::Parse($node_urn);
$rspecdoc->setAttribute("component_uuid", $node_urn);
$rspecdoc->setAttribute("component_urn", $node_urn);
$rspecdoc->setAttribute("component_manager_uuid", $cm_urn);
$rspecdoc->setAttribute("component_manager_urn", $cm_urn);
my $urn = GeniHRN::Generate($auth, "image", "emulab-ops");
$urn .= "//" . $osid;
$noderef->{'disk_image'}->{'name'} = $urn;
}
if (exists($ref->{'interface'})) {
my $interfaces = $ref->{'interface'};
foreach my $ifaceref (@{ $interfaces }) {
my $iface_id = $ifaceref->{'virtual_id'};
my $compid = $ifaceref->{'component_id'};
my $virtid = $ifaceref->{'virtual_id'};
my $compid = $ifaceref->{'component_id'};
# Not supposed to happen, but does cause of issues
# with tunnels and rspec stitching.
next
if (!defined($compid));
foreach my $oref (@{ $noderef->{'interface'} }) {
if ($oref->{'virtual_id'} eq $iface_id) {
# write the solution back into the original rspec.
$oref->{'component_id'} = $compid;
foreach my $linkref (GeniXML::FindNodes("n:interface",
$rspecdoc)->get_nodelist()) {
my $ovirtid = GeniXML::GetText("virtual_id", $linkref);
if ($ovirtid eq $virtid) {
$linkref->setAttribute("component_id", $compid);
last;
}
}
......@@ -307,133 +340,225 @@ sub MapResources($$$$)
}
}
}
if (exists($rspec->{'link'})) {
foreach my $ref (@{ $rspec->{'link'} }) {
my %cms = ();
my @cms = ();
foreach my $ifaceref (@{ $ref->{'interface_ref'} }) {
my $virtual_node_id = $ifaceref->{'virtual_node_id'};
my $node_cm = $node_cms{$virtual_node_id};
if (!exists($cms{$node_cm})) {
push(@cms, {'id' => $node_cm});
$cms{$node_cm} = $node_cm;
}
#
# One last chore; we had to skip tunnels above since assign does
# not know how to deal with them.
#
if (exists($topo->{'link'})) {
foreach my $ref (@{ $topo->{'link'} }) {
my $linkname = $ref->{'virtual_id'};
# Skip non-tunnels.
next
if (! (exists($ref->{'link_type'}) &&
$ref->{'link_type'} eq "tunnel"));
my $ifaceref0 = $ref->{'interface_ref'}->[0];
my $ifaceref1 = $ref->{'interface_ref'}->[1];
# Do not want to add this twice.
my $virtid0 = $ifaceref0->{'virtual_node_id'};
my $resource0 = $vname2res{$virtid0};
my $fragment0 = $cmurn2frags{$resource0->manager_urn()};
my $rspecdoc = AddLinkToRspec($fragment0, $ref);
return -1
if (!defined($rspecdoc));
# Only one of these in the array, for building combined rspec.
$vname2doc{$linkname} = $rspecdoc;
my $virtid1 = $ifaceref1->{'virtual_node_id'};
my $resource1 = $vname2res{$virtid1};
my $fragment1 = $cmurn2frags{$resource1->manager_urn()};
if ($resource0->manager_urn() ne $resource1->manager_urn()) {
$rspecdoc = AddLinkToRspec($fragment1, $ref);
return -1
if (!defined($rspecdoc));
}
$ref->{'component_manager'} = [@cms];
}
}
if ($verbose) {
print STDERR "Final rspec:\n";
print STDERR Dumper($rspec);
#
# Compare to previous rspec; this tells us if we need to request
# a new ticket. Easier to use the rspec then the manifest since
# the CM added a ton of cruft to it.
#
foreach my $manager_urn (keys(%cmurn2res)) {
my $resource = $cmurn2res{$manager_urn};
if (1) {
my $rspecdoc = GeniResource::ConvertRspec($rspec);
my $rspecstr = GeniXML::Serialize($rspecdoc);
print STDERR "In XML:\n";
print STDERR "$rspecstr\n";
# We got the ticket on a previous loop.
next
if ($resource->HaveTicket());
my $fragment = $cmurn2frags{$manager_urn};
my $frag_string = GeniXML::Serialize($fragment);
# Must stash for redeemticket.
$resource->stashrspec($frag_string);
next
if (! $resource->HaveRspec());
#
# The idea is to do a diff of the rspec. If there are no
# meaningful changes, then we can skip trying to get a new
# ticket.
#
my $rspec = $resource->Rspec();
my $diff = XML::SemanticDiff->new();
my $rspec_string = GeniXML::Serialize($rspec);
$resource->setmodified(0);
print STDERR "Comparing old and new rspecs for $resource\n";
foreach my $change ($diff->compare($rspec_string, $frag_string)) {
print STDERR " $change->{message} in context $change->{context}\n";
$resource->setmodified(1);
}
if ($resource->modified() && $resource->ManagerApiLevel() == 0) {
print STDERR "*** Difference to rspec for level 0 $resource. ".
"Not allowed, aborting\n";
return -1;
}
}
#
# Combine the fragments into final rspec document.
#
my $rspec = CombineRspecDocs(values(%vname2doc));
return -1
if (!defined($rspec));
# Make sure all that namespace stuff is done correctly.
$rspec = GeniXML::Parse(GeniXML::Serialize($rspec));
if ($verbose) {
print STDERR "Final rspec:\n";
my $rspecstr = GeniXML::Serialize($rspec, 1);
print STDERR "$rspecstr\n";
}
$$rspecref = $rspec;
return 0;
}
sub GetTickets($$$$)
sub GetTickets($)
{
my ($experiment, $verbose, $user, $rspec) = @_;
my %cm_urns = ();
my %nodemap = ();
my ($vtop) = @_;
my $experiment = $vtop->experiment();
my $verbose = $vtop->verbose();
my $user = $vtop->user();
my $rspec = $vtop->genirspec();
my $eventkey = $experiment->eventkey();
my $keyhash = $experiment->keyhash();
Register($experiment, $user) == 0
or return -1;
foreach my $ref (@{ $rspec->{'node'} }) {
#
# Delete the fake lan nodes before sending off the request.
# They were needed for the mapping above, but the remote CM
# will barf on them.
#
if (0 && exists($ref->{'node_type'}) &&
exists($ref->{'node_type'}->{'type_name'}) &&
$ref->{'node_type'}->{'type_name'} eq "lan") {
next;
}
my $virtual_id = $ref->{'virtual_id'};
my $node_urn = $ref->{'component_urn'};
my ($auth,$type,$node_id) = GeniHRN::Parse($node_urn);
my $cm = GeniHRN::Generate($auth, "authority", "cm");
#
# Get the resource objects.
#
my @resources = GeniResource->LookupAll($experiment);
if (! @resources) {
print STDERR "RedeemTickets: No resource objects\n";
return 0;
}
my @todo = ();
#
# Get the resource objects for below.
#
if (!exists($cm_urns{$cm})) {
my $resource = GeniResource->Lookup($experiment->idx(), $cm);
if (!defined($resource)) {
print STDERR "Could not get GeniResource for $cm\n";
return -1;
}
$nodemap{$virtual_id} = $resource;
#
# We got the ticket on a previous loop.
#
next
if ($resource->HaveTicket());
$cm_urns{$cm} = $resource;
}
#
# This is how we get the client side to do cooked mode properly.
#
$ref->{'tarfiles'} = "/usr/local/etc/emulab ".
"$TBDOCBASE/downloads/geniclient.tar";
foreach my $resource (@resources) {
next
if ($resource->HaveTicket() || !$resource->modified());
push(@todo, $resource);
}
# No tickets needed, return now.
return 0
if (! scalar(keys(%cm_urns)));
if (! @todo);
#
# XXX Convert to a proper XML looking thing. This is just a temporay
# hack for now.
# Before sending off the rspec, add the stuff that we need for
# full cooked mode to work.
#
my $rspecdoc = GeniResource::ConvertRspec($rspec);
my $rspecstr = GeniXML::Serialize($rspecdoc);
foreach my $ref (GeniXML::FindNodes("n:node",
$rspec)->get_nodelist()) {
my $vname = GeniXML::GetVirtualId($ref);
my $node = $vtop->VnameToNode($vname);
# XXX Boot mode; simple vs full. Have not decided the
# proper interface for this yet.
my $bootmode = $experiment->GetEnvVariable("${vname}_BOOTMODE");
next
if (defined($bootmode) && lc($bootmode) eq "basic");
if (!defined($node)) {
print STDERR "*** GetTickets: No proxy node for $vname\n";
return -1;
}