Skip to content

GitLab

Commit b1989b05 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

Checkpoint first cut at cross CM stitching of vlans.

parent 7fef4024
Hide whitespace changes
Inline Side-by-side
protogeni/lib/GeniCM.pm.in
View file @ b1989b05
......@@ -39,6 +39,7 @@ use libtestbed qw(SENDMAIL);
use emutil;
use EmulabConstants;
use libEmulab;
use Lan;
use English;
use Data::Dumper;
use XML::Simple;
......@@ -74,6 +75,7 @@ my $TARFILES_SETUP = "$TB/bin/tarfiles_setup";
my $MAPPER = "$TB/bin/mapper";
my $VTOPGEN = "$TB/bin/vtopgen";
my $SNMPIT = "$TB/bin/snmpit";
my $RESERVEVLANS = "$TB/sbin/protogeni/reservevlans";
my $NEWGROUP = "$TB/bin/newgroup";
my $PRERENDER = "$TB/libexec/vis/prerender";
my $XMLLINT = "/usr/local/bin/xmllint";
......@@ -655,6 +657,9 @@ sub GetTicketAuxAux($$$$$$$$$)
my %nodemap = ();
my @nodeids = ();
my %lannodes = ();
# For stitching, keep track of external nodes and links.
my %external_nodemap = ();
my %external_linkmap = ();
#
# If this is a ticket update, we want to seed the namemap with
......@@ -747,8 +752,10 @@ sub GetTicketAuxAux($$$$$$$$$)
my $node;
# Let remote nodes pass through.
next
if (! GeniXML::IsLocalNode($ref));
if (! GeniXML::IsLocalNode($ref)) {
$external_nodemap{$node_nickname} = $ref;
next;
}
#
# Lan nodes are fake and do not go into the virt topo. Need
......@@ -1097,6 +1104,7 @@ sub GetTicketAuxAux($$$$$$$$$)
foreach my $ref (@interfaces) {
my $node_nickname = GeniXML::GetInterfaceNodeId($ref);
my $iface_id = GeniXML::GetInterfaceId($ref);
my ($iface_ref,$iface_name,$iface_vport);
if (!defined($iface_id)) {
$response =
......@@ -1134,15 +1142,58 @@ sub GetTicketAuxAux($$$$$$$$$)
next;
}
if (!exists($ifacemap{$node_nickname})) {
# Might be the other side. Skip for now; might bite later.
next
if (!exists($namemap{$node_nickname}));
$response =
GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"$lanname: No such virtual_node_id: ".
"$node_nickname");
goto bad;
#
# If the interface refers to a node at another site, then
# lets try to stitch together a vlan. Assign will fail if
# the user has specified something impossible.
#
if (!exists($namemap{$node_nickname}) &&
exists($external_nodemap{$node_nickname})) {
#
# I have completely punted on how we represent external
# links in the DB so that we know what to put into the
# virtual topology. For now, just hardwire the one test
# case.
#
my $noderef = $external_nodemap{$node_nickname};
my $other_cm = GeniXML::GetManagerId($noderef);
if (! ($other_cm eq "urn:publicid:IDN+" .
"myelab.testbed.emulab.net+authority+cm" ||
$other_cm eq "urn:publicid:IDN+" .
"emulab.net+authority+cm")) {
$response =
GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"$lanname: links to external node");
goto bad;
}
#
# Stick in a reference to the fake bbg node that
# corresponds to where the link comes in.
#
my $virtnode =
$virtexperiment->NewTableRow("virt_nodes",
{"vname" => $node_nickname,
"type" => "bbgeni",
"osname" => '',
"ips" => '', # deprecated
"cmd_line"=> '', # bogus
"fixed" => "bbg1"});
if (!defined($virtnode)) {
print STDERR "Error creating bbg node\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto bad;
}
$iface_name = "";
$iface_vport = 0;
goto stitch;
}
else {
$response =
GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"$lanname: No such virtual_node_id: ".
"$node_nickname");
goto bad;
}
}
#
......@@ -1155,8 +1206,8 @@ sub GetTicketAuxAux($$$$$$$$$)
"$node_nickname:$iface_id");
goto bad;
}
my $iface_ref = $ifacemap{$node_nickname}->{$iface_id}->{"rspec"};
my $iface_name = GeniXML::GetText("component_id", $iface_ref);
$iface_ref = $ifacemap{$node_nickname}->{$iface_id}->{"rspec"};
$iface_name = GeniXML::GetText("component_id", $iface_ref);
if (!defined($iface_name)) {
$iface_name = "";
}
......@@ -1172,8 +1223,9 @@ sub GetTicketAuxAux($$$$$$$$$)
$iface_name = "";
$trivial_ok = 1;
}
my $iface_vport = $ifacemap{$node_nickname}->{$iface_id}->{"vport"};
$iface_vport = $ifacemap{$node_nickname}->{$iface_id}->{"vport"};
stitch:
# XXX
my $ip = "10.10.${linknum}.${ifacenum}";
my $mask = "255.255.255.0";
......@@ -1348,12 +1400,20 @@ sub GetTicketAuxAux($$$$$$$$$)
$solution)->get_nodelist()) {
my $virtual_id = GeniXML::GetVirtualId($ref);
my $component_id = GeniXML::GetNodeId($ref);
if (!exists($nodemap{$virtual_id})) {
if (!(exists($nodemap{$virtual_id}) ||
exists($external_nodemap{$virtual_id}))) {
$response =
GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Mapper inserted nodes you did not want");
goto bad;
}
#
# If this was an external node we placed into the topo, then
# just skip it. Revisit this later.
#
next
if (exists($external_nodemap{$virtual_id}));
my $rspec = $nodemap{$virtual_id}->{'rspec'};
my $virtnode = $nodemap{$virtual_id}->{'virtnode'};
my $node = GeniUtil::LookupNode($component_id);
......@@ -1417,11 +1477,20 @@ sub GetTicketAuxAux($$$$$$$$$)
&& defined($virtual_port_id)) {
$virtual_node_id = $iface2node{$virtual_port_id};
}
my $component_id = GeniXML::GetText("component_id",
$iface_ref);
#
# If this was an external node we placed into the topo, skip
# it here, but we have to remember it cause we have to contact
# the other CM to coordinate the vlan reservation.
#
if (exists($external_nodemap{$virtual_node_id})) {
$external_linkmap{$nickname} = [$ref, $iface_ref];
next;
}
my $component_id = GeniXML::GetText("component_id", $iface_ref);
if (!defined($virtual_node_id) || !defined($virtual_port_id)) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Virtual node ID or virtual interface ID missing on interface");
"Virtual node ID or virtual interface ".
"ID missing on interface");
goto bad;
}
my $vportp = $vportmap{"$virtual_node_id:$virtual_port_id"};
......@@ -1438,6 +1507,7 @@ sub GetTicketAuxAux($$$$$$$$$)
}
}
}
# Store the virt topo again since we changed it above.
$virtexperiment->Dump();
if ($virtexperiment->Store()) {
......@@ -1445,6 +1515,53 @@ sub GetTicketAuxAux($$$$$$$$$)
goto bad;
}
#
# Now contact external CMs to coordinate vlans.
#
foreach my $linkname (keys(%external_linkmap)) {
my ($linkref, $ifaceref) = @{ $external_linkmap{$linkname} };
my $slice_urn = $slice->urn();
#
# Already have a reserved tag? This could happen if the other CM
# acted first and talked to this CM before we saw the ticket
# request. Or this is an update and we already have tags reserved.
#
my $tag = VLan::GetReservedVlanTag($slice_experiment, $linkname);
goto tagged
if (defined($tag));
my ($fh, $filename) = tempfile(UNLINK => 0);
if (!defined($fh)) {
print STDERR "Could not create temp file for rspec\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto bad;
}
print $fh GeniXML::Serialize($rspec);
close($fh);
system("$RESERVEVLANS '$slice_urn' '$linkname' $filename");
if ($?) {
unlink($filename);
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not reserve vlan tags for $linkname");
goto bad;
}
unlink($filename);
#
# Need to find out what vlan was assigned.
#
$tag = VLan::GetReservedVlanTag($slice_experiment, $linkname);
if (!defined($tag)) {
print STDERR "Did not find the reserved tag for $linkname\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Error reserving vlan tag");
goto bad;
}
tagged:
GeniXML::SetText("vlantag", $linkref, $tag);
}
print GeniXML::Serialize($rspec);
#
......@@ -1730,6 +1847,9 @@ sub SliverWorkAux($$$$$$$)
my @freelinks= ();
my %iface2node = ();
my $needplabslice = 0;
# For stitching, keep track of external nodes and links.
my %external_nodemap = ();
my %external_linkmap = ();
print GeniXML::Serialize($rspec);
......@@ -1826,8 +1946,10 @@ sub SliverWorkAux($$$$$$$)
my $manager_id = GeniXML::GetManagerId($ref);
# Let remote nodes pass through.
next
if (! GeniXML::IsLocalNode($ref));
if (! GeniXML::IsLocalNode($ref)) {
$external_nodemap{$node_nickname} = $ref;
next;
}
#
# Lan nodes are fake and do not go into the virt topo. Need
......@@ -2359,6 +2481,15 @@ sub SliverWorkAux($$$$$$$)
next
if (exists($lannodes{$node_id}));
#
# If the interface refers to a node at another site, then
# we can ignore it. It was just a placeholder for getting
# the vlan tag reserved.
#
if (!exists($nodemap{$node_id}) &&
exists($external_nodemap{$node_id})) {
next;
}
my $nodesliver = $nodemap{$node_id};
if (!defined($nodesliver)) {
......@@ -2504,7 +2635,21 @@ sub SliverWorkAux($$$$$$$)
#
# Must have the topofile for node boot. Might need locking on this.
#
if (!$v2) {
if ($v2) {
require Lan;
#
# We want to reserve the vlan tags now so that we can put them
# into the manifest. Use snmpit for this, since ELABINELAB will
# need to ask the outer boss for the reserved tags.
#
system("$SNMPIT -A $pid $eid");
if ($?) {
$message = "Could not reserve vlan tags";
goto bad;
}
}
elsif (!$v2) {
require Lan;
if (system("$GENTOPOFILE $pid $eid")) {
......@@ -2525,35 +2670,39 @@ sub SliverWorkAux($$$$$$$)
$message = "Could not set up vlans";
goto bad;
}
foreach my $linkref (GeniXML::FindNodes("n:link",
$rspec)->get_nodelist()) {
my $vname = GeniXML::GetVirtualId($linkref);
my $vlan;
my $lan = Lan->Lookup($experiment, $vname, 1);
if (!defined($lan)) {
print STDERR "No lan object for $vname\n";
next;
}
if ($lan->type() eq "vlan") {
$vlan = VLan->Lookup($experiment, $vname);
}
elsif (defined($lan->link())) {
$vlan = VLan->Lookup($lan->link());
}
}
foreach my $linkref (GeniXML::FindNodes("n:link",
$manifest)->get_nodelist()) {
my $vname = GeniXML::GetVirtualId($linkref);
my $vlan;
if (!defined($vlan)) {
print STDERR "Could not find a vlan for $vname\n";
next;
}
my $tag;
my $lan = Lan->Lookup($experiment, $vname, 1);
if (!defined($lan)) {
print STDERR "No lan object for $vname\n";
next;
}
if ($lan->type() eq "vlan") {
$vlan = VLan->Lookup($experiment, $vname);
}
elsif (defined($lan->link())) {
$vlan = VLan->Lookup($lan->link());
}
if (!defined($vlan)) {
print STDERR "Could not find a vlan for $vname\n";
next;
}
my $tag;
if ($v2) {
$tag = $vlan->GetReservedVlanTag();
}
else {
$vlan->GetTag(\$tag);
if (!defined($tag)) {
print STDERR "No tag for $vlan\n";
next;
}
GeniXML::SetText("vlantag", $linkref, $tag);
}
if (!defined($tag)) {
print STDERR "No tag for $vlan\n";
next;
}
GeniXML::SetText("vlantag", $linkref, $tag);
}
# Set up plab nodes all at once.
......@@ -3015,7 +3164,7 @@ sub DeleteSliverAux($$$)
my $eid = $experiment->eid();
if (!$impotent) {
system("$SNMPIT -r $pid $eid");
system("$SNMPIT -C -r $pid $eid");
if ($?) {
print STDERR "Could not tear down vlans\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
......@@ -4046,7 +4195,7 @@ sub CleanupDeadSlice($;$)
my $pid = $experiment->pid();
my $eid = $experiment->eid();
system("$SNMPIT -r $pid $eid");
system("$SNMPIT -C -r $pid $eid");
if ($?) {
print STDERR "Could not tear down vlans\n";
return -1;
......@@ -4073,7 +4222,7 @@ sub CleanupDeadSlice($;$)
# A firewalled slice gets special treatment.
#
if ($slice->needsfirewall()) {
print "Calling undoFWNodes ...\n";
print STDERR "Calling undoFWNodes ...\n";
if (undoFWNodes($experiment, 1) != 0) {
print STDERR "FireWall cleanup failed\n";
......
protogeni/lib/GeniCMV2.pm.in
View file @ b1989b05
......@@ -40,6 +40,7 @@ use Date::Parse;
use POSIX qw(strftime tmpnam);
use Time::Local;
use Compress::Zlib;
use File::Temp qw(tempfile);
use MIME::Base64;
# Configure variables
......@@ -50,6 +51,7 @@ my $TBAUDIT = "@TBAUDITEMAIL@";
my $BOSSNODE = "@BOSSNODE@";
my $OURDOMAIN = "@OURDOMAIN@";
my $PGENIDOMAIN = "@PROTOGENI_DOMAIN@";
my $ELABINELAB = "@ELABINELAB@";
my $CREATEEXPT = "$TB/bin/batchexp";
my $ENDEXPT = "$TB/bin/endexp";
my $NALLOC = "$TB/bin/nalloc";
......@@ -66,6 +68,7 @@ my $TARFILES_SETUP = "$TB/bin/tarfiles_setup";
my $MAPPER = "$TB/bin/mapper";
my $VTOPGEN = "$TB/bin/vtopgen";
my $SNMPIT = "$TB/bin/snmpit";
my $XMLLINT = "/usr/local/bin/xmllint";
my $PRERENDER = "$TB/libexec/vis/prerender";
my $EMULAB_PEMFILE = "@prefix@/etc/genicm.pem";
# Just one of these, at Utah.
......@@ -338,6 +341,35 @@ sub CreateSliver($)
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
#
# Watch for a placeholder slice and update it.
#
if ($slice->isplaceholder()) {
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
#
# Confirm that the slice certificate is the same.
#
if ($slice->cert() ne $credential->target_cert()->cert()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice certificate mismatch");
}
my $user =
GeniCM::CreateUserFromCertificate($credential->owner_cert());
if (!defined($user)) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create user");
}
if ($slice->ConvertPlaceholder($user) != 0) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not convert placeholder");
}
$slice->UnLock();
}
if (defined($aggregate)) {
return GeniResponse->Create(GENIRESPONSE_REFUSED, undef,
"Must delete existing slice first");
......@@ -403,7 +435,7 @@ sub CreateSliver($)
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not start sliver");
}
GeniCM::UpdateManifest($slice);
# GeniCM::UpdateManifest($slice);
$sliver_manifest = $aggregate->GetManifest(1);
if (!defined($sliver_manifest)) {
print STDERR "CreateSliver: Could not get manifest for $aggregate\n";
......@@ -779,7 +811,7 @@ sub SliverAction($$$$$)
if (GeniResponse::IsResponse($response));
if ($action eq "start" || $action eq "restart") {
GeniCM::UpdateManifest($slice);
# GeniCM::UpdateManifest($slice);
}
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_SUCCESS);
......@@ -1071,6 +1103,37 @@ sub GetTicket($)
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN(), undef,
"Credential does not match the URN");
}
#
# Watch for a placeholder slice and update it.
#
if ($slice->isplaceholder()) {
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
#
# Confirm that the slice certificate is the same.
#
if ($slice->cert() ne $credential->target_cert()->cert()) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice certificate mismatch");
}
my $user =
GeniCM::CreateUserFromCertificate($credential->owner_cert());
if (!defined($user)) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not create user");
}
if ($slice->ConvertPlaceholder($user) != 0) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not convert placeholder");
}
$slice->UnLock();
}
#
# GetTicket applies only to slices that are not active. Must
# use UpdateSliver() for an active sliver.
......@@ -1473,12 +1536,271 @@ sub ListHistory($)
"type" => $type});
}
sub ReserveVlanTags($)
{
my ($argref) = @_;
my $credentials = $argref->{'credentials'};
my $slice_urn = $argref->{'slice_urn'};
my $slice_cert = $argref->{'slice_cert'};
my $rspecstr = $argref->{'rspec'};
my $linkname = $argref->{'linkname'};
my $taglist = $argref->{'taglist'};
my $response;
my $actualtag;
# List of vlans to delete after getting the tags.
my @delete = ();
my %linkmap = ();
require Lan;
if (! (defined($credentials) && defined($slice_cert) &&
defined($taglist) &&
defined($linkname) && defined($slice_urn) && defined($rspecstr))) {
return GeniResponse->MalformedArgsResponse("Missing arguments");
}
if (! ($linkname =~ /^[-\w]*$/)) {
return GeniResponse->MalformedArgsResponse("Bad linkname");
}
foreach my $tag (@{ $taglist }) {
if (! ($tag =~ /^\d*$/)) {
return GeniResponse->MalformedArgsResponse("Bad tag in list");
}
}
my $credential = CheckCredentials($credentials);
return $credential
if (GeniResponse::IsResponse($credential));
$credential->HasPrivilege( "pi" ) or
$credential->HasPrivilege( "bind" ) or
return GeniResponse->Create( GENIRESPONSE_FORBIDDEN, undef,
"Insufficient privilege" );
my (undef,$callertype,$callerid) = GeniHRN::Parse($credential->owner_urn());
if (! ($callertype eq "authority" && $callerid eq "cm")) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef, "Not a CM");
}
my $slice_certificate = GeniCertificate->LoadFromString($slice_cert);
if (!defined($slice_certificate)) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Bad slice certificate");
}
if ($slice_urn ne $slice_certificate->urn()) {
return GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice URN mismatch");
}
my $slice = GeniSlice->Lookup($slice_urn);
if (defined($slice)) {
#
# Already exists locally.
#
if ($slice->Lock() != 0) {
return GeniResponse->BusyResponse();
}
#
# Confirm that the certificate is the same.
#
if ($slice->cert() ne $slice_cert) {
$response = GeniResponse->Create(GENIRESPONSE_FORBIDDEN, undef,
"Slice certificate mismatch");
goto done;
}
}
else {
#
# Create a placeholder slice. Have to watch for a concurrent
# slice creation through the normal path, in which case the user
# needs to set if the slice was first created on this path.
#
my ($auth,$type,$id) = GeniHRN::Parse($slice_urn);
my $sa_urn = GeniHRN::Generate($auth, "authority", "sa");
my $authority = GeniAuthority->CreateFromRegistry("SA", $sa_urn);
if (!defined($authority)) {
print STDERR "Could not create authority from registry\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto done;
}
$slice = GeniSlice->Create($slice_certificate,
undef, $authority, undef, 1);
if (!defined($slice)) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto done;
}
# Slice is returned locked. We will not bother to remove this slice
# on failure since it will get aged out automatically, and it avoids
# dealing with concurrency issues on this path.
}
my $slice_experiment = GeniCM::GeniExperiment($slice);
if (!defined($slice_experiment)) {
print STDERR "Could not create new Geni slice experiment!\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto done;
}
my $pid = $slice_experiment->pid();
my $eid = $slice_experiment->eid();
#
# Run xmllint on the rspec to catch format errors.
#
my ($fh, $filename) = tempfile(UNLINK => 0);
if (!defined($fh)) {
print STDERR "Could not create temp file for rspec\n";
$response = GeniResponse->Create(GENIRESPONSE_ERROR);
goto done;
}
print $fh $rspecstr;
close($fh);
my $xmlerrors = `$XMLLINT --noout $filename 2>&1`;
unlink($filename);
if ($?) {
$response = GeniResponse->Create(GENIRESPONSE_BADARGS,
$xmlerrors,
"rspec is not well formed");
goto done;
}
my $rspec = GeniXML::Parse($rspecstr);
if (! defined($rspec)) {
$response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,