From fd015646586e767e53561b438151317e67aa8d8d Mon Sep 17 00:00:00 2001 From: "Leigh B. Stoller" Date: Fri, 18 Dec 2009 22:30:13 +0000 Subject: [PATCH] Changes to support the SPP nodes. My approach was a little odd. What I did was create node table entries for the three SPP nodes. These are designated as local, shared nodes, reserved to a holding experiment. This allowed me to use all of the existing shared node pool support, albeit with a couple of tweaks in libvtop that I will not bother to mention since they are hideous (another thing I need to fix). The virtual nodes that are created on the spp nodes are figments; they will never be setup, booted or torn down. They exist simply as place holders in the DB, in order hold the reserved bandwidth on the network interfaces. In other words, you can create as many of these imaginary spp nodes (in different slices if you like) as there are interfaces on the spp node. Or you can create a single spp imaginary node with all of the interfaces. You get the idea; its the reserved bandwidth that drives the allocation. There are also some minor spp specific changes in vnode_setup.in to avoid trying to generalize things. I will return to this later as needed. See this wiki page for info and sample rspecs: https://www.protogeni.net/trac/protogeni/wiki/SPPNodes --- protogeni/lib/GeniCM.pm.in | 96 ++++++++++++++++++++++++++++++++++++ protogeni/lib/GeniCMV2.pm.in | 3 ++ tbsetup/libvtop.pm.in | 80 +++++++++++++++++++++--------- tbsetup/ptopgen.in | 10 ++-- tbsetup/vnode_setup.in | 11 +++++ 5 files changed, 173 insertions(+), 27 deletions(-) diff --git a/protogeni/lib/GeniCM.pm.in b/protogeni/lib/GeniCM.pm.in index d74622e68..19487b19e 100644 --- a/protogeni/lib/GeniCM.pm.in +++ b/protogeni/lib/GeniCM.pm.in @@ -41,6 +41,7 @@ use libdb qw(TBGetSiteVar EXPTSTATE_SWAPPED EXPTSTATE_ACTIVE TBOPSPID TBDB_NODESTATE_TBFAILED); use User; use Node; +use Lan; use OSinfo; use Image; use Interface; @@ -738,6 +739,7 @@ sub GetTicketAuxAux($$$$$$$$) $ref->{'exclusive'} = $exclusive = 0; # Kludge for libvtop. $virtexperiment->multiplex_factor(1); + $virtexperiment->encap_style("vlan"); } } else { @@ -2095,6 +2097,34 @@ sub SliverWorkAux($$$$$$$) $message = "Could not set up vlans"; goto bad; } + foreach my $linkref (@{$rspec->{'link'}}) { + my $vname = $linkref->{"nickname"} || $linkref->{"virtual_id"}; + my $vlan; + + my $lan = Lan->Lookup($experiment, $vname, 1); + if (!defined($lan)) { + print STDERR "No lan object for $vname\n"; + next; + } + if ($lan->type() eq "vlan") { + $vlan = VLan->Lookup($experiment, $vname); + } + elsif (defined($lan->link())) { + $vlan = VLan->Lookup($lan->link()); + } + + if (!defined($vlan)) { + print STDERR "Could not find a vlan for $vname\n"; + next; + } + my $tag; + $vlan->GetTag(\$tag); + if (!defined($tag)) { + print STDERR "No tag for $vlan\n"; + next; + } + $linkref->{"vlantag"} = $tag; + } } # Set up plab nodes all at once. @@ -3955,5 +3985,71 @@ sub CreateRspec($) { } +# +# Update the manifest with the new vlan tags. +# +sub UpdateManifest($) +{ + my ($slice) = @_; + + my $experiment = GeniExperiment($slice); + if (!defined($experiment)) { + print STDERR "No local experiment for $slice\n"; + return -1; + } + my $aggregate = GeniAggregate->SliceAggregate($slice); + if (!defined($aggregate)) { + print STDERR "No aggregate for $slice\n"; + return -1; + } + my $rspec = $aggregate->GetManifest(0); + if (!defined($rspec)) { + print STDERR "No manifest for $slice/$aggregate\n"; + return -1; + } + my @vlanlist = (); + my %vlantags = (); + VLan->ExperimentVLans($experiment, \@vlanlist); + foreach my $vlan (@vlanlist) { + my $tag; + $vlan->GetTag(\$tag); + if (!defined($tag)) { + print STDERR "UpdateManifest: No tag for $vlan\n"; + } + $vlantags{$vlan->vname()} = $tag; + } + foreach my $linkref (@{$rspec->{'link'}}) { + my $vname = $linkref->{"nickname"} || $linkref->{"virtual_id"}; + my $tag = (exists($vlantags{$vname}) ? $vlantags{$vname} : undef); + if (!defined($tag)) { + delete($linkref->{"vlantag"}) + if (exists($linkref->{"vlantag"})); + } + else { + $linkref->{"vlantag"} = $tag; + } + + } + my $manifest = + eval { XMLout($rspec, "NoAttr" => 1, RootName => "manifest") }; + if ($@) { + print STDERR "Manifest: XMLout error: $@\n"; + print STDERR Dumper($rspec); + return -1; + } + + # + # Move this elsewhere. + # + $manifest = DBQuoteSpecial($manifest); + my $slice_uuid = $slice->uuid(); + + DBQueryWarn("update geni_manifests set ". + " manifest=$manifest ". + "where slice_uuid='$slice_uuid'"); + + return 0; +} + # _Always_ make sure that this 1 is at the end of the file... 1; diff --git a/protogeni/lib/GeniCMV2.pm.in b/protogeni/lib/GeniCMV2.pm.in index 4ceb9760b..4283fd83f 100644 --- a/protogeni/lib/GeniCMV2.pm.in +++ b/protogeni/lib/GeniCMV2.pm.in @@ -655,6 +655,9 @@ sub SliverAction($$$$) goto bad if (GeniResponse::IsResponse($response)); + if ($action eq "start") { + GeniCM::UpdateManifest($slice); + } $slice->UnLock(); return GeniResponse->Create(GENIRESPONSE_SUCCESS); } diff --git a/tbsetup/libvtop.pm.in b/tbsetup/libvtop.pm.in index aad59ab7a..80222e12c 100644 --- a/tbsetup/libvtop.pm.in +++ b/tbsetup/libvtop.pm.in @@ -1094,6 +1094,7 @@ sub LoadVirtLans($) if ($vlanmember->is_accesspoint()); $virtlan->_sharednodes(0); $virtlan->_geninodes(0); + $virtlan->_needvlan(0); if (defined($encap) && ($encap eq "vtun" || $encap eq "gre" || $encap eq "egre")) { @@ -1171,6 +1172,16 @@ sub LoadVirtLans($) # Temporary, for generating rspecs. push(@{ $vlanmember->virt_node()->_virtifaces() }, $vlanmember); + + # Terrible. + $vlanmember->_reservebw(0); + $vlanmember->_needtrunk(0); + if ($vlanmember->virt_node()->type() eq "sppvm") { + $vlanmember->_reservebw($bandwidth); + $vlanmember->_needtrunk(1); + $virtlan->_needvlan(1); + $self->printdb(" Forcing $vlanmember to reserve shared bandwidth\n"); + } } return 0; } @@ -1767,14 +1778,16 @@ sub GenVirtLans($) if ($sharednodes) { my $newencap; - - if ($sharednodes != $allnodes) { + + if ($sharednodes != $allnodes || $vlan->_needvlan()) { # # Change the encap type to vlan since that is supported. # if ($nodesdo{"vlan"} == $allnodes) { $newencap = "vlan"; } + # Force this on. + $emulated = 1; } else { if ($nodesdo{"veth-en"} == $allnodes) { @@ -1784,11 +1797,15 @@ sub GenVirtLans($) elsif ($nodesdo{"veth-ne"} == $allnodes) { $newencap = "veth-ne"; } + elsif ($nodesdo{"vlan"} == $allnodes) { + $newencap = "vlan"; + } + # Force this on. + $emulated = 1; } if (defined($newencap)) { - $emulated = 1; $encapval = $newencap; - $vlan->_emulated(1); + $vlan->_emulated($emulated); $vlan->_encapstyle($newencap); $self->printdb("Converting encapstyle to ". @@ -3551,8 +3568,7 @@ sub AllocNodes($) my $reservation = $pnode->Reservation(); if (!defined($reservation) || (! $reservation->SameExperiment($self->experiment()) && - ! (defined($pnode->sharing_mode()) && - $pnode->sharing_mode() eq "shared_local"))) { + ! $pnode->erole() eq "sharedhost")) { tbinfo("$pnode is not in shared mode.\n"); $rerun++; } @@ -4216,8 +4232,9 @@ sub InterpLinks($) # an underlying vlan. # # - if (! ($virtnodeA->_onsharednode() && - $virtnodeB->_onsharednode())) { + if ((!($virtnodeA->_onsharednode() && + $virtnodeB->_onsharednode())) || + $virtlan->_needvlan()) { my $lanid = "v" . "$lan" . $vlanid++; $protovlan = ProtoLan->Create($experiment, $lanid, @@ -4248,10 +4265,20 @@ sub InterpLinks($) # # We need to reserve the shared bandwidth. # - if (exists($self->delaylinks()->{$plink})) { - my (undef,$bandwidth,undef,undef, - undef,$rbandwidth,undef) = - @{$self->delaylinks()->{$plink}}; + if (exists($self->delaylinks()->{$plink}) || + $member0->_reservebw() || $member1->_reservebw()) { + my ($bandwidth,$rbandwidth) = 0; + + if (exists($self->delaylinks()->{$plink})) { + (undef,$bandwidth,undef,undef, + undef,$rbandwidth,undef) = + @{$self->delaylinks()->{$plink}}; + } + else { + $bandwidth = $member0->_reservebw(); + $rbandwidth = $member1->_reservebw(); + } + if ($virtnodeA->_onsharednode() && !$self->impotent() && $virtifaceA->ReserveSharedBandwidth($bandwidth)) { @@ -4372,7 +4399,8 @@ sub InterpLinks($) my $protovlan; if (!$virtlan->_sharednodes() || - $virtlan->_sharednodes() != $virtlan->memberlist()) { + $virtlan->_sharednodes() != $virtlan->memberlist() || + $virtlan->_needvlan()) { if (exists($protovlans{$lan})) { $protovlan = $protovlans{$lan}; } @@ -4400,10 +4428,18 @@ sub InterpLinks($) # # We need to reserve the shared bandwidth. # - if (exists($self->delaylinks()->{$plink})) { - my (undef,$bandwidth,undef,undef, - undef,$rbandwidth,undef) = - @{$self->delaylinks()->{$plink}}; + if (exists($self->delaylinks()->{$plink}) || + $member0->_reservebw()) { + my ($bandwidth,$rbandwidth) = 0; + + if (exists($self->delaylinks()->{$plink})) { + (undef,$bandwidth,undef,undef, + undef,$rbandwidth,undef) = + @{$self->delaylinks()->{$plink}}; + } + else { + $bandwidth = $rbandwidth = $member0->_reservebw(); + } my $maxbw = max($bandwidth, $rbandwidth); if ($virtnodeA->_onsharednode() && !$self->impotent() && @@ -4774,10 +4810,9 @@ sub InitializePhysNode($$$) # this experiment; just skip it since it was setup when its # holding experiment swapped it in. # - if (defined($pnode->sharing_mode()) && - $pnode->sharing_mode() eq "shared_local" && - !$pnode->isvirtnode()) { - $self->printdb("InitPnode: Skipping shared_local node $pnode\n"); + if (defined($pnode->sharing_mode()) && !$pnode->isvirtnode() && + $pnode->erole() eq "sharedhost") { + $self->printdb("InitPnode: Skipping shared host $pnode\n"); return 0; } @@ -5169,7 +5204,8 @@ sub NewVirtIface($$$$;$) # If the pnode is a shared host, we do not want to do this; the physical # interfaces are all set up the right way, do not mess it up. # - if (defined($pport) && $isvdev && !$pnode->_sharedhost()) { + if (defined($pport) && $isvdev && + (!$pnode->_sharedhost() || $member->_needtrunk())) { my $speed = $self->interfacespeedmbps(physinterfacetype($pnode,$pport), "ethernet"); my $trunk = ($type eq "vlan" ? 1 : 0); diff --git a/tbsetup/ptopgen.in b/tbsetup/ptopgen.in index 6d82cd414..d3cf897b3 100644 --- a/tbsetup/ptopgen.in +++ b/tbsetup/ptopgen.in @@ -238,7 +238,7 @@ my %nodetoavailable; $result = DBQueryFatal("SELECT n.node_id, n.eventstate, n.role, n.uuid, " . "nt.isremotenode, " . - "dedicated_wa_types.attrvalue, b.sharing_mode, " . + "dedicated_wa_types.attrvalue, b.erole, " . "n.reserved_pid, b.eid " . "from nodes as n " . "left join reserved as b on n.node_id=b.node_id " . @@ -251,7 +251,7 @@ $result = DBQueryFatal("SELECT n.node_id, n.eventstate, n.role, n.uuid, " . " on nt.type=dedicated_wa_types.type " . "where nt.isvirtnode = 0 or nt.isvirtnode is null;"); while (($node,$eventstate, $role, $uuid, $isremotenode, - $wa_attrvalue, $sharing_mode, + $wa_attrvalue, $erole, $reserved_pid, $reserved_eid) = $result->fetchrow_array) { if (defined($uuid) && $uuid ne "") { @@ -265,8 +265,8 @@ while (($node,$eventstate, $role, $uuid, $isremotenode, || $eventstate eq TBDB_NODESTATE_PXEWAIT || $eventstate eq TBDB_NODESTATE_POWEROFF || $eventstate eq TBDB_NODESTATE_ALWAYSUP); - my $isshared = (defined($sharing_mode) - && $sharing_mode eq "shared_local" + my $isshared = (defined($erole) + && $erole eq "sharedhost" && $useshared && $isup); my $isreserved = (defined($reserved_eid) @@ -528,7 +528,7 @@ if (defined($exempt_eid)) { # In shared mode, allow allocated nodes whose sharing_mode is set. if ($useshared) { $free_condition = "($free_condition or ". - "(b.node_id is not null && b.sharing_mode='shared_local' && ". + "(b.node_id is not null && b.erole='sharedhost' && ". " np.eventstate='" . TBDB_NODESTATE_ISUP . "'))"; } diff --git a/tbsetup/vnode_setup.in b/tbsetup/vnode_setup.in index 7f5303c7d..b6808eee5 100644 --- a/tbsetup/vnode_setup.in +++ b/tbsetup/vnode_setup.in @@ -222,6 +222,17 @@ foreach my $node (@nodes) { next if (!$nodeobj->isvirtnode()); + # Special hack for SPP nodes. Need to generalize this. + if ($shared && $nodeobj->type eq "sppvm") { + if ($mode eq "teardown") { + $nodeobj->SetEventState(TBDB_NODESTATE_SHUTDOWN()); + } + else { + $nodeobj->SetEventState(TBDB_NODESTATE_ISUP()); + } + next; + } + if (($plabonly || $jailonly) and !(($plabonly && $plab) || ($jailonly && (($jailed || $remote) && !($plab || $geninode))))) { -- GitLab