Commit 02083681 authored by Leigh B. Stoller's avatar Leigh B. Stoller

A set of changes to allow real jails using our existing jails

support. Can even create multiple jailed nodes on the same physical
node. Sorry, no sharing of physical nodes yet (between slices).
Also no link support yet; coming later.

The syntax is an extension of the current hack syntax:

        " <node uuid=\"" + node_uuid + "\" " +\
        "       nickname=\"geni1\" "+\
        "       phys_nickname=\"geni1\" "+\
        "       virtualization_type=\"emulab-vnode\" " +\
        "       virtualization_subtype=\"emulab-jail\"> " +\
        " </node>"

This only works on sites that already can do jails.
parent cdbb96a4
......@@ -38,6 +38,7 @@ my $OURDOMAIN = "@OURDOMAIN@";
my $PGENIDOMAIN = "@PROTOGENI_DOMAIN@";
my $SIGNCRED = "$TB/sbin/signgenicred";
my $VERIFYCRED = "$TB/sbin/verifygenicred";
my $NODEREBOOT = "$TB/bin/node_reboot";
# Cache of instances to avoid regenerating them.
my %aggregates = ();
......@@ -511,7 +512,13 @@ sub NewCredential($$)
}
#
# Start all the slivers in the aggregate.
# Start all the slivers in the aggregate. Start is special since it
# sorta means reboot, and the only thing we reboot are nodes. And,
# since we might have multiple vnodes on a pnode, we want to be efficient
# about it.
#
# XXX Is is assumed that there is a single toplevel aggregate for the
# slice, so we can get all the nodes.
#
sub Start($)
{
......@@ -525,11 +532,42 @@ sub Start($)
print STDERR "Could not get sliver list for $self\n";
return -1;
}
my %reboots = ();
foreach my $sliver (@slivers) {
if ($sliver->Start() != 0) {
print STDERR "Could not start $sliver in $self\n";
next;
next
if (ref($sliver) ne "GeniSliver::Node");
my $node = Node->Lookup($sliver->uuid());
if (!defined($node)) {
print STDERR "Could not map $sliver to a node\n";
return -1;
}
# Remote nodes are handled special.
if ($node->isremotenode()) {
$sliver->Start();
}
# We assume local nodes are not shared, so reboot pnode; this
# gets all the vnodes on that pnode.
if ($node->isvirtnode()) {
$reboots{$node->phys_nodeid()} = 1;
}
# node_reboot is smart enough to know that if a pnode is rebooted
# it can ignore the vnodes on it, so do not optimize this here.
$reboots{$node->node_id} = 1;
}
my @node_ids = keys(%reboots);
if (@node_ids) {
#
# Should waiting be an option?
#
system("$NODEREBOOT -s @node_ids");
return -1
if ($?);
}
return 0;
}
......@@ -587,8 +625,8 @@ sub UnProvision($)
push(@links, $sliver);
}
elsif (ref($sliver) eq "GeniAggregate") {
# Not really a node, but a sub aggregate.
unshift(@nodes, $sliver);
print STDERR "Unprovision: Unknown aggregate $sliver in $self\n";
return -1;
}
elsif (ref($sliver) eq "GeniSliver::Node") {
push(@nodes, $sliver);
......
......@@ -392,7 +392,6 @@ sub GetTicket($)
# Firewall hack; just a flag in the rspec for now.
#
if (exists($rspec->{'needsfirewall'}) && $rspec->{'needsfirewall'}) {
print STDERR "firewall: " . $rspec->{'needsfirewall'} . "\n";
if ($slice->SetFirewallFlag($rspec->{'needsfirewall'}) != 0) {
$slice->UnLock();
return GeniResponse->Create(GENIRESPONSE_ERROR);
......@@ -414,7 +413,8 @@ sub GetTicket($)
# XXX Simpleminded.
#
my %namemap = ();
my %uuidmap = ();
my %physmap = ();
my %virtmap = ();
my @nodeids = ();
my @dealloc;
my $pid = $experiment->pid();
......@@ -423,14 +423,21 @@ sub GetTicket($)
foreach my $ref (@{$rspec->{'node'}}) {
my $resource_uuid = $ref->{'uuid'};
my $node_nickname = $ref->{'nickname'};
my $phys_nickname = $ref->{'phys_nickname'};
my $virtualization_type = $ref->{'virtualization_type'};
my $sliver_uuid = NewUUID();
my $node;
#
# Mostly for debugging right now, allow a wildcard.
#
if ($resource_uuid eq "*") {
$node = FindFreeNode($virtualization_type, @nodeids);
if (defined($phys_nickname) && exists($physmap{$phys_nickname})) {
$node = $physmap{$phys_nickname};
}
else {
$node = FindFreeNode($virtualization_type, @nodeids);
}
if (!defined($node)) {
$response = GeniResponse->Create(GENIRESPONSE_UNAVAILABLE,
......@@ -451,6 +458,10 @@ sub GetTicket($)
goto bad;
}
}
# We assign the sliver uuids here so that user can associate
# slivers with parts of the rspec later.
$ref->{'sliver_uuid'} = $sliver_uuid;
#
# Widearea nodes do not need to be allocated, but for now all
# I allow is a plabdslice node.
......@@ -462,11 +473,11 @@ sub GetTicket($)
"Only plab widearea nodes");
goto bad;
}
next;
goto skipalloc;
}
#
# See if the node is already reserved.
# See if the node is already reserved.
#
my $reservation = $node->Reservation();
if (defined($reservation)) {
......@@ -475,11 +486,17 @@ sub GetTicket($)
"$resource_uuid ($node) not available");
goto bad;
}
push(@nodeids, $node->node_id());
$uuidmap{$resource_uuid} = $node;
# watch for duplicates, as for multiple vnodes on a pnode.
push(@nodeids, $node->node_id())
if (! grep {$_ eq $node->node_id()} @nodeids);
skipalloc:
$virtmap{$sliver_uuid} = $node;
# For wildcarded nodes in links.
$namemap{$node_nickname} = $node
$namemap{$node_nickname} = $sliver_uuid
if (defined($node_nickname));
$physmap{$phys_nickname} = $node
if (defined($phys_nickname));
}
#
......@@ -544,11 +561,10 @@ sub GetTicket($)
my $node_uuid = $ref->{'node_uuid'};
my $node_nickname = $ref->{'node_nickname'};
my $iface_name = $ref->{'iface_name'};
my $node;
if ($node_uuid eq "*" && !defined($node_nickname)) {
if (!defined($node_nickname)) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Need nickname for wildcarded link");
"Need nickname for links");
goto bad;
}
......@@ -563,27 +579,19 @@ sub GetTicket($)
next;
}
# Map the nickname to the actual sliver (node) request.
my $sliver_uuid = $namemap{$node_nickname};
my $node = $virtmap{$sliver_uuid};
#
# First map the node if its wildcarded.
# Fill in the physnode if its wildcarded.
#
if ($node_uuid eq "*") {
if (!exists($namemap{$node_nickname})) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not map wildcarded link");
goto bad;
}
$node = $namemap{$node_nickname};
$ref->{'node_uuid'} = $node->uuid();
}
elsif (!exists($uuidmap{$node_uuid})) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"Could not map link node");
goto bad;
}
else {
$node = $uuidmap{$node_uuid};
}
# and the sliver_uuid we assigned above.
$ref->{'sliver_uuid'} = $sliver_uuid;
#
# Now do wildcarded interfaces.
#
......@@ -823,6 +831,7 @@ sub ModifySliver($$$$$$)
#
# Figure out what nodes to allocate or free.
#
my %namemap = ();
my %nodelist = ();
my %linklist = ();
my %toalloc = ();
......@@ -847,7 +856,7 @@ sub ModifySliver($$$$$$)
}
foreach my $s (@slivers) {
if (ref($s) eq "GeniSliver::Node") {
$nodelist{$s->resource_uuid()} = $s;
$nodelist{$s->uuid()} = $s;
}
elsif (ref($s) eq "GeniAggregate::Link" ||
ref($s) eq "GeniAggregate::Tunnel") {
......@@ -1065,9 +1074,11 @@ sub ModifySliver($$$$$$)
my @plabnodes = ();
foreach my $ref (@{$rspec->{'node'}}) {
my $resource_uuid = $ref->{'uuid'};
my $sliver_uuid = $ref->{'sliver_uuid'};
# Already in the aggregate?
next
if (grep {$_ eq $resource_uuid} keys(%nodelist));
if (grep {$_ eq $sliver_uuid} keys(%nodelist));
my $node = Node->Lookup($resource_uuid);
if (!defined($node)) {
......@@ -1077,12 +1088,13 @@ sub ModifySliver($$$$$$)
my $sliver = GeniSliver::Node->Create($slice,
$owner,
$resource_uuid,
$sliver_uuid,
$ref);
if (!defined($sliver)) {
$message = "Could not create GeniSliver object for $resource_uuid";
goto bad;
}
$slivers{$resource_uuid} = $sliver;
$slivers{$sliver_uuid} = $sliver;
#
# Remove this from %toalloc; if there is an error, the slivers are
......@@ -1092,7 +1104,14 @@ sub ModifySliver($$$$$$)
delete($toalloc{$resource_uuid});
# Used below.
push(@allocated, $node);
push(@allocated, $node)
if (! grep {$_->node_id() eq $node->node_id()} @allocated);
# Add to the aggregate.
if ($sliver->SetAggregate($aggregate) != 0) {
$message = "Could not set aggregate for $sliver to $aggregate";
goto bad;
}
# See below; setup all pnodes at once.
if ($node->isremotenode()) {
......@@ -1130,10 +1149,13 @@ sub ModifySliver($$$$$$)
my $node1ref = (@{$linkref->{'linkendpoints'}})[0];
my $node2ref = (@{$linkref->{'linkendpoints'}})[1];
my $node1sliver = $slivers{$node1ref->{'node_uuid'}} ||
$nodelist{$node1ref->{'node_uuid'}};
my $node2sliver = $slivers{$node2ref->{'node_uuid'}} ||
$nodelist{$node2ref->{'node_uuid'}};
my $node1sliver_uuid = $namemap{$node1ref->{'sliver_uuid'}};
my $node2sliver_uuid = $namemap{$node2ref->{'sliver_uuid'}};
my $node1sliver = $slivers{$node1sliver_uuid} ||
$nodelist{$node1sliver_uuid};
my $node2sliver = $slivers{$node2sliver_uuid} ||
$nodelist{$node2sliver_uuid};
my $tunnel = GeniAggregate::Tunnel->Create($slice,
$owner,
......@@ -1146,6 +1168,12 @@ sub ModifySliver($$$$$$)
goto bad;
}
$slivers{$tunnel->uuid()} = $tunnel;
# Add to the aggregate.
if ($tunnel->SetAggregate($aggregate) != 0) {
$message = "Could not set aggregate for $tunnel to $aggregate";
goto bad;
}
next;
}
......@@ -1157,17 +1185,26 @@ sub ModifySliver($$$$$$)
}
$slivers{$linkaggregate->uuid()} = $linkaggregate;
# Add to the aggregate.
if ($linkaggregate->SetAggregate($aggregate) != 0) {
$message = "Could not set aggregate for $linkaggregate ".
"to $aggregate";
goto bad;
}
foreach my $ref (@{$rspec->{'link'}->{$linkname}->{'linkendpoints'}}) {
my $node_uuid = $ref->{'node_uuid'};
my $iface_name = $ref->{'iface_name'};
my $nodesliver = $slivers{$node_uuid} || $nodelist{$node_uuid};
my $sliver_uuid = $ref->{'sliver_uuid'};
my $iface_name = $ref->{'iface_name'};
my $nodesliver = $slivers{$sliver_uuid} ||
$nodelist{$sliver_uuid};
if (!defined($nodesliver)) {
$message = "Link $linkname specifies a non-existent node";
goto bad;
}
my $nodeobject= Node->Lookup($node_uuid);
my $nodeobject= Node->Lookup($nodesliver->resource_uuid());
if (!defined($nodeobject)) {
$message = "Could not find node object for $node_uuid";
$message = "Could not find node object for $nodesliver";
goto bad;
}
my $interface = Interface->LookupByIface($nodeobject, $iface_name);
......@@ -1212,11 +1249,6 @@ sub ModifySliver($$$$$$)
if (!$impotent && $sliver->Provision() != 0) {
$message = "Could not provision $sliver";
goto bad;
}
if ($sliver->SetAggregate($aggregate) != 0) {
$message = "Could not set aggregate for $sliver to $aggregate";
goto bad;
}
}
......@@ -2050,7 +2082,7 @@ sub SliceStatus($)
next
if ($sliver->resource_type() ne "Node");
my $node_uuid = $sliver->resource_uuid();
my $node_uuid = $sliver->uuid();
my $node = Node->Lookup($node_uuid);
if (!defined($node)) {
$slice->UnLock();
......@@ -2150,7 +2182,7 @@ sub SliverStatus($)
next
if ($sliver->resource_type() ne "Node");
my $node_uuid = $sliver->resource_uuid();
my $node_uuid = $sliver->uuid();
my $node = Node->Lookup($node_uuid);
if (!defined($node)) {
$slice->UnLock();
......@@ -2303,7 +2335,7 @@ sub CleanupDeadSlice($;$)
print "Calling undoFWNodes ...\n";
if (undoFWNodes($experiment) != 0) {
if (undoFWNodes($experiment, 1) != 0) {
print STDERR "FireWall cleanup failed\n";
return -1;
}
......@@ -2389,6 +2421,15 @@ sub CleanupDeadSlice($;$)
my $experiment = $slice->GetExperiment();
if (defined($experiment)) {
my @pnodes = $experiment->NodeList(1, 1);
my $pnodes = scalar(@pnodes);
# Ignore the firewall node in this test; released in endexp.
$pnodes--
if ($slice->needsfirewall());
if ($pnodes != 0) {
print STDERR "There are still nodes allocated to $experiment!\n";
return -1;
}
$experiment->LockDown(0);
my $pid = $experiment->pid();
......@@ -2438,7 +2479,7 @@ sub GeniExperiment($)
or return undef;
print NS "source tb_compat.tcl\n";
print NS "set ns [new Simulator]\n";
print NS "tb-set-security-level Yellow\n";
print NS "tb-set-security-level Blue\n";
print NS "\$ns run\n";
close(NS);
}
......
......@@ -24,6 +24,7 @@ use GeniUsage;
# Hate to import all this crap; need a utility library.
use emutil qw(TBGetUniqueIndex);
use Experiment;
use OSinfo;
use English;
use XML::Simple;
use Data::Dumper;
......@@ -690,11 +691,10 @@ use Experiment;
use XML::Simple;
use libdb qw(TBDB_ALLOCSTATE_RES_INIT_DIRTY);
sub Create($$$$$)
sub Create($$$$$$)
{
my ($class, $slice, $user, $resource_uuid, $rspec) = @_;
my ($class, $slice, $user, $resource_uuid, $sliver_uuid, $rspec) = @_;
my $virtualization_type = $rspec->{'virtualization_type'};
my $uuid = $resource_uuid;
my $experiment = $slice->GetExperiment();
if (!defined($experiment)) {
......@@ -743,11 +743,19 @@ sub Create($$$$$)
return undef;
}
}
elsif (exists($rspec->{'virtualization_subtype'})) {
#
# See if a jail node is requested.
#
if ($rspec->{'virtualization_subtype'} eq "emulab-jail") {
$vtype = "pcvm";
}
}
#
# Create a virtual node on the physnode.
#
my @vnodes;
my @vnodes = ($sliver_uuid);
if (Node::CreateVnodes(\@vnodes,
{"pid" => $experiment->pid(),
"eid" => $experiment->eid(),
......@@ -759,10 +767,9 @@ sub Create($$$$$)
return undef;
}
my $vnode = Node->Lookup($vnodes[0]);
$uuid = $vnode->uuid();
$hrn = "${PGENIDOMAIN}." . $vnode->node_id()
}
return GeniSliver->Create($slice, $user, $uuid, $resource_uuid,
return GeniSliver->Create($slice, $user, $sliver_uuid, $resource_uuid,
"Node", $hrn, $nickname, $rspec);
}
......@@ -808,7 +815,7 @@ sub Provision($;$)
}
my $pid = $experiment->pid();
my $eid = $experiment->eid();
if ($experiment->InsertVirtNode($node) != 0) {
print STDERR "Could not add virtnode entry for $node to $self\n";
return -1;
......@@ -841,19 +848,55 @@ sub Provision($;$)
print STDERR "Could not get pnode object for $node\n";
return -1;
}
if ($experiment->InsertVirtNode($pnode) != 0) {
print STDERR "Could not add virtnode entry for $pnode to $self\n";
return -1;
# Do this early.
#
# XXX This is the wrong place to do this.
my $osid = undef;
if (exists($self->rspec()->{'virtualization_subtype'}) &&
$self->rspec()->{'virtualization_subtype'} eq "emulab-jail") {
my $osinfo = OSinfo->LookupByName("FBSD-STD");
if (!defined($osinfo)) {
print STDERR "Could not find osinfo for FBSD-STD\n";
return -1;
}
# Need to resolve to the specfic OSID.
my $nextosinfo = $osinfo->ResolveNextOSID();
if (!defined($nextosinfo)) {
print STDERR "Could not resolve nextosid for $osinfo\n";
return -1;
}
$osid = $nextosinfo->osid();
#
# Gack.
#
my $cmdline = "/kernel.jail";
if ($nextosinfo->OSBootCmd("vnodehost", \$cmdline) != 0) {
print STDERR "Could not resolve cmdline for $nextosinfo\n";
return -1;
}
$pnode->Update({"def_boot_cmd_line" => $cmdline});
$pnode->ModifyReservation({"sharing_mode" => "shared"});
}
# The pnode might have multiple vnodes on it ... only insert once.
if (! $experiment->HasVirtNode($pnode)) {
if ($experiment->InsertVirtNode($pnode) != 0) {
print STDERR "Could not add virtnode for $pnode to $self\n";
return -1;
}
$pnode->ModifyReservation({"genisliver_idx" => $self->idx()});
}
$pnode->ModifyReservation({"genisliver_idx" => $self->idx()});
# Not redirected. Use local tmcd anyway.
$node->ModifyReservation({"genisliver_idx" => $self->idx()})
if (!$redirected);
# Set it to boot the default OS.
if ($pnode->SelectOS() != 0) {
# Set it to boot the proper os.
if ($pnode->SelectOS($osid) != 0) {
return -1;
}
}
......@@ -866,6 +909,9 @@ sub Provision($;$)
return -1;
}
}
# See below; remote nodes are currently assumed to be running.
$self->SetStatus("ready")
if (!$node->isremotenode());
return 0;
}
......@@ -924,13 +970,29 @@ sub UnProvision($)
print STDERR "Could not get pnode object for $pnode_id\n";
return -1;
}
if ($experiment->DeleteVirtNode($pnode) != 0) {
print STDERR
"Could remove virtnode entry for $pnode from $self\n";
#
# If this is the last virtnode on the physnode, release the
# physnode too.
#
my @vnodes;
if ($pnode->VirtualNodes(\@vnodes) != 0) {
print STDERR "Could not get vnode list for $pnode\n";
return -1;
}
system("$NFREE -x -q $pid $eid $pnode_id");
$pnode->Refresh();
if (scalar(@vnodes) > 1) {
system("$NFREE -q $pid $eid $node_id");
}
else {
if ($experiment->DeleteVirtNode($pnode) != 0) {
print STDERR
"Could remove virtnode entry for $pnode from $self\n";
return -1;
}
system("$NFREE -x -q $pid $eid $pnode_id");
$pnode->Refresh();
}
}
else {
system("$NFREE -q $pid $eid $node_id");
......@@ -987,19 +1049,10 @@ sub Start($)
goto done;
}
#
# Reboot pnode if not already running.
#
if ($self->rspec()->{'virtualization_type'} eq "emulab-vnode" &&
$self->status() eq "created") {
$node_id = $node->phys_nodeid();
}
#
# Reboot and wait?
#
system("$NODEREBOOT -s $node_id");
$self->SetStatus("ready")
if (!$?);
return -1
if ($?);
}
......
......@@ -648,7 +648,11 @@ sub Release($$)
my $reservation = $node->Reservation();
next
if (!defined($reservation));
# Watch for duplicates, as in multiple vnodes on a pnode.
next
if (grep {$_ eq $node->node_id()} @nodeids);
if ($reservation->SameExperiment($experiment)) {
push(@nodeids, $node->node_id());
push(@nodes, $node);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment