Commit 9512772e authored by Mike Hibler's avatar Mike Hibler

Initial "Inner Plab" support. In your NS file, you declare one node:

tb-set-node-plab-role $plc plc

to make it the PLC node.  Then any number of other nodes are declared as:

tb-set-node-plab-role $plab1 node

to make them inner plab nodes.  Unlike elabinelab, there is no magic
"tb-plab-in-elab" command which implies the topology, you put all the
plab nodes in a LAN or whatever yourself.  This may or may not be a good idea.

Anyway, these NS commands set DB state in virt_nodes and reserved much like
elabinelab.  During swapin, the dhcpd.conf file is rewritten so that
inner plab nodes have their "filename" set to "pxelinux.0" and their
"next-server" set to the designated PLC node.  The PLC node will then be
loaded/booted before anything is done to the inner-plab nodes.  After
it comes up, the inner plab nodes are rebooted and declared as up.
There is a new tmcd command "eplabconfig" (suggestions for a new name
welcom!), which returns info like:

    NAME=plc ROLE=plc IP=155.98.36.3 MAC=00d0b713f57d
    NAME=plab1 ROLE=node IP=155.98.36.10 MAC=0002b3877a4f
    NAME=plab2 ROLE=node IP=155.98.36.34 MAC=00d0b7141057

to just the PLC node (returns nothing to any other node).

The implications of this setup are:

 * The PLC node must act as a TFTP server as we have discussed in the past.
   The TMCC info above is hopefully enough to configure pxelinux, if not
   we can change it.

 * The PLC node is responsible for loading the disks of inner plab nodes.
   This is implied by the setup, where we change the dhcpd.conf file before
   doing anything to the inner nodes.  Thus, once the inner nodes are
   rebooted, they will be talking pxelinux with PLC, and not to boss.
   This step is dubious, as we could no doubt load the disks faster than
   whatever plab uses can.  But it simplified the setup (and is more
   realistic!).  The alternative, which is something that might be useful
   anyway, is to introduce a "state" after which nodes have been reloaded
   but before they are rebooted.  With that, we can reload the plab nodes
   and then change the dhcpd.conf file so when they reboot they start
   talking to the PLC.
parent 940b5538
......@@ -174,7 +174,9 @@ use vars qw(@ISA @EXPORT);
TBExptDestroy TBIPtoNodeID TBNodeBootReset TBNodeStateWait
TBLeaderMailList ExpGroup TBExptSetSwapUID TBExptSetThumbNail
TBNodeAllocCheck TBPlabNodeUsername MarkPhysNodeDown
TBExptIsElabInElab TBBatchUnLockExp TBExptIsBatchExp
TBExptIsElabInElab TBExptIsPlabInElab
TBExptPlabInElabPLC TBExptPlabInElabNodes
TBBatchUnLockExp TBExptIsBatchExp
TBExptFirewall TBNodeFirewall TBExptFirewallAndPort
TBSetExptFirewallVlan TBClearExptFirewallVlan
......@@ -1922,7 +1924,8 @@ sub TBBootWhat($;$)
# WARNING!!!
#
# DO NOT change this function without making corresponding changes to
# pxe/bootinfo_mysql.c.
#
# ALWAYS find exactly the same resulting OSID given the same inputs.
#
my $query_result =
......@@ -3783,6 +3786,71 @@ sub TBExptIsElabInElab($$$;$)
return 1;
}
#
# Get plabinelab info for an experiment. Returns 1 in plabinelab
# if any node in the experiment is either a plab 'plc' or 'node'.
#
# usage TBExptIsPlabInElab(char *pid, char *eid, int \*plabinelab)
# Return 1 if success.
# Return 0 if error.
#
sub TBExptIsPlabInElab($$$)
{
my ($pid, $eid, $plabinelab) = @_;
my $query_result =
DBQueryFatal("select plab_role from virt_nodes ".
"where pid='$pid' and eid='$eid' ".
"and plab_role!='none'");
if ($query_result->numrows == 0) {
$$plabinelab = 0;
} else {
$$plabinelab = 1;
}
return 1;
}
#
# Return the PLC node for a swapped in plabinelab experiment.
# Returns 0 if no PLC. Returns 1 and the name of the node otherwise.
#
sub TBExptPlabInElabPLC($$$)
{
my ($pid, $eid, $plcnode) = @_;
my $query_result =
DBQueryFatal("select node_id from reserved ".
"where pid='$pid' and eid='$eid' and plab_role='plc'");
if ($query_result->numrows > 0) {
my @row = $query_result->fetchrow_array();
if (defined($row[0])) {
$$plcnode = $row[0];
return 1;
}
}
return 0;
}
#
# Return a list of inner plab nodes for a swapped in plabinelab experiment.
# Returns 0 on failure. Returns 1 and a list of nodes otherwise.
#
sub TBExptPlabInElabNodes($$$)
{
my ($pid, $eid, $nodes) = @_;
my $query_result =
DBQueryFatal("select node_id from reserved ".
"where pid='$pid' and eid='$eid' and plab_role='node'");
while (my @row = $query_result->fetchrow_array()) {
push(@{$nodes}, $row[0]);
}
return 0;
}
#
# Similar function for batchmode.
#
......
......@@ -33,7 +33,7 @@ SBIN_STUFF = resetvlans console_setup.proxy sched_reload named_setup \
sfskey_update sfskey_update.proxy rmuser idleswap \
newnode_reboot savelogs.proxy eventsys.proxy \
elabinelab snmpit.proxy panic repos_daemon node_attributes \
nfstrace
nfstrace plabinelab
CTRLBIN_STUFF = console_setup.proxy sfskey_update.proxy \
savelogs.proxy eventsys.proxy
......
......@@ -90,6 +90,7 @@ my $dolastload = 1;
# Protos
sub SetupReload($$$);
sub FirewallSetup($);
sub os_setup_one($$$);
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
......@@ -183,12 +184,21 @@ my $firewallimageid;
# Ditto ElabinElab.
#
my $elabinelab;
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab)) {
die("*** $0:\n".
" Could not get elabinelab status for experiment $pid/$eid\n");
}
#
# Ditto PlabinElab.
#
my $plabinelab = 0;
my $plcnode;
my $plcimageid;
if (TBExptPlabInElabPLC($pid, $eid, \$plcnode)) {
$plabinelab = 1;
}
#
# Get the set of nodes, as well as the nodes table information for them.
#
......@@ -566,6 +576,44 @@ if ($firewalled) {
delete $nodes{$node};
}
#
# Likewise, setup a PLC node before other plabinelab nodes.
# XXX right now, we setup PLC before ANY other node, whether it is
# part of the inner plab or not.
#
if ($plabinelab) {
my $node = $plcnode;
TBDebugTimeStamp("rebooting/reloading PLC node");
if (!os_setup_one($node, $plcimageid, "PLC")) {
tbwarn "PLC node $node failed to boot".
"This has been reported to testbed-ops.";
SENDMAIL($TBOPS, "1 node is down",
"Node:\n".
" $node\n".
"in pid/eid $pid/$eid failed to boot after loading OS.\n\n".
"The nodes have been freed.\n");
$failed++;
goto tballdone;
}
#
# Check for cancelation. PLC setup may have taken awhile.
#
if (!$canceled) {
TBGetCancelFlag($pid, $eid, \$canceled);
if ($canceled) {
tbnotice "Swap canceled; will terminate os_setup early!";
goto tballdone;
}
}
#
# remove it from the nodelist
#
delete $nodes{$node};
}
#
# We need to issue the reboots and the reloads in parallel.
#
......@@ -703,6 +751,25 @@ if (!$TESTMODE) {
}
TBDebugTimeStamp("rebooting/reloading finished");
#
# XXX declare the inner plab nodes as UP since we won't be hearing from
# them again (they are talking only to their PLC).
#
if ($plabinelab) {
my @plabnodes = ();
TBExptPlabInElabNodes($pid, $eid, \@plabnodes);
foreach my $node (@plabnodes) {
if (exists($nodes{$node})) {
tbnotice "Not waiting for emulated plab node $node";
SetNodeBootStatus($node, NODEBOOTSTATUS_OKAY);
TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_READY());
$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_READY();
TBSetNodeEventState($node, TBDB_NODESTATE_ISUP());
delete($nodes{$node});
}
}
}
#
# Remaining nodes we need to wait for. Why do we wait in the face of errors
# above? So that they enter a reasonably known state before we try to tear
......@@ -1156,6 +1223,10 @@ sub SetupReload($$$)
# XXX firewall is treated special
if ($firewalled && ($node eq $firewall)) {
$firewallimageid = $imageid;
}
# as is a plabinelab PLC node
elsif ($plabinelab && ($node eq $plcnode)) {
$plcimageid = $imageid;
} elsif (!defined($reloads{$imageid})) {
$reloads{$imageid} = [ $node ];
} else {
......@@ -1188,37 +1259,44 @@ sub ForkCmd($) {
}
#
# A firewall has booted up. Might need to do something.
# Setup the firewall node before anything else.
#
sub FirewallBoot()
sub FirewallSetup($)
{
#
# The only case that currently matters is if the experiment is
# elabinelab. In this case we want to turn off the firewall so
# the nodes can boot/reload normally. Later, after we set up the
# inner elab, we turn the firewall back on.
#
return 1
if (!$elabinelab);
my ($node) = @_;
#
# We use the elabinelab program to do this, since it knows what it
# might want to do (and helpfully, is setuid so it can ssh over).
#
system("$elab_setup -f $pid $eid");
return 0
if ($?);
return 1;
if (os_setup_one($node, $firewallimageid, "Firewall")) {
#
# Firewall has booted, perform any final actions.
#
# The only case that currently matters is if the experiment is
# elabinelab. In this case we want to turn off the firewall so
# the nodes can boot/reload normally. Later, after we set up the
# inner elab, we turn the firewall back on.
#
if ($elabinelab) {
#
# We use the elabinelab program to do this, since it knows what it
# might want to do (and helpfully, is setuid so it can ssh over).
#
system("$elab_setup -f $pid $eid");
if ($?) {
tbwarn "Firewall Boot Setup failed!";
return 0;
}
}
return 1;
}
return 0;
}
#
# This is a scaled-down version of what the rest of os_setup does.
# We don't have to do all the asychronous hoohaw here.
# Setup a single node, waiting for completion (reload, reboot)
# before returning.
#
sub FirewallSetup($)
sub os_setup_one($$$)
{
my ($node) = @_;
my ($node,$imageid,$msgstr) = @_;
#
# XXX this is probably not entirely right.
......@@ -1230,7 +1308,7 @@ sub FirewallSetup($)
#
# Reload the node if necessary
#
if (defined($firewallimageid)) {
if (defined($imageid)) {
delete $reboots{$node};
delete $reconfigs{$node};
......@@ -1243,7 +1321,7 @@ sub FirewallSetup($)
$reload_args{'debug'} = $dbg;
$reload_args{'waitmode'} = 1;
$reload_args{'imageid'} = $firewallimageid;
$reload_args{'imageid'} = $imageid;
$reload_args{'nodelist'} = [ @nodelist ];
if (osload(\%reload_args, $reload_failures) != 0) {
......@@ -1254,7 +1332,7 @@ sub FirewallSetup($)
# Gak! waitmode in osload only waits for the reload to complete
# in the frisbee MFS, the node still has to reboot after that.
#
TBDebugTimeStamp("firewall reload done, waiting for reboot");
TBDebugTimeStamp("$msgstr reload done, waiting for reboot");
my $wstart = time;
my $actual_state;
my $waittime = (60 * 7);
......@@ -1266,7 +1344,7 @@ sub FirewallSetup($)
if (!TBNodeStateWait($node, $wstart, $waittime, \$actual_state,
(TBDB_NODESTATE_TBFAILED, TBDB_NODESTATE_ISUP))) {
if ($actual_state eq TBDB_NODESTATE_TBFAILED) {
tbwarn "Firewall $node reported a TBFAILED event";
tbwarn "$msgstr $node reported a TBFAILED event";
return 0;
}
print "$node is alive and well\n";
......@@ -1274,7 +1352,7 @@ sub FirewallSetup($)
TBSetNodeAllocState($node, TBDB_ALLOCSTATE_RES_READY());
$nodeAllocStates{$node} = TBDB_ALLOCSTATE_RES_READY();
} else {
tbwarn "Firewall $node reload timed-out";
tbwarn "$msgstr $node reload timed-out";
return 0;
}
}
......@@ -1326,13 +1404,5 @@ sub FirewallSetup($)
}
}
#
# Firewall has booted, perform any final actions.
#
if (!FirewallBoot()) {
tbwarn "Firewall Boot Setup failed!";
return 0;
}
return 1;
}
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
# Copyright (c) 2004-2006 University of Utah and the Flux Group.
# All rights reserved.
#
use English;
use Getopt::Std;
# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use libtblog;
#
# Do things necessary for setting up inner elab experiment.
#
sub usage()
{
print STDOUT "Usage: plabinelab [-d] [-u] pid eid\n";
print STDOUT " plabinelab [-d] [-k] pid eid\n";
print STDOUT " plabinelab [-d] -r pid eid [node ...]\n";
exit(-1);
}
my $optlist = "dkur";
my $debug = 1;
my $killmode = 0;
my $update = 0;
my $remove = 0;
#
# Configure variables
#
my $TB = "@prefix@";
my $makeconf = "$TB/sbin/dhcpd_makeconf";
# PXE boot path
my $pxeboot = "/tftpboot/pxelinux.0";
# Locals
my $plabinelab;
# Protos
sub SetUp();
sub TearDown();
sub RemoveNodes(@);
sub AddNodes();
sub RestartDHCPD();
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
# Turn off line buffering on output
#
$| = 1;
# Be careful not to exit on transient error
$libdb::DBQUERY_MAXTRIES = 30;
# Locals
my $dbuid;
my $user_name;
my $user_email;
my $query_result;
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"k"})) {
$killmode = 1;
}
if (defined($options{"u"})) {
$update = 1;
}
if (defined($options{"r"})) {
$remove = 1;
}
if (! @ARGV) {
usage();
}
my ($pid,$eid,@nodelist) = @ARGV;
#
# Untaint the arguments.
#
if ($pid =~ /^([-\w]+)$/) {
$pid = $1;
}
else {
die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\w]+)$/) {
$eid = $1;
}
else {
die("Tainted argument $eid!\n");
}
#
# Verify user and get his DB uid.
#
if (! UNIX2DBUID($UID, \$dbuid)) {
die("*** $0:\n".
" You do not exist in the Emulab Database.\n");
}
#
# Get email info for user.
#
if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
die("*** $0:\n".
" Cannot determine your name and email address.\n");
}
TBDebugTimeStampsOn();
TBDebugTimeStamp("PlabInElab setup starting");
#
# Get plabinelab status to make sure, and to see if we need to fire off
# an experiment inside once its setup.
#
if (!TBExptIsPlabInElab($pid, $eid, \$plabinelab)) {
die("*** $0:\n".
" Could not get plabinelab status for experiment $pid/$eid\n");
}
exit(0)
if (!$plabinelab);
#
# Get a list of the nodes involved. Not all nodes in the experiment
# have to be part of the plab (plab_role == 'none') and one node is the
# distinguished PLC node (plab_role == 'plc').
#
my $plcnode;
my @plabnodes = ();
$query_result =
DBQueryFatal("select node_id,plab_role from reserved ".
"where pid='$pid' and eid='$eid' and plab_role!='none'");
while (my ($node,$role) = $query_result->fetchrow_array()) {
if ($role eq "plc") {
$plcnode = $node;
} else {
push(@plabnodes, $node);
}
}
my $rv;
if ($killmode) {
$rv = TearDown();
} elsif ($remove) {
$rv = RemoveNodes(@nodelist);
} elsif ($update) {
$rv = AddNodes();
} else {
$rv = SetUp();
}
TBDebugTimeStamp("PlabInElab setup done");
exit($rv);
sub SetUp()
{
if ($plcnode || @plabnodes > 0) {
#
# Currently we jigger the pxe_boot_path field for inner plab nodes
# so that they run pxelinux when rebooted. pxelinux is the easiest
# way of downloading a "Linux MFS". Maybe someday we'll do it with
# pxeboot.
#
if (@plabnodes > 0) {
DBQueryFatal("update nodes set pxe_boot_path='$pxeboot' ".
"where (".
join(" or ", map("node_id='$_'", @plabnodes)) . ")");
}
#
# Now mark all plab involved nodes as being ready to boot inside,
# so that dhcpd_makeconf knows what nodes to change the entries for.
#
push(@plabnodes, $plcnode);
DBQueryFatal("update reserved set plab_boot=1 ".
"where pid='$pid' and eid='$eid' and (".
join(" or ", map("node_id='$_'", @plabnodes)) . ")");
#
# Now that we are done fixing up the DB with info that affects the
# DHCPD config file, we can restart DHCPD.
#
RestartDHCPD();
}
}
#
# Take down all inner plab nodes.
#
# In theory, nfree will clean up nodes.pxe_boot_path and
# the reserved table as well as recreating the DHCPD config file.
# However, a lot can happen between now and nfree (e.g., forcing a
# node into the MFS which would fail due to pxe_boot_path) so we do
# it ourselves to be safe.
#
sub TearDown()
{
#
# Reset the PXE boot
#
if ($plcnode || @plabnodes > 0) {
if (@plabnodes > 0) {
DBQueryFatal("update nodes set pxe_boot_path=NULL ".
"where (".
join(" or ", map("node_id='$_'", @plabnodes)) . ")");
}
#
# Update the reserved table to mark nodes as no longer part
# of the plab.
#
DBQueryFatal("update reserved set plab_role='none',plab_boot=0 ".
"where pid='$pid' and eid='$eid'");
#
# and restart DHCPD
#
RestartDHCPD();
}
return 0;
}
#
# Remove some nodes from an inner plab (on tbswap UPDATE).
# Tricky because the user could remove the PLC node as well.
#
sub RemoveNodes(@)
{
my @newnodes = @_;
my @nodes = ();
#
# Grab the list of nodes. We want to clear the reserved table bits so
# that we can regen the DHCPD file.
#
foreach my $node (@newnodes) {
# Untaint the nodes.
if ($node =~ /^([-\w]+)$/) {
$node = $1;
}
else {
die("*** Tainted node name: $node\n");
}
if (($node eq $plcnode) && @plabnodes > 0) {
tbwarn "Removing PLC node, expect problems!";
}
push(@nodes, $node);
}
return 0
if (!@nodes);
#
# Reset pxe_boot_path
#
DBQueryFatal("update nodes set pxe_boot_path=NULL ".
"where (".
join(" or ", map("node_id='$_'", @nodes)) . ")");
#
# Update the reserved table
#
DBQueryFatal("update reserved set plab_role='none',plab_boot=0 ".
"where pid='$pid' and eid='$eid' and (".
join(" or ", map("node_id='$_'", @nodes)) . ")");
#
# and restart DHCPD
#
RestartDHCPD();
return 0;
}
#
# Add some nodes as part of a tbswap UPDATE
# No different than a first-time setup right now.
#
sub AddNodes()
{
return SetUp();
}
sub RestartDHCPD()
{
print "Regenerating DHCPD config file and restarting daemon.\n";
system("$makeconf -i -r");
if ($?) {
die("*** $0:\n".
" Failed to reconfig/restart DHCPD.\n");
}
}
......@@ -100,6 +100,7 @@ my $updateReboot = 0;
my $updateReconfig = 1;
my $update_Eventsys_restart = 0;
my $elabinelab = 0;
my $plabinelab = 0;
my $force = 0;
my $errors = 0;
my $updatehosed = 0;
......@@ -215,6 +216,17 @@ if (!$force) {
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab)) {
tbdie("Could not get elabinelab status for experiment $pid/$eid");
}
# and plabinelab status.
if (! TBExptIsPlabInElab($pid, $eid, \$plabinelab)) {
tbdie("Could not get plabinelab status for experiment $pid/$eid");
}
if ($elabinelab && $plabinelab) {
tberror "Cannot get my head around Plab in Elab in Elab!\n";
print "Failingly finished swap-$swapop for $pid/$eid. " .
TBTimeStamp() . "\n";
TBDebugTimeStamp("tbswap $swapop finished (failed)");
exit(1);
}
#
# See if the experiment is firewalled
......@@ -474,6 +486,19 @@ sub doSwapout($) {
}
if ($type >= CLEANUP) {
#
# Undo plab in elab specialness.
# No need to worry about VLANs here, as all the special abilities
# involve the control network.
#
if (! $TESTMODE && $plabinelab) {
print "Tearing down plabinelab.\n";
if (system("plabinelab -k $pid $eid")) {
tberror "Failed to teardown plabinelab!";
$swapout_errors = 1;
}
}
#
# We're not attempting a retry;
#
......@@ -580,6 +605,17 @@ sub doSwapout($) {
TBDebugTimeStamp("vnode_setup -k finished");
}
#
# Undo plabinelab setup
#
if (!$TESTMODE && $plabinelab && @failedpnodes > 0) {
print "Removing failed nodes from inner plab.\n";
if (system("plabinelab -r $pid $eid @failedpnodes")) {
tberror "Failed to remove inner nodes!";
$swapout_errors = 1;
}
}
#
# Release all failed nodes.
#
......@@ -832,6 +868,12 @@ sub doSwapin($) {
tberror "Failed to remove inner nodes!";
return 1;
}
} elsif ($plabinelab) {
print "Removing nodes from inner plab.\n";
if (system("plabinelab -r $pid $eid @physnodes")) {
tberror "Failed to remove inner nodes!";
return 1;
}
}
#
......@@ -962,6 +1004,24 @@ sub doSwapin($) {
return 1;
}
#
# PlabinElab setup. This is currently just tweaking out the dhcpd.conf
# file and that must be done before os_setup (i.e., before nodes are
# rebooted).
#
if ($plabinelab && !$TESTMODE && $type > UPDATE_RECOVER) {
# for UPDATE and RETRY we pass in the -u to pick up new nodes
my $optarg = ($type == REAL ? "" : "-u");
print "Setting up plabinelab.\n";
TBDebugTimeStamp("plabinelab setup started");
if (system("plabinelab $optarg $pid $eid")) {
tberror "Failed to setup plabinelab!";
return 1;
}
TBDebugTimeStamp("plabinelab setup finished");
}
#
# If user specified -reboot to update,
# and we are successfully performing the update,
......@@ -1010,7 +1070,7 @@ sub doSwapin($) {
TBDebugTimeStamp("gentopofile finished");
# XXX fer now hack
if (0 && !$firewalled && !$elabinelab &&
if (0 && !$firewalled && !$elabinelab && !$plabinelab &&
($pid eq "testbed" || $pid eq "tbres")) {
DBQueryWarn("update experiments set ".
" savedisk=1 where pid='$pid' and eid='$eid'");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!