Commit 7cfa7aa6 authored by David Johnson's avatar David Johnson
Browse files

libosload rewrite and HP Procurve switch support.

libosload_new contains the new libosload.  It provides the same
API as the original libosload, but the functionality has been
split out into well-defined methods that can be overridden by
per-node type subclasses in small chunks.  This is all protected
by the NewOsload feature.

libosload_switch.pm.in and libossetup_switch.pm.in contain the HP
switch support, which requires the new libosload and new
libossetup.  libossetup_switch doesn't do anything too radical, but
it does support running a node it is "lighting up" through multiple
operations -- i.e., first a RELOAD, then a RECONFIG.  This was
necessary because switches require a config to be pushed to them
(well, they do in the way I elected to support them), and they
cannot fetch configuration in the normal Emulab way via tmcd.

Eventually, that multi-op support will be factored out into
libossetup more generically so that other node types which require
push-based configuration can leverage it.

libosload_switch contains the HP switch loading code.  It's built
around expect-like interaction on the serial console.  It supports
xmodem and tftp uploads, but tries to only flash the switch lazily.
Thus, if the switch is already loaded with an image and a user
(or the reload daemon, or anybody) reloads it with the same image,
it won't actually get reloaded.  You can now force this reload
with os_load.

libosload_switch also contains reconfig code (i.e., to wipe
switch configs and recreate based on which experiment it is in).
libossetup_switch directly invokes this instead of going through
libosload, since configuring is not really part of loading.  It
is just convenient since configuration also happens over the
serial console, and all the osload code for the switch can be
reused.

Also, of course, there are miscellaneous changes to support the
new libosload in os_load itself.
parent 649a30ac
......@@ -89,7 +89,8 @@ LIB_STUFF = libtbsetup.pm exitonwarn.pm libtestbed.pm snmpit_intel.pm \
libadminmfs.pm libtblog.pm libtblog_simple.pm libArchive.pm \
power_mail.pm power_whol.pm Template.pm power_rmcp.pm \
power_ilo.pm libvtop.pm libptop.pm libossetup.pm \
power_ipmi.pm
power_ipmi.pm libosload_new.pm libosload_switch.pm \
libossetup_switch.pm
# These scripts installed setuid, with sudo.
SETUID_BIN_SCRIPTS = node_reboot eventsys_control tarfiles_setup savelogs \
......
This diff is collapsed.
This diff is collapsed.
......@@ -28,6 +28,7 @@ use NodeType;
use libtblog;
use English;
use Data::Dumper;
use EmulabFeatures;
# Configure variables
my $TB = "@prefix@";
......@@ -73,6 +74,7 @@ sub New($$$@)
my $self = {};
$self->{'USER'} = $user;
$self->{'EXPT'} = $experiment;
$self->{'GROUP'} = (defined($experiment)) ? $experiment->GetGroup() : undef;
$self->{'HASH'} = {};
$self->{'NODES'} = {};
$self->{'PHYSNODES'} = {};
......@@ -99,6 +101,15 @@ sub New($$$@)
# XXX
$self->noretry(0);
$self->canceled(0);
if (EmulabFeatures->FeatureEnabled("NewOsload",$user,
$self->{'GROUP'},$experiment)) {
print STDERR "libossetup New NewOsload enabled\n";
require libosload_new;
# XXX -- @nodelist
$self->loadobj(libosload_new->New());
}
return $self;
}
# Break circular reference someplace to avoid exit errors.
......@@ -118,6 +129,7 @@ sub sharednodes($) { return $_[0]->{'SHAREDNODES'}; }
sub oplist($) { return $_[0]->{'OPLIST'}; }
sub user($) { return $_[0]->{'USER'}; }
sub experiment($) { return $_[0]->{'EXPT'}; }
sub group($) { return $_[0]->{'GROUP'}; }
sub IncrFailCount($) { $_[0]->{'FAILCOUNT'}++ }
sub failed($) { return $_[0]->{'FAILCOUNT'}; }
......@@ -529,8 +541,23 @@ sub LightUpNodes($@)
$reload_args{'nodelist'} = [ @list ];
$reload_args{'nodeflags'} = \%nodeflags;
my $pid = osload(\%reload_args, $reload_failures);
push(@children, [ $pid, \&osload_wait,
my $pid;
my $coderef;
if (EmulabFeatures->FeatureEnabled("NewOsload",$self->user(),
$self->group(),
$self->experiment())) {
($self->loadobj())->debug($self->debug());
$pid = ($self->loadobj())->osload(\%reload_args, $reload_failures);
$coderef = sub {
my $childpid = shift;
return ($self->loadobj())->osload_wait($childpid);
};
}
else {
$pid = osload(\%reload_args, $reload_failures);
$coderef = \&osload_wait;
}
push(@children, [ $pid, $coderef, ,
[ @list ], $reload_failures ]);
sleep(5);
}
......@@ -792,6 +819,10 @@ sub NewType($$)
if ($type eq "pcfedphys" || $type eq "pcfed") {
$type = "protogeni";
}
# elsif ($type eq 'hp5406') {
# print STDERR "changing type from $type to switch\n";
# $type = "switch";
# }
elsif (defined($typeinfo) && $typeinfo->issubnode()) {
$type = "subnode";
}
......@@ -800,12 +831,17 @@ sub NewType($$)
my $newtype = eval { $packname->New($self); };
# Not loaded?
if ($@) {
#print STDERR $@;
return undef;
print STDERR "module load failed: " . $@ . "\n";
eval "require libossetup_$type";
$newtype = eval { $packname->New($self); };
if ($@) {
print STDERR "module load failed: " . $@ . "\n";
return undef;
}
}
$self->{'TYPECACHE'}->{$type} = $newtype;
print STDERR "Created type object $type\n"
print STDERR "Created type object $type with parent $self\n"
if ($self->debug());
return $newtype;
......@@ -854,6 +890,8 @@ use English;
use Data::Dumper;
use overload ('""' => 'Stringify');
use vars qw($AUTOLOAD);
sub New($$$)
{
my ($class, $type, $parent) = @_;
......@@ -862,6 +900,7 @@ sub New($$$)
$self->{'TYPE'} = $type;
$self->{'NODES'} = {};
$self->{'PARENT'} = $parent;
print STDERR "just set parent in $self to $parent\n";
$self->{'FAILCOUNT'} = 0;
$self->{'TODOLIST'} = {};
......@@ -877,6 +916,23 @@ sub todo($) { return $_[0]->{'TODOLIST'}; }
sub todolist($) { return values(%{ $_[0]->{'TODOLIST'} }); }
sub IncrFailCount($) { $_[0]->{'FAILCOUNT'}++ }
sub AUTOLOAD {
my $self = shift;
my $type = ref($self) or die("$self is not an object\n");
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
if (@_) {
return $self->{'HASH'}->{$name} = shift;
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
print STDERR "$self: tried to access unknown slot $name\n";
return undef;
}
#
# Add a node to the list.
#
......@@ -894,6 +950,7 @@ sub AddNode($$)
# This sets the OS that should boot, as well as any reloads, reboots
# and reconfigs that are needed.
#
print STDERR "trying get parent in $self\n";
$self->parent()->SetOS($node);
return 0;
}
......@@ -1262,7 +1319,15 @@ sub AddNode($$)
$reload_args{'imageid'} = $image->imageid();
$reload_args{'nodelist'} = [ $node_id ];
osload(\%reload_args, $reload_failures);
if (EmulabFeatures->FeatureEnabled("NewOsload",$self->user(),
$self->group(),
$self->experiment())) {
($parent->loadobj())->debug($self->debug());
($parent->loadobj())->osload(\%reload_args, $reload_failures);
}
else {
osload(\%reload_args, $reload_failures)
}
# Reset this. Updated in Volunteers() below.
$node->_setupoperation($libossetup::NOSTATE);
......
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2010 University of Utah and the Flux Group.
# All rights reserved.
#
package libossetup_switch;
use strict;
use lib '@prefix@/lib';
use libossetup;
use base qw(libossetup_handler);
use vars qw($AUTOLOAD);
use libdb;
use libtestbed;
use libossetup;
# XXX only works with newer version of libosload!
use libosload_new;
use libosload_switch;
use libtblog;
use Node;
use English;
use Data::Dumper;
use overload ('""' => 'Stringify');
#
# A constructor for an object to handle all nodes of this type.
#
sub New($$) {
my ($class, $parent) = @_;
my $self = $class->SUPER::New("switch", $parent);
$self->{'OPLIST'} = {};
bless($self, $class);
my $switchobj = libosload_switch->New();
$self->switchobj($switchobj);
$switchobj->debug(($self->parent())->debug());
print STDERR "$self: new object \n";# . Dumper($self) . "\n";
return $self;
}
sub AUTOLOAD {
my $self = shift;
my $type = ref($self) or die("$self is not an object\n");
my $name = $AUTOLOAD;
$name =~ s/.*://; # strip fully-qualified portion
if (@_) {
return $self->{'HASH'}->{$name} = shift;
}
elsif (exists($self->{'HASH'}->{$name})) {
return $self->{'HASH'}->{$name};
}
print STDERR "libossetup_switch: tried to access unknown slot $name\n";
return undef;
}
sub AddNode($$)
{
my ($self, $node) = @_;
my $node_id = $node->node_id();
$self->SUPER::AddNode($node);
print STDERR "$self AddNode($node_id): so=".$node->_setupoperation()." and as=" . $node->allocstate() . "\n";
#
# Setup multi-operation list so that when WaitDone
# gets called on a node, we don't free the node, but
# rather do the next operation!
#
if ($node->_setupoperation() == $libossetup::RELOAD) {
$self->{OPLIST}->{$node_id} = [ $libossetup::RELOAD ];
if ($node->allocstate() eq TBDB_ALLOCSTATE_RES_INIT_DIRTY()) {
# need to do a reconfig after the reload
push @{$self->{OPLIST}->{$node_id}}, $libossetup::RECONFIG;
print STDERR "$self AddNode($node_id): so=".$node->_setupoperation()." and as=" . $node->allocstate() . "\n";
}
}
else {
if ($node->allocstate() eq TBDB_ALLOCSTATE_RES_RECONFIG()
|| $node->allocstate() eq TBDB_ALLOCSTATE_RES_INIT_DIRTY()) {
$self->{OPLIST}->{$node_id} = [ $libossetup::RECONFIG ];
$node->_setupoperation($libossetup::RECONFIG);
#
# XXX hack -- our node is already ISUP, and we need to force it
# out of that state so taht WaitForNodes doesn't beat our type
# handler object to forcing it to SHUTDOWN
#
print STDERR "$self AddNode($node_id): forcing to SHUTDOWN before RECONFIG\n";
TBSetNodeEventState($node_id,TBDB_NODESTATE_SHUTDOWN);
}
elsif ($node->allocstate() ne TBDB_ALLOCSTATE_RES_READY()) {
# only reboot node if assign_wrapper just pulled it into expt.
# (e.g. it isnt ALLOCSTATE_RES_READY)
$self->{OPLIST}->{$node_id} = [ $libossetup::REBOOT ];
$node->_setupoperation($libossetup::REBOOT);
}
}
print STDERR "$self AddNode OPLIST: " . Dumper($self->{OPLIST});
# don't ever retry switches
$node->_retrycount(0);
#
# XXX should ask libosload_new
# These guys take a long time
#
$node->_maxwait(15*60);
$node->_setupstatus($libossetup::SETUP_OKAY);
return 0;
}
sub LightUpNodes($@)
{
#
# This needs to trigger reloads and reconfigs as appropriate.
# If a reload is going to be followed by a reconfig, then we need to
# tell osload that, AND make the reconfig happen via the multi-op thing.
#
my ($self, @nodelist) = @_;
my @children = ();
my @failed = ();
my $parent = $self->parent();
#
# Set up lists of what we are going to do.
#
my %reloads = ();
my %reboots = ();
my %reconfigs = ();
my %rebooted = ();
foreach my $node (@nodelist) {
my $node_id = $node->node_id();
my $op = $node->_setupoperation();
my $action;
# Nothing to do.
next
if ($op == $NOSTATE);
if ($op == $RELOAD) {
my $image = $node->_loadimage();
if (!exists($reloads{$image->imageid()})) {
$reloads{$image->imageid()} = [];
}
push(@{ $reloads{$image->imageid()} }, $node);
$action = "reloaded with $image";
}
elsif ($op == $REBOOT) {
$reboots{$node_id} = $node;
$action = "rebooted";
}
elsif ($op == $RECONFIG) {
$reconfigs{$node_id} = $node;
$action = "reconfiged";
}
print STDERR "$node_id will be $action\n";
}
# XXX Caller wants a list.
return 0
if ($parent->impotent());
#
# Now fire them off.
#
foreach my $imageid ( keys(%reloads) ) {
my @nlist = @{ $reloads{$imageid} };
my @list = ();
my %nodeflags = ();
foreach my $node (@nlist) {
my $node_id = $node->node_id();
# The osload library gets ids.
push(@list, $node_id);
$node->SetAllocState(TBDB_ALLOCSTATE_RES_RELOAD());
# No point in reboot/reconfig obviously, since node will reboot!
delete $reboots{$node_id};
delete $reconfigs{$node_id};
$rebooted{$node_id} = 1;
}
my %reload_args = ();
my $reload_failures = {};
$reload_args{'debug'} = $parent->debug();
$reload_args{'asyncmode'} = 1;
$reload_args{'imageid'} = $imageid;
$reload_args{'nodelist'} = [ @list ];
$reload_args{'nodeflags'} = \%nodeflags;
my $pid;
my $coderef;
if (EmulabFeatures->FeatureEnabled("NewOsload",$parent->user(),
$parent->group(),
$parent->experiment())) {
($parent->loadobj())->debug($parent->debug());
$pid = ($parent->loadobj())->osload(\%reload_args,$reload_failures);
print STDERR "$self: kicking off wait for osload pid $pid\n";
$coderef = sub {
my $childpid = shift;
return ($parent->loadobj())->osload_wait($childpid);
};
}
else {
$pid = osload(\%reload_args, $reload_failures);
$coderef = \&osload_wait;
}
push(@children, [ $pid, $coderef, ,
[ @list ], $reload_failures ]);
sleep(5);
}
#
# Fire off the reboots.
#
if (keys(%reboots)) {
foreach my $node_id (keys(%reboots)) {
my $node = $self->node($node_id);
if (defined($node->allocstate()) &&
$node->allocstate() eq TBDB_ALLOCSTATE_RES_INIT_CLEAN()) {
$node->SetAllocState(TBDB_ALLOCSTATE_RES_REBOOT_CLEAN());
}
else {
$node->SetAllocState(TBDB_ALLOCSTATE_RES_REBOOT_DIRTY());
}
# Needed for vnode_setup.
$rebooted{$node_id} = 1;
}
my @list = keys(%reboots);
my %reboot_args = ();
my $reboot_failures = {};
$reboot_args{'debug'} = $parent->debug();
$reboot_args{'waitmode'} = 0;
$reboot_args{'asyncmode'} = 1;
$reboot_args{'nodelist'} = [ @list ];
my $pid = nodereboot(\%reboot_args, $reboot_failures);
push(@children, [ $pid, \&nodereboot_wait,
[ @list ], $reboot_failures ]);
sleep(2);
}
#
# Fire off the reconfigs.
#
if (keys(%reconfigs)) {
my @list = keys(%reconfigs);
my $reconfig_failures = {};
my $switchobj = $self->switchobj();
foreach my $node_id (@list) {
my $node = Node->Lookup($node_id);
$switchobj->AddNode($node,undef,undef);
$switchobj->SetupReconfigure($node);
my $pid = $switchobj->Reconfigure($node,0);
push @children, [ $pid, sub {
my ($waitstatus,$retval,@output);
while (1) {
$switchobj->WaitForNode($node);
print STDERR "$switchobj->WaitForNode($node_id) -> $waitstatus";
if ($waitstatus > 0) {
return $retval;
}
elsif ($waitstatus == 0) {
# we need to keep waiting for this node, so don't check
# the db
;
}
sleep(2);
}
},
[ $node_id ], $reconfig_failures ];
}
}
#
# Wait for all of the children to exit. We look at the $pid to know if
# command failed/ended immediately; otherwise we need to wait on it.
# For any failures, record the node failures for later so that we do
# not wait for them needlessly.
#
while (@children) {
my ($pid, $waitfunc, $listref, $hashref) = @{ pop(@children) };
# This is not likely to happen.
next
if ($pid == 0);
if ($pid > 0) {
next
if (! &$waitfunc($pid));
}
#
# Failure. Record the failures for later. If the $pid<0 then the
# entire list failed. Otherwise, have to scan the return hash to
# find the failures.
#
my @nlist = ();
if ($pid < 0) {
@nlist = @{ $listref };
}
else {
foreach my $node_id (keys(%{ $hashref })) {
push(@nlist, $node_id)
if ($hashref->{$node_id});
}
}
#
# Mark all of the failed nodes so that the caller knows there
# was a failure.
#
foreach my $node_id (@nlist) {
my $node = $self->node($node_id);
$node->SetAllocState(TBDB_ALLOCSTATE_DOWN());
}
}
return scalar(@failed);
}
#
# Wait function signals some local cluster nodes are done waiting.
#
sub WaitDone($@)
{
my ($self, @nodelist) = @_;
my $parent = $self->parent();
my $experiment = $parent->experiment();
my $pid = $experiment->pid();
my $eid = $experiment->eid();
#
# Then per node processing.
#
foreach my $node (@nodelist) {
my $node_id = $node->node_id();
my $setupstatus = $node->_setupstatus();
my $eventstate = $node->eventstate();
print STDERR "$self: WaitDone: $node_id,$setupstatus,$eventstate\n";
if ($eventstate eq TBDB_NODESTATE_ISUP()) {
#
# If there is another op in our list, keep going with that op --
# i.e., keep the node in our todo list!
#
my $op = shift @{$self->{OPLIST}->{$node_id}};
if (@{$self->{OPLIST}->{$node_id}}) {
my $opname = "RELOAD";
if ($op == $libossetup::RECONFIG) {
$opname = "RECONFIG";
}
elsif ($op == $libossetup::REBOOT) {
$opname = "REBOOT";
}
print STDERR "$self OPLIST: " . Dumper($self->{OPLIST});
print STDERR "$self WaitDone: $node_id finished $opname; doing next op!\n";
$node->_setupoperation($self->{OPLIST}->{$node_id}->[0]);
next;
}
#
# Otherwise, handle the node normally!
#
#
# Must call the generic WaitDone handler too.
#
$self->SUPER::WaitDone(@nodelist);
print "$node_id is alive and well\n";
$node->SetBootStatus(NODEBOOTSTATUS_OKAY);
$node->SetAllocState(TBDB_ALLOCSTATE_RES_READY());
# Set this so we know a successful reboot was done.
# Important for VMs that depend on this node.
$node->_rebooted(1)
if ($node->_setupoperation() != $libossetup::NOSTATE);
next;
}
#
# If we had an error, don't continue! SUPER method handles that.
#
# Must call the generic WaitDone handler too.
#
$self->SUPER::WaitDone(@nodelist);
# Fall through on failure.
if ($eventstate eq TBDB_NODESTATE_TBFAILED()) {
tbwarn("$node_id reported a TBFAILED event\n");
}
else {
tbwarn("$node_id failed to boot\n");
}
$node->SetBootStatus(NODEBOOTSTATUS_FAILED);
#
# Reload failures are terminal.
#
if ($node->_canfail() && $setupstatus != $libossetup::RELOAD_FAILED &&
!($experiment->canceled() || $parent->noretry())) {
$parent->add_failed_node_inform_user($node_id);
$parent->add_failed_node_nonfatal($node_id);
tbnotice("Continuing with experiment setup anyway ...\n");
next;
}
#
# XXX don't do this for now!
#
# If the user has picked a standard image and it fails to boot,
# something is wrong, so reserve it to checkup experiment. If the
# image belongs to the user, then we assume its the image at fault,
# and allow it to be returned to the pool (caller, tbswap will end
# doing the nfree on nodes with a DOWN allocstate).
#
my $osinfo = $node->_bootosinfo();
if (0 && (! defined($osinfo->pid()) || $osinfo->pid() eq TBOPSPID())) {
$node->MarkAsIll();
$node->InsertNodeLogEntry($parent->user(),
TB_DEFAULT_NODELOGTYPE(),
"'Moved to hwcheckup by os_setup; ".
"failed to boot image for $osinfo " .
"in $pid/$eid'");
$parent->add_failed_node_inform_tbopsfatal($node_id);
}
else {
$parent->add_failed_node_inform_tbopswarn($node_id);