Commit a495bcfb authored by Chad Barb's avatar Chad Barb
Browse files

New tbswap mode 'update'. (a.k.a. 'reswap')
Re-assigns experiment, fixing already assigned nodes in place;
tries not to reboot nodes. Doesn't clear port counters,
restart event system, etc.

A few more things remain to be considered for 'general' use
adding new nodes to experiments and modifying topologies,
but for replacing failed nodes in experiments
or removing virt_nodes from experiments, should work fine.
parent 3556ab99
......@@ -6,7 +6,6 @@
# All rights reserved.
#
# This function as the main assign loop. It converts the virtual
# topology into a top input including LAN and delay translation. It
# then snapshots the current testbed physical state and runs assign,
......@@ -270,8 +269,9 @@ if ($updating) {
"WHERE pid='$pid' AND eid='$eid'");
while (($vname,$reserved) = $result->fetchrow_array) {
$fixed_nodes{$vname} = $reserved;
$alreadyAllocated{$reserved} = 1;
$reserved_nodes{$vname} = $reserved;
$fixed_nodes{$vname} = $reserved;
$alreadyAllocated{$reserved} = "unused";
}
$result->finish;
}
......@@ -292,10 +292,6 @@ while (($vname,$ips,$type,$fixed,$isremote,$isvirt) =
if (defined($fixed) && $fixed eq "") {
undef($fixed);
}
if (defined($reserved) && $reserved eq "") {
undef($reserved);
}
# REMOTENODE HACK
#
......@@ -802,17 +798,23 @@ if( scalar(@simnodelist) > 0 ) {
}
# Print out fixed nodes
# But _not_ nodes which aren't in the experiment any more.
# CRB you can only fix nodes and delaynodes, right?
$reused_count = 0;
foreach $fixed (keys(%fixed_nodes)) {
if (!$isremotenode{$fixed}) {
if (!$isremotenode{$fixed} &&
(exists $nodes{$fixed} || exists $delaynodes{$fixed}) ) {
print TOPFILE "fix-node $fixed $fixed_nodes{$fixed}\n";
if ($reserved_nodes{$fixed}) { $reused_count++; }
}
}
close TOPFILE;
# Set estimations
$minimum_nodes = $nodes + keys(%delaynodes)/$DELAYCAPACITY;
$maximum_nodes = $nodes + keys(%delaynodes);
$minimum_nodes = $nodes + keys(%delaynodes)/$DELAYCAPACITY - $reused_count;
$maximum_nodes = $nodes + keys(%delaynodes) - $reused_count;
DBQueryFatal("UPDATE experiments set maximum_nodes=$maximum_nodes, " .
"minimum_nodes=$minimum_nodes where pid=\"$pid\" and eid=\"$eid\"");
print "Minimum nodes = $minimum_nodes\n";
......@@ -930,7 +932,24 @@ while (1) {
next;
}
if (!$alreadyAllocated{$physical}) {
if ($alreadyAllocated{$physical}) {
#
# Mark node as being reused.
#
# Look at virtual node being mapped to node;
# if it wasn't in the previous map, mark node for reboot.
#
if (! exists $reserved_nodes{$virtual} ||
$reserved_nodes{$virtual} ne $physical ||
$alreadyAllocated{$physical} eq "reboot") {
$alreadyAllocated{$physical} = "reboot";
} else {
$alreadyAllocated{$physical} = "reused";
}
} else {
#
# This is a new node; we'll have to reserve it.
#
$toreserve{$physical} = 1;
}
......@@ -1026,6 +1045,29 @@ while (1) {
}
TBDebugTimeStamp("reserving finished");
my %tolose = ();
foreach $node (keys(%alreadyAllocated)) {
if ($alreadyAllocated{$node} eq "unused") {
#
# Node was used in previous incarnation, but not any more.
#
$tolose{$node} = 1;
} elsif ($alreadyAllocated{$node} eq "reboot") {
#
# Node is being reused, but for a different purpose, so
# it should be rebooted.
#
TBSetNodeAllocState( $node, TBDB_ALLOCSTATE_RES_INIT_DIRTY() );
}
}
if ((keys %tolose) > 0) {
if (system("nfree $pid $eid " . join(" ", keys(%tolose)))) {
print "Failed to free no-longer-needed nodes!";
}
}
last;
}
}
......
......@@ -8,11 +8,15 @@
use English;
# tbswap
# XXX: handle error cases for update? (backup the db?)
# XXX: Shouldn't do idempotent stuff twice for update.
# XXX: repush/calc routing for update??? (tbprerun)
# XXX: previz for update??? (tbprerun)
# XXX: make snmpit faster for update.
sub usage()
{
print STDERR "Usage: $0 { in | out } [-force] pid eid\n";
print STDERR "Usage: $0 { in | out [-force] | update [-restart] } pid eid\n";
exit(-1);
}
......@@ -38,8 +42,13 @@ require exitonwarn; # exitonwarn isn't really a module, so just require it
#
# Actual swap-in and swap-out functions, defined below.
#
sub doSwapout;
sub doSwapin;
sub doSwapout($);
sub doSwapin($);
sub REAL() { return 4; }
sub CLEANUP() { return 3; }
sub RETRY() { return 2; }
sub UPDATE() { return 1; }
#
# Turn off line buffering on output
......@@ -47,6 +56,7 @@ sub doSwapin;
$| = 1;
my $restart = 0;
my $force = 0;
my $errors = 0;
my $retry = 0;
......@@ -56,13 +66,16 @@ my $os_setup_pid;
my $cleanvlans;
#
# First argument is either "in" or "out";
# First argument is either "in", "out", or "update";
# this value goes into $swapop.
#
my $swapop = shift;
if (!$swapop || (($swapop ne "in") && ($swapop ne "out"))) {
if (!$swapop ||
(($swapop ne "in") &&
($swapop ne "out") &&
($swapop ne "update"))) {
usage();
}
......@@ -74,6 +87,8 @@ while ($#ARGV > 1) {
$arg = shift;
if ($arg eq "-force") {
$force = 1;
} elsif ($arg eq "-restart") {
$restart = 1;
} else {
usage();
}
......@@ -116,7 +131,7 @@ if ($swapop eq "in") {
$nextState = EXPTSTATE_ACTIVATING;
$suggestion = "Must be swapped out.";
$suggestUseOfForce = 0;
} else {
} elsif ($swapop eq "out") {
if (! $TESTMODE) {
$desiredState = EXPTSTATE_ACTIVE;
$suggestion = "Must be running.";
......@@ -126,6 +141,11 @@ if ($swapop eq "in") {
}
$nextState = EXPTSTATE_SWAPPING;
$suggestUseOfForce = 1;
} elsif ($swapop eq "update") {
$desiredState = EXPTSTATE_ACTIVE;
$nextState = EXPTSTATE_ACTIVATING; # XXX add an updating state?
$suggestion = "Must be running.";
$suggestUseOfForce = 0;
}
#
......@@ -171,18 +191,56 @@ if (! $errors) {
undef $nextState;
if (! $errors) {
if ($swapop eq "in") {
if ($swapop eq "out") {
#
# Swap in
# Swap out
#
$errors = doSwapout(REAL);
if (! $errors) {
$nextState = EXPTSTATE_SWAPPED;
} else {
# leave $nextState undefined
# experiment will stay in SWAPPING.
}
#
# $retry flag gets set to 1 by doSwapin() if
# another attempt is appropriate.
#
$retry = 0;
# Update Accounting Information
#
TBSetExpSwapTime($pid, $eid);
}
}
$errors = doSwapin();
if (! $errors) {
if ($swapop eq "update") {
$errors = doSwapout(UPDATE);
if (! $errors) {
# leave $nextState undefined.
# (this will get defined below.)
} else {
# Error; Clean up and put experiment in SWAPPED.
print STDERR "Cleaning up after errors.\n";
doSwapout(CLEANUP);
$nextState = EXPTSTATE_SWAPPED;
}
#
# Update Accounting Information
#
TBSetExpSwapTime($pid, $eid);
}
}
if (! $errors) {
if ($swapop eq "in" || $swapop eq "update") {
#
# Swap in
#
if ($swapop eq "in") {
$errors = doSwapin(REAL);
} else {
$errors = doSwapin(UPDATE);
}
my $retries = 2;
......@@ -191,24 +249,19 @@ if (! $errors) {
# a) there were errors,
# b) doswapin() indicated (via $retry) a retry is appropriate,
# c) we haven't tried too many times already.
# d) (XXX temporary) We're not doing an update.
#
# NOTE that if $TESTMODE == 1, retries are impossible, since
# $retry will never get set to 1.
#
while ($errors && $retry && $retries) {
while ($errors && $retry && $retries && $swapop eq "in") {
$retries--;
print STDERR "Cleaning up after errors; will try again.\n";
#
# Leave $retry == 1 for doSwapout(), so it only
# deallocates failed nodes.
#
doSwapout();
# XXX should errors during swapout affect retry mechanism?
doSwapout(RETRY);
$retry = 0;
print STDERR "Trying again...\n";
$errors = doSwapin();
$errors = doSwapin(RETRY);
}
if (! $errors) {
......@@ -229,13 +282,7 @@ if (! $errors) {
} else {
print STDERR "Cleaning up after errors.\n";
#
# Set retry to 0; even though swapin indicated we _could_ retry,
# we want swapout to do a full cleanup.
#
$retry = 0;
doSwapout();
doSwapout(CLEANUP);
#
# Regardless of how well cleanup swapout worked,
......@@ -243,28 +290,6 @@ if (! $errors) {
#
$nextState = EXPTSTATE_SWAPPED;
}
} else {
#
# Swap out
#
$errors = doSwapout();
#
# Update Accounting Information
#
TBSetExpSwapTime($pid, $eid);
if (! $errors) {
#
# Swapout worked; send exp to SWAPPED.
#
$nextState = EXPTSTATE_SWAPPED;
} else {
#
# Swapout didn't fully work; leave exp in SWAPPING.
#
undef $nextState;
}
}
}
......@@ -309,7 +334,8 @@ exit($errors);
#
##
sub doSwapout {
sub doSwapout($) {
my $type = shift; # REAL==4, CLEANUP==3, RETRY==2, UPDATE==1.
my $swapout_errors = 0;
#
......@@ -324,18 +350,21 @@ sub doSwapout {
if (! $TESTMODE) {
if (! $DISABLE_EVENTS) {
print "Stopping the event system\n";
if (system("eventsys_control stop $pid $eid")) {
print STDERR "*** Failed to stop the event system.\n";
$swapout_errors = 1;
if ($type >= RETRY) {
print "Stopping the event system\n";
if (system("eventsys_control stop $pid $eid")) {
print STDERR "*** Failed to stop the event system.\n";
$swapout_errors = 1;
}
}
}
#
# If this is an actual swapout, always clean up VLANs
# If this is an actual swapout, or an update/retry, always clean up VLANs
# Otherwise, only do it if swapin previously messed them up.
#
if ($swapop eq "out" || $cleanvlans) {
# XXX Is doing this all the time cromulent?
if ($type != CLEANUP || $cleanvlans) {
TBDebugTimeStamp("snmpit started");
print STDERR "Removing VLANs.\n";
if (system("snmpit -r $pid $eid")) {
......@@ -350,17 +379,20 @@ sub doSwapout {
#
# This is a hack. We need a more general os_teardown, but for now
# we just kill off the vnode stuff.
# (don't kill off vnodes when UPDATEing.)
#
print "Tearing down virtual nodes.\n";
TBDebugTimeStamp("vnode_setup -k started");
if (system("vnode_setup -d -k $pid $eid")) {
print STDERR "*** Failed to tear down vnodes.\n";
$swapout_errors = 1;
if ($type >= RETRY) {
print "Tearing down virtual nodes.\n";
TBDebugTimeStamp("vnode_setup -k started");
if (system("vnode_setup -d -k $pid $eid")) {
print STDERR "*** Failed to tear down vnodes.\n";
$swapout_errors = 1;
}
TBDebugTimeStamp("vnode_setup finished");
}
TBDebugTimeStamp("vnode_setup finished");
}
if (! $retry) {
if ($type >= CLEANUP) {
#
# We're not attempting a retry;
# remove all nodes from the experiment.
......@@ -375,7 +407,7 @@ sub doSwapout {
TBDebugTimeStamp("nfree finished");
} else {
#
# Since $retry == 1, we are preparing for an experiment retry.
# $type == RETRY or $type == UPDATE.
# Therefore, don't deallocate nodes which have been successfully
# incorporated into the experiment (i.e., are RES_READY).
# (nfree will send deallocated nodes to RES_FREE_DIRTY)
......@@ -396,6 +428,9 @@ sub doSwapout {
}
}
# XXX reboot nodes [e.g., put them in RES_DIRTY()]
# if doing an update -restart.
if (@failedNodes > 0) {
TBDebugTimeStamp("nfree started");
#
......@@ -419,9 +454,9 @@ sub doSwapout {
#
# Only reset mountpoints if this is an actual swapout, and
# not a failed swapin.
# not a failed swapin(cleanup), update, or retry.
#
if ($swapop eq "out") {
if ($type == REAL) {
print "Resetting mountpoints.\n";
TBDebugTimeStamp("exports started");
if (system("exports_setup")) {
......@@ -430,6 +465,10 @@ sub doSwapout {
TBDebugTimeStamp("exports finished");
}
#
# Resetting named maps and email lists is fast and idempotent,
# so whatever.
#
print "Resetting named maps.\n";
TBDebugTimeStamp("named started");
if (system("named_setup")) {
......@@ -445,6 +484,10 @@ sub doSwapout {
TBDebugTimeStamp("genelists finished");
}
#
# Wipe the DB clean.
#
print STDERR "Resetting DB.\n";
DBQueryWarn("DELETE from delays where pid='$pid' and eid='$eid'")
or $swapout_errors++;
......@@ -473,7 +516,13 @@ sub doSwapout {
#
##
sub doSwapin {
sub doSwapin($) {
my $type = shift; # REAL==4, RETRY==2, UPDATE==1. (CLEANUP is nonsense.)
#
# By default, errors are not tagged as 'retry-suitable.'
#
$retry = 0;
#
# assign_wrapper does all the virtual to physical mapping
# and updating the DB state.
......@@ -569,15 +618,21 @@ sub doSwapin {
}
TBDebugTimeStamp("genelists finished");
print "Clearing port counters.\n";
TBDebugTimeStamp("portstats started");
if (system("portstats -z -a -q $pid $eid")) {
print STDERR "*** WARNING: Failed to clear port counters.\n";
#
# This is a non-fatal error.
#
#
# Don't clear port counters on UPDATE.
# (XXX should clear new nodes' port counters.)
if ($type >= RETRY) {
print "Clearing port counters.\n";
TBDebugTimeStamp("portstats started");
if (system("portstats -z -a -q $pid $eid")) {
print STDERR "*** WARNING: Failed to clear port counters.\n";
#
# This is a non-fatal error.
#
}
TBDebugTimeStamp("portstats finished");
}
TBDebugTimeStamp("portstats finished");
#
# OK, let's see how that os_setup did
......@@ -608,15 +663,19 @@ sub doSwapin {
# Okay, start the event system now that we know all the nodes have
# rebooted (os_setup is done). This only takes a moment (puts itself
# in the background), so its not enough of a delay to worry about.
# Don't do this during an update, since we didn't kill the
# event system previously, so starting it again will fail!
#
if (! $DISABLE_EVENTS) {
print "Starting the event system.\n";
TBDebugTimeStamp("eventsys_control started");
if (system("eventsys_control start $pid $eid")) {
print STDERR "*** Failed to start the event system.\n";
return 1;
if ($type != UPDATE) {
print "Starting the event system.\n";
TBDebugTimeStamp("eventsys_control started");
if (system("eventsys_control start $pid $eid")) {
print STDERR "*** Failed to start the event system.\n";
return 1;
}
TBDebugTimeStamp("eventsys_control finished");
}
TBDebugTimeStamp("eventsys_control finished");
}
return 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment