Commit 375f87c1 authored by Shashi Guruprasad's avatar Shashi Guruprasad

Changes to do auto re-swap of expts with simnodes when an nse on a simhost

(or more than one simhost) is unable to keep up with real-time. It includes
changes to assign_wrapper to handle swap modify for simnodes, the simple
algorithm in nseswap that bumps up the nodeweight of simnodes being hosted
on a simhost that reports "can't keep up with real-time" (aka nse violation),
ptopgen and sim.tcl to prefer nodes that already have the FBSD-NSE image.
Also, changes to other files to send out NSESWAP event.

One unrelated change: We now have per-swapin .top files and assign.log
files along with .ptop files. This helps in debugging across multiple
swapins since files remain in the form of
<pid>-<eid>-<process_id>.{top,ptop} and assign-<pid>-<eid>-<process_id>.log
Also useful for archiving.
parent 50352918
......@@ -353,7 +353,7 @@ if { $simcode_present == 1 } {
# flushed and we can start cleanly
exec $CLIENTVARDIR/boot/rc.ifc disable
exec $CLIENTVARDIR/boot/rc.ifc enable
# Now, we configure IPTaps for links between real and simulated nodes
set i 0
......@@ -461,10 +461,13 @@ set vnode [lindex $pideidlist 0]
set eid [lindex $pideidlist 1]
set pid [lindex $pideidlist 2]
set logpath "/proj/$pid/exp/$eid/logs/nse-$vnode.log"
set simobjname [$ns set tbname]
set nseswap_cmdline "$CLIENTBINDIR/tevc -s $boss -e $pid/$eid now $simobjname NSESWAP SIMHOST=$vnode"
# Configuring the Scheduler to monitor the event system
set evsink [new TbEventSink]
$evsink event-server "elvin://$boss"
$evsink nseswap_cmdline $nseswap_cmdline
#if { $objnamelist != {} } {
# $evsink objnamelist [join $objnamelist ","]
#}
......
......@@ -30,7 +30,7 @@ TbEventSink::callback(event_handle_t handle,
char buf[7][64];
int len = 64;
char argsbuf[8192];
buf[0][0] = buf[1][0] = buf[2][0] = buf[3][0] = 0;
buf[4][0] = buf[5][0] = buf[6][0] = buf[7][0] = 0;
event_notification_get_site(handle, notification, buf[0], len);
......@@ -41,10 +41,10 @@ TbEventSink::callback(event_handle_t handle,
event_notification_get_objname(handle, notification, buf[5], len);
event_notification_get_eventtype(handle, notification, buf[6], len);
event_notification_get_arguments(handle, notification, argsbuf, sizeof(argsbuf));
struct timeval now;
gettimeofday(&now, NULL);
info("Received Event: %lu:%ld \"%s\"\n", now.tv_sec, now.tv_usec, argsbuf);
// All events coming in to NSE should be NSEEVENT events and we just need to
......@@ -52,11 +52,11 @@ TbEventSink::callback(event_handle_t handle,
// we just print a warning in the log. Also we need a mechanism
// to report the error back to the experimenter
// XXX: fix needed
if( Tcl_GlobalEval( Tcl::instance().interp(), argsbuf ) != TCL_OK ) {
error( "Tcl Eval error in code: \"%s\"\n", argsbuf );
}
}
......@@ -67,7 +67,7 @@ TbEventSink::~TbEventSink() {
*/
if (event_unregister(ehandle) == 0) {
fatal("could not unregister with event system");
}
}
}
}
......@@ -81,22 +81,22 @@ TbEventSink::init() {
} else {
loginit(1, "Testbed-NSE");
}
if( server[0] == 0 ) {
fatal("event system server unknown\n");
}
/*
* Get our IP address. Thats how we name ourselves to the
* Testbed Event System.
* Testbed Event System.
*/
struct hostent *he;
struct in_addr myip;
if (gethostname(buf, sizeof(buf)) < 0) {
fatal("could not get hostname");
}
if (! (he = gethostbyname(buf))) {
fatal("could not get IP address from hostname");
}
......@@ -104,7 +104,7 @@ TbEventSink::init() {
strncpy(ipaddr, inet_ntoa(myip), sizeof(ipaddr));
/*
* Register with the event system.
* Register with the event system.
*/
ehandle = event_register(server, 0);
if (ehandle == NULL) {
......@@ -116,7 +116,7 @@ TbEventSink::init() {
int
TbEventSink::poll() {
int rv;
rv = event_poll(ehandle);
if (rv)
fatal("event_poll failed, err=%d\n", rv);
......@@ -131,7 +131,7 @@ TbEventSink::poll() {
int TbEventSink::command(int argc, const char*const* argv)
{
if( argc == 3 ) {
if(strcmp(argv[1], "event-server") == 0) {
strncpy(server, argv[2], sizeof(server));
......@@ -145,10 +145,14 @@ int TbEventSink::command(int argc, const char*const* argv)
strncpy(logfile, argv[2], sizeof(logfile));
return(TCL_OK);
}
if(strcmp(argv[1], "nseswap_cmdline") == 0) {
strncpy(nseswap_cmdline, argv[2], sizeof(nseswap_cmdline));
return(TCL_OK);
}
}
return (TclObject::command(argc, argv));
}
}
void
TbEventSink::subscribe() {
......@@ -176,7 +180,7 @@ TbEventSink::subscribe() {
}
tuple->eventtype = ADDRESSTUPLE_ANY;
if (!event_subscribe(ehandle, callback, tuple, this)) {
fatal("could not subscribe to NSE events for objects %s", tuple->objname );
}
......@@ -184,6 +188,22 @@ TbEventSink::subscribe() {
}
void
TbEventSink::send_nseswap() {
if (nseswap_cmdline[0] != '\0') {
int ret = system(nseswap_cmdline);
if (ret == -1) {
fatal("Could not execute cmd:%s\n",
nseswap_cmdline);
} else if (ret != 0) {
fatal("cmd:\"%s\" exited with failure with return code:%d\n",
nseswap_cmdline, ret);
}
info("Sending NSESWAP event with cmdline: \"%s\"\n",
nseswap_cmdline);
}
}
static class TbResolverClass : public TclClass {
public:
TbResolverClass() : TclClass("TbResolver") {}
......@@ -194,7 +214,7 @@ public:
int TbResolver::command(int argc, const char*const* argv)
{
if( argc == 3 ) {
if(strcmp(argv[1], "lookup") == 0) {
struct hostent *he = gethostbyname(argv[2]);
......@@ -212,6 +232,6 @@ int TbResolver::command(int argc, const char*const* argv)
return(TCL_OK);
}
}
return (TclObject::command(argc, argv));
}
}
......@@ -31,6 +31,7 @@ public:
bzero(server, sizeof(server));
bzero(objnamelist, sizeof(objnamelist));
bzero(logfile, sizeof(logfile));
bzero(nseswap_cmdline, sizeof(nseswap_cmdline));
}
~TbEventSink();
virtual int command(int argc, const char*const* argv);
......@@ -38,6 +39,7 @@ public:
void init();
void subscribe();
int poll();
void send_nseswap();
private:
......@@ -47,6 +49,7 @@ private:
char ipaddr[BUFSIZ];
char objnamelist[BUFSIZ];
char logfile[MAXPATHLEN];
char nseswap_cmdline[MAXPATHLEN];
static void
callback(event_handle_t handle,
......
......@@ -591,6 +591,7 @@ handle_simevent(event_handle_t handle, sched_event_t *eventp)
char evtype[TBDB_FLEN_EVEVENTTYPE];
int rcode;
char cmd[BUFSIZ];
char argsbuf[BUFSIZ];
if (! event_notification_get_eventtype(handle,
eventp->notification,
......@@ -629,7 +630,9 @@ handle_simevent(event_handle_t handle, sched_event_t *eventp)
sprintf(cmd, "endexp %s %s", pid, eid);
}
else if (!strcmp(evtype, TBDB_EVENTTYPE_NSESWAP)) {
sprintf(cmd, "nseswap %s %s", pid, eid);
event_notification_get_arguments(handle, eventp->notification,
argsbuf, sizeof(argsbuf));
sprintf(cmd, "nseswap %s %s %s", pid, eid, argsbuf);
}
rcode = system(cmd);
......
......@@ -128,7 +128,11 @@ if (defined($options{"n"})) {
my $pid = $ARGV[0];
my $eid = $ARGV[1];
my $ptopfile = "$pid-$eid-$$.ptop";
my $topfile = "$eid.top";
# Since the topfile could change across
# swapins and modifies, it makes sense
# to store all of them. Helps in
# degugging.
my $topfile = "$pid-$eid-$$.top";
sub fatal ($$)
{
......@@ -296,6 +300,7 @@ my $minimum_nodes;
my $maximum_nodes;
my $reserved_pcount = 0;
my $reserved_vcount = 0;
my $reserved_simcount= 0;
my $remotecount = 0;
my $virtcount = 0;
my $plabcount = 0;
......@@ -329,7 +334,9 @@ my %expt_stats = (# pnodes include jailnodes and delaynodes.
maxlinks => 0,
);
my $nsenode_id = 0;
my $simhost_id = 0;
my %pnode2simhostid;
my %simhostid2pnode;
# Counters for generating IDs.
my $virtnode_id = 0;
......@@ -398,8 +405,11 @@ LoadExperiment();
# If updating, load current experiment resources. We have to be careful
# of how this is merged in with the (new) desired topology. See below.
#
LoadCurrent()
if ($updating);
if ($updating) {
LoadCurrent();
print STDERR "Resetting DB before updating.\n";
TBExptRemovePhysicalState( $pid, $eid );
}
#
# Check Max Concurrent for OSID violations.
......@@ -580,6 +590,11 @@ sub RunAssign ($)
my $violations = 0;
my $score = -1;
# Saving up assign.log coz each swapin/modify is
# different and it is nice to have every mapping
# for debugging and archiving purposes
system("/bin/cp assign.log assign-$pid-$eid-$$.log");
open(ASSIGNFP, "assign.log") or
fatal($WRAPPER_FAILED|$WRAPPER_FAILED_CANRECOVER,
"Could not open assign logfile!");
......@@ -2067,7 +2082,7 @@ sub requires_delay {
my $targetbandwidth = $_[0];
my $bestbandwidth = getbandwidth($targetbandwidth);
printdb "Checking $bestbandwidth against $targetbandwidth\n";
# XXX temporary hack: && $bestbandwidth < $S100Kbs
# XXX temporary hack: && $bestbandwidth >= $S100Kbs
# A delay node is not being added for the 10Mb 0ms link
# Permanent fix required
if ($bestbandwidth == $targetbandwidth && $bestbandwidth >= $S100Kbs) {
......@@ -2103,11 +2118,12 @@ sub InitPnode($pnode, $vnode)
my $vname;
my $osid;
my $role;
my $simhost_violation;
# XXX NSE hack: if the vnode is simulated, we just
# choose FBSD-NSE and static routing
if (virtnodeisvnode($vnode) && virtnodeissim($vnode)) {
$osid = TBOSID("emulab-ops", "FBSD-NSE" );
$osid = TBOSID(TB_OPSPID, "FBSD-NSE" );
DBQueryFatal("UPDATE nodes set def_boot_cmd_line=''," .
" startstatus='none'," .
......@@ -2121,9 +2137,9 @@ sub InitPnode($pnode, $vnode)
" routertype='" . TBDB_ROUTERTYPE_STATIC() . "'" .
" where node_id='$pnode'");
$vname = "simhost-${nsenode_id}";
$vname = newvname_simhost($pnode);
$role = TBDB_RSRVROLE_SIMHOST;
$nsenode_id++;
$simhost_violation = 0;
}
elsif (virtnodeisremote($vnode) && $v2vmap{$vnode} ne $pnode) {
#
......@@ -2209,13 +2225,16 @@ sub InitPnode($pnode, $vnode)
#
# Set the vname and role.
#
if (defined($vname) || defined($role)) {
if (defined($vname) || defined($role) || defined($simhost_violation)) {
my $sets = "";
$sets .= "vname='$vname' "
if (defined($vname));
$sets .= (defined($vname) ? "," : "") . "erole='$role' "
if (defined($role));
$sets .= (defined($role) || defined($vname) ? "," : "") .
"simhost_violation='$simhost_violation' "
if (defined($simhost_violation));
DBQueryFatal("update reserved set $sets where node_id='$pnode'");
}
......@@ -2640,6 +2659,36 @@ sub newvname($$)
}
}
#
# Give me a new vname for an internally allocated node. We have to
# watch for names that were made up previously (say, if this is an
# update). Not allowed to reuse names of course. We do not mark nodes
# as hosting, so have to infer this from reserved_pnodes. I'm sure
# there is a better way to do this.
#
sub newvname_simhost($)
{
my ($pnode) = @_;
#
# First check to see if this pnode was already allocated (update)
#
if (defined($pnode2simhostid{$pnode})) {
return $pnode2simhostid{$pnode};
}
while (1) {
my $newvname = "simhost-" . $simhost_id;
$simhost_id++;
if (!defined($simhostid2pnode{$newvname})) {
$simhostid2pnode{$newvname} = $pnode;
$pnode2simhostid{$pnode} = $newvname;
return $newvname;
}
}
}
#
# Load up phys info. Not much to it.
#
......@@ -2781,6 +2830,7 @@ sub LoadVirtNodes()
# Extend the DB info with this stuff:
# Easy access ...
$rowref->{"__nodeweight"} = undef;
$rowref->{"__isremotenode"} = $isremote;
$rowref->{"__isvirtnode"} = $isvirt;
$rowref->{"__issubnode"} = $issub;
......@@ -3262,16 +3312,26 @@ sub CreateTopFile()
# Yuck
$desirestr .= " +load:" . (($cpu_usage - 1) / 5.0);
}
print TOPFILE "node $vname $type $subnodestr $desirestr\n";
my $typestr = $type;
if (virtnodeisvirt($vname)) {
$virtnode_count++;
}
elsif (virtnodeissim($vname)) {
$simnode_count++;
my $query_result = DBQueryFatal("select nodeweight from ".
"virt_simnode_attributes ".
"where pid='$pid' and ".
"eid='$eid' and ".
"vname='$vname'");
my ($nodeweight) = $query_result->fetchrow_array();
if ($nodeweight) {
$typestr = "$type:$nodeweight";
}
}
else {
$physnode_count++;
}
print TOPFILE "node $vname $typestr $subnodestr $desirestr\n";
}
}
......@@ -3757,13 +3817,13 @@ sub LoadCurrent()
my $query_result =
DBQueryFatal("select r.vname,r.node_id,n.phys_nodeid, ".
" nt.isvirtnode,nt.isremotenode,nt.isplabdslice ".
" from reserved as r ".
" nt.isvirtnode,nt.isremotenode,nt.isplabdslice,r.erole,".
"r.simhost_violation from reserved as r ".
"left join nodes as n on n.node_id=r.node_id ".
"left join node_types as nt on nt.type=n.type ".
"where r.pid='$pid' and r.eid='$eid'");
while (($vname,$reserved,$physnode,$isvirt,$isremote,$isplab) =
while (my ($vname,$reserved,$physnode,$isvirt,$isremote,$isplab,$erole,$simhost_violation) =
$query_result->fetchrow_array) {
#
......@@ -3791,19 +3851,50 @@ sub LoadCurrent()
$fixed_nodes{$vname} = $physnode
if (!defined($fixed_nodes{$vname}));
$reserved_vcount++;
}
}
else {
$reserved_v2pmap{$vname} = $reserved;
physnodesetreuse($reserved, "unused");
# Allow for the user to "move" a node. Yuck!
$fixed_nodes{$vname} = $reserved
if (!defined($fixed_nodes{$vname}));
$reserved_pcount++;
if ($erole eq TBDB_RSRVROLE_SIMHOST) {
printdb "simhost:$reserved has vname:$vname with ".
"simhost_violation:$simhost_violation\n";
$pnode2simhostid{$reserved} = $vname;
$simhostid2pnode{$vname} = $reserved;
physnodesetreuse($reserved, "unused");
$reserved_pcount++;
# We fix nodes for pnodes that have not reported
# a violation (i.e. can't keep up with real-time)
if ( ! $simhost_violation ) {
my $query_result2 =
DBQueryFatal("select vname from v2pmap".
" where pid='$pid' and eid='$eid'".
" and node_id='$reserved'");
printdb "$vname:$reserved has ";
printdb $query_result2->numrows;
printdb " simnodes\n";
while (my ($simnode) = $query_result2->fetchrow_array()) {
$fixed_nodes{$simnode} = $reserved
if (!defined($fixed_nodes{$simnode}));
$reserved_v2pmap{$simnode} = $reserved;
printdb "Fixing simnode:$simnode -> $reserved\n";
$reserved_simcount++;
}
}
}
else {
$reserved_v2pmap{$vname} = $reserved;
physnodesetreuse($reserved, "unused");
printdb "non simhost:$reserved has vname:$vname\n";
# Allow for the user to "move" a node. Yuck!
$fixed_nodes{$vname} = $reserved
if (!defined($fixed_nodes{$vname}));
$reserved_pcount++;
}
}
}
print "Reserved pnodes = $reserved_pcount\n"
if ($reserved_pcount);
print "Reserved vnodes = $reserved_vcount\n"
if ($reserved_vcount);
print "Reserved simnodes = $reserved_simcount\n"
if ($reserved_simcount);
}
......@@ -91,6 +91,10 @@ Simulator instproc node {args} {
# simulated nodes have type 'sim'
if { $simulated == 1 } {
tb-set-hardware $curnode sim
# This allows assign to prefer pnodes
# that already have FBSD-NSE as the default
# boot osid over others
$curnode add-desire "FBSD-NSE" 0.9
}
set node_list($curnode) {}
set last_class $curnode
......
......@@ -405,6 +405,8 @@ namespace eval GLOBALS {
foreach pnode [array names p2vmapsim] {
real_set nseconfig($pnode) {}
append nseconfig($pnode) "set [$sim set objname] [$sim set createcmd]\n"
append nseconfig($pnode) \
"\$[$sim set objname] set tbname \{$v2vmap([$sim set objname])\}\n"
append nseconfig($pnode) "[$sim set nseconfig]\n\n"
}
......@@ -620,4 +622,4 @@ namespace eval GLOBALS {
$sim spitxml_finish
}
tb_nseparse_cleanup_and_exit
\ No newline at end of file
tb_nseparse_cleanup_and_exit
......@@ -6,6 +6,7 @@
# All rights reserved.
#
use Fcntl ':flock';
use English;
use Getopt::Std;
use Socket;
......@@ -20,10 +21,11 @@ use Socket;
sub usage()
{
print STDOUT
"Usage: nseswap pid eid\n";
"Usage: nseswap [-v] pid eid <eventargs>\n";
exit(-1);
}
my $optlist = "v";
#
# Configure variables
......@@ -32,10 +34,22 @@ my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $CONTROL = "@USERNODE@";
my $TESTMODE = @TESTMODE@;
my $TBLOGS = "@TBLOGSEMAIL@";
# Locals
my $pid;
my $eid;
my $eventargs;
my $simhost = "";
my $max_retries = 3;
my $verbose = 0;
sub printdb ($)
{
if ($verbose) {
print $_[0];
}
}
#
# Turn off line buffering on output
......@@ -55,13 +69,163 @@ use lib "@prefix@/lib";
use libdb;
use libtestbed;
if (@ARGV != 2) {
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"v"})) {
$verbose = 1;
}
if (@ARGV != 3) {
usage();
}
$pid = $ARGV[0];
$eid = $ARGV[1];
$eventargs = $ARGV[2];
#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w.]+)$/) {
$pid = $1;
}
else {
die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\@\w.]+)$/) {
$eid = $1;
}
else {
die("Tainted argument $eid!\n");
}
my $argpat = q(SIMHOST=([-\w]+));
if ( $eventargs =~ /$argpat/ ) {
$simhost = $1;
}
my $lockfile = "/var/tmp/$pid-$eid-nseswap-lockfile";
my $query_result =
DBQueryFatal("select node_id from reserved where pid='$pid' and ".
"eid='$eid' and vname='$simhost'");
if (! $query_result->numrows) {
# print warning in some log
print STDERR "*** $0: \"$simhost\" is not in the reserved table\n";
exit(1);
}
my ($node_id) = $query_result->fetchrow_array();
# Update the DB with info from the NSESWAP event
# and be done with it
DBQueryFatal("lock tables reserved write");
DBQueryFatal("update reserved set simhost_violation='1' ".
"where node_id='$node_id' ".
"and pid='$pid' and eid='$eid'");
printdb "node:$node_id simhost=$simhost simhost_violation set\n";
DBQueryFatal("unlock tables");
$query_result =
DBQueryFatal("select vname from v2pmap where ".
"pid='$pid' and eid='$eid' and node_id='$node_id'");
# Nothing to do right now. Code will be added when rest of the
# infrastructure is in place
while( ($vname) = $query_result->fetchrow_array() ) {
my $query2_result = DBQueryFatal("select nodeweight from ".
"virt_simnode_attributes where ".
"pid='$pid' and eid='$eid' and ".
"vname='$vname'");
my $nodeweight = 2;
if ( $query2_result->numrows ) {
($nodeweight) = $query2_result->fetchrow_array();
$nodeweight *= 2;
}
DBQueryFatal("replace into virt_simnode_attributes ".
"(pid,eid,vname,nodeweight) values ".
"('$pid','$eid','$vname','$nodeweight')");
}
#
# We need to serialize this script since multiple pnodes
# could be reporting an error where nse can't keep up.
# The first pnode that caused this will run and wait for
# a little while to see if there are other pnodes reporting
# errors. Eventually, the first nseswap will cause re-swapin
# of the experiment. The subsequent nseswap scripts will just
# update the DB and be done with it
#
umask(002);
open(LOCK, ">>$lockfile") || fatal("Couldn't open $lockfile\n");
if (flock(LOCK, LOCK_EX|LOCK_NB)) {
swapout_on_max_retries();
# We wait for a few seconds to let any other pnodes that may not
# be able to track real-time
sleep(20);
DBQueryFatal("update experiments set sim_reswap_count=sim_reswap_count+1 ".
"where eid='$eid' and pid='$pid'");
# do a swap modify
system("swapexp -e -r -s modify $pid $eid");
}
#
# Close the lock file. Exiting releases it, but might as well.
#
close(LOCK);
exit(0);
sub swapout_on_max_retries() {
my $query_result =
DBQueryFatal("select sim_reswap_count from experiments where eid='$eid' ".
"and pid='$pid'");
my ($sim_reswap_count) = $query_result->fetchrow_array();
if ($sim_reswap_count >= $max_retries) {
my $dbuid;
my $user_name;
my $user_email;
#
# Verify user and get his DB uid.
#
if (! UNIX2DBUID($UID, \$dbuid)) {
die("*** $0:\n".
" You do not exist in the Emulab Database.\n");
}
#
# Get email info for user.
#
if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
die("*** $0:\n".
" Cannot determine your name and email address.\n");
}
$message =
"Experiment $pid/$eid reached max retries:$max_retries trying to re-map \n".
"simulated nodes. Forcibly swapping out the experiment\n";
SENDMAIL("$user_name <$user_email>",
"Experiment $pid/$eid Swapping out",
$message,
$TBOPS,
"Bcc: $TBLOGS");
system("swapexp -f -s out $pid $eid");
sleep(10);
exit(2);
}
return;
}
......@@ -224,8 +224,8 @@ if (defined($exempt_eid)) {
}
$result =
DBQueryFatal("select a.node_id,a.type,a.phys_nodeid,t.class,t.issubnode " .
"from nodes as a ".
DBQueryFatal("select a.node_id,a.type,a.phys_nodeid,t.class,t.issubnode," .
"a.def_boot_osid from nodes as a ".
"left join reserved as b on a.node_id=b.node_id ".
"left join reserved as m on a.phys_nodeid=m.node_id ".
"left join nodes as np on a.phys_nodeid=np.node_id ".
......@@ -239,13 +239,15 @@ $result =
# to use all nodes), or if there is no entry in the perms table for
# the type/class of node.
#
while (($node,$type,$physnode,$class,$issubnode) = $result->fetchrow_array) {
while (($node,$type,$physnode,$class,$issubnode,$def_boot_osid)
= $result->fetchrow_array) {
$nodes{$node} = $type
if (!defined($pid) ||
($permissions{$type} && $permissions{$class}));
if ($issubnode) {
$subnode_of{$node} = $physnode;
}
$node_def_boot_osid{$node} = $def_boot_osid;
}
foreach $node (keys(%nodes)) {
......@@ -259,6 +261,16 @@ foreach $node (keys(%nodes)) {
my @flags;
my $needvirtgoo = 0;
# XXX temporary hack until node reboot avoidance