start-experiment 10.1 KB
Newer Older
1
#!/usr/bin/perl -w
Mike Hibler's avatar
Mike Hibler committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
Mike Hibler's avatar
Mike Hibler committed
5 6
# All rights reserved.
#
7 8

use strict;
9
use Getopt::Std;
10 11

my $TEVC = "/usr/testbed/bin/tevc";
12
my $LOGHOLE = "/usr/testbed/bin/loghole";
13
my $PYTHON = "/usr/local/bin/python";
14
my $PERL = "/usr/bin/perl";
15
my $EVENTSYS = "/usr/testbed/bin/eventsys_control";
16
my $NODELIST = "/usr/testbed/bin/node_list";
17
my $EXPINFO = "/usr/testbed/bin/expinfo";
Mike Hibler's avatar
Mike Hibler committed
18
my $INITCOND = "init-elabnodes.pl";
19

Mike Hibler's avatar
Mike Hibler committed
20
my $realplab = 0;
21
my $initelab = 0;
22
my $sanityChecks = 0;
Mike Hibler's avatar
Mike Hibler committed
23

24
my $UNKNOWN = "<unknown>";
25
my $stub_cmd = "/bin/sh /local/pelab/stub/auto-stub.sh";
26
my $stub_cmdargs = $UNKNOWN;
27
my $mon_cmd = "/bin/sh /local/pelab/monitor/auto-monitor.sh";
Mike Hibler's avatar
Mike Hibler committed
28
my $mon_cmdargs = $UNKNOWN;
29

30 31
sub get_cmdargs($$);

32 33
sub usage()
{
Mike Hibler's avatar
Mike Hibler committed
34
    print "Usage: $0 [-C] [ -S command-line ] [ -s command-args ] pid eid\n".
35 36
	  "  -p              monitor real planetlab nodes\n".
	  "  -i              initialize elab nodes with historic plab data\n". 
37 38
	  "  -N s|m          use new stub (s) or monitor (m)\n".
	  "  -O s|m          use old stub (s) or monitor (m)\n".
39 40 41 42
	  "  -S command-line replace the current stub command arguments\n".
	  "  -s command-args append args to stub command line\n".
	  "  -M command-line replace the current monitor command arguments\n".
	  "  -m command-args append args to monitor command line\n".
43
      "  -t              start tcpdumps for cross-correlation checks\n".
44
	  "  -C              show current command line(s)\n";
45 46
    exit(1);
}
47
my $optlist = "CS:s:M:m:piN:O:t";
48

Mike Hibler's avatar
Mike Hibler committed
49 50 51 52 53 54 55
my $pelabdir;
if ($0 =~ /(.*)\/[^\/]+$/) {
    $pelabdir = $1;
} else {
    $pelabdir = ".";
}

56 57 58 59 60 61 62 63
#
# Parse command arguments.
#
my %options = ();
if (!getopts($optlist, \%options)) {
    print "error: cannot parse options\n";
    usage();
}
64 65 66 67
#
# Require the pid and eid
#
if (@ARGV != 2) {
68
    usage();
69 70
}
my ($pid, $eid) = @ARGV;
71 72 73 74 75 76 77 78 79 80 81

#
# Make sure the experiment is swapped in!
#
my @expinfo = `$EXPINFO $pid $eid`;
if (grep /No such experiment/, @expinfo) {
    die "*** ERROR: Experiment does not exist!\n";
} elsif (! grep(/State: active/,@expinfo)) {
    die "*** ERROR: Experiment is not swapped in!\n";
}

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
#$stub_cmdargs = get_cmdargs("plab", $stub_cmd);
#$mon_cmdargs = get_cmdargs("elab", $mon_cmd);
#if (defined($options{"S"})) {
#    $stub_cmdargs = $options{"S"};
#}
#if (defined($options{"s"})) {
#    if ($stub_cmdargs eq $UNKNOWN) {
#	$stub_cmdargs = $options{"s"};
#    } else {
#	$stub_cmdargs .= " " . $options{"s"};
#    }
#}
#if (defined($options{"M"})) {
#    $mon_cmdargs = $options{"M"};
#}
#if (defined($options{"m"})) {
#    if ($mon_cmdargs eq $UNKNOWN) {
#	$mon_cmdargs = $options{"m"};
#    } else {
#	$mon_cmdargs .= " " . $options{"m"};
#    }
#}
#if (defined($options{"C"})) {
#    my $cmdline = $stub_cmd;
#    if ($stub_cmdargs ne $UNKNOWN) {
#	$cmdline .= " $stub_cmdargs";
#    }
#    print "Current stub command line: '$cmdline'\n";
#
#    $cmdline = $mon_cmd;
#    if ($mon_cmdargs ne $UNKNOWN) {
#	$cmdline .= " $mon_cmdargs";
#    }
#    print "Current monitor command line: '$cmdline'\n";
#}
Mike Hibler's avatar
Mike Hibler committed
117 118
if (defined($options{"p"})) {
    $realplab = 1;
Mike Hibler's avatar
Mike Hibler committed
119 120 121 122 123
    if ($mon_cmdargs eq $UNKNOWN) {
	$mon_cmdargs = "-p";
    } else {
	$mon_cmdargs .= " " . "-p";
    }
124
}
125 126 127 128 129
if (defined($options{"i"})) {
    $initelab = 1;
    print "WARNING: -i only makes sense with -p\n"
	if (!$realplab);
}
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
#if (defined($options{"N"})) {
#    if ($options{"N"} eq "s") {
#	$stub_cmd = "/bin/sh /local/pelab/magent/auto-magent.sh";
#	$stub_cmdargs = ""
#	    if ($stub_cmdargs eq $UNKNOWN);
#    } elsif ($options{"N"} eq "m") {
#	$mon_cmd = "/bin/sh /local/pelab/dbmonitor/auto-dbmonitor.sh";
#	$mon_cmdargs = ""
#	    if ($mon_cmdargs eq $UNKNOWN);
#    } else {
#	die "Unknown -N type '", $options{"N"}, "'\n";
#    }
#}
#if (defined($options{"O"})) {
#    if ($options{"O"} eq "s") {
#	$stub_cmd = "/bin/sh /local/pelab/stub/auto-stub.sh";
#	$stub_cmdargs = ""
#	    if ($stub_cmdargs eq $UNKNOWN);
#    } elsif ($options{"O"} eq "m") {
#	$mon_cmd = "/bin/sh /local/pelab/monitor/auto-monitor.sh";
#	$mon_cmdargs = ""
#	    if ($mon_cmdargs eq $UNKNOWN);
#    } else {
#	die "Unknown -O type '", $options{"O"}, "'\n";
#    }
#}
156 157 158
if (defined($options{"t"})) {
    $sanityChecks = 1;
}
159

Mike Hibler's avatar
Mike Hibler committed
160 161 162
my $use_magent = ($stub_cmd =~ /magent/) ? 1 : 0;
my $use_simplemodel = ($mon_cmd =~ /dbmonitor/) ? 1 : 0;

163
#
164
# Let's help some poor bozos (Rob) get the right command line args
165 166 167 168 169
#
print "##### Stashing node list for experiment\n";
my @vnodes = split /\s+/,`$NODELIST -v -e $pid,$eid`;
my $hasfakeplab = grep /plab-\d+/, @vnodes;
my $hasrealplab = grep /planet-\d+/, @vnodes;
170
print "HFPL = $hasfakeplab HRPL = $hasrealplab RPL = $realplab\n";
171 172 173 174 175 176
if (!$hasfakeplab && $hasrealplab && !$realplab) {
    die "*** ERROR: Forgot to specify -p!\n";
} elsif ($hasfakeplab && !$hasrealplab && $realplab) {
    die "*** ERROR: Should not use -p!\n";
}

177
#
178
# Make sure any old stubs/monitors are dead
179
#
180
print "##### Stopping old stubs and monitors\n";
Mike Hibler's avatar
Mike Hibler committed
181 182 183 184 185 186 187 188
if ($realplab) {
    if (system "$TEVC -e $pid/$eid now planetstubs stop") {
	die "Error running tevc\n";
    }
} else {
    if (system "$TEVC -e $pid/$eid now plabstubs stop") {
	die "Error running tevc\n";
    }
189 190
}
if (system "$TEVC -e $pid/$eid now monitorgroup stop") {
191 192 193 194 195 196 197
    die "Error running tevc\n";
}

#
# Make sure link logs get cleaned up
#
print "##### Rolling link tracing files \n";
Mike Hibler's avatar
Mike Hibler committed
198
if (system "$TEVC -e $pid/$eid now planet-tracemon snapshot") {
199 200
    die "Error running tevc\n";
}
Mike Hibler's avatar
Mike Hibler committed
201
if (system "$TEVC -e $pid/$eid now planet-tracemon stop") {
202 203
    die "Error running tevc\n";
}
204

205 206 207 208 209 210 211 212 213
#
# Smack down host tcpdumps
#
print "##### Stopping host tcpdumps\n";
if (system "$TEVC -e $pid/$eid now tdhosts stop") {
    die "Error running tevc\n";
}


214 215 216
#
# Restart the server program

217 218 219 220 221 222 223 224
#
# Clean out the logs so that we can start fresh
#
print "##### Cleaning old logfiles (ignore 'No match' errors)\n";
if (system "$LOGHOLE -e $pid/$eid clean -f -r") {
    die "Error running loghole\n";
}

225 226 227 228
#
# Reset the links so that we remove any delay changes we might have previously
# made
#
229 230 231 232 233 234 235 236
print "##### Resetting all links in the experiment..";
print "clear..";
if (system "$TEVC -e $pid/$eid now elabc clear") {
    die "Error running tevc elabc clear\n";
}
print "reset\n";
if (system "$TEVC -w -e $pid/$eid now elabc reset") {
    die "Error running tevc elabc reset\n";
237 238
}

239 240 241 242 243 244 245 246 247
#
# XXX stash node list for experiment since nodes don't have node_list
# right now.  This is needed by the dbmonitor if it is in use.
#
print "##### Stashing node list for experiment\n";
if (system "$NODELIST -m -e $pid,$eid > /proj/$pid/exp/$eid/tmp/node_list") {
    warn "Could not create node list for dbmonitor; it will fail if used\n";
}

248 249 250 251 252
#
# Restart the event system so the 'create' event, necessary for pelab to work,
# gets fired again
#
print "##### Restarting the event system\n";
253
if (system "$EVENTSYS -e $pid,$eid replay") {
254 255
    die "Error controlling the event system\n";
}
256

257 258 259
print "##### Waiting for event system to start\n";
sleep(10);

Mike Hibler's avatar
Mike Hibler committed
260 261 262 263 264 265 266 267 268 269 270 271
#
# Fetch and store the initial conditions for plab nodes if desired.
# We both stash them in a file and set them directly.  The former is
# read by the monitor at start up to inform the stubs.  The latter is
# for backward compat with the old monitor/stubs that did not do it for
# themselves.
#
if ($realplab && $initelab && !$use_simplemodel) {
    print "##### Stashing initial path conditions for plab nodes\n";
    if (system "$pelabdir/$INITCOND -o /proj/$pid/exp/$eid/tmp/initial-conditions.txt $pid $eid") {
	warn "Could not acquire initial path conditions\n";
    }
272 273 274 275
#    print "##### Initializing Emulab link characteristics\n";
#    if (system "$pelabdir/$INITCOND -C $pid $eid") {
#	warn "Error initializing Emulab links with initial path conditions\n";
#    }
276 277 278
} else {
    # avoid old stale conditions
    unlink("/proj/$pid/exp/$eid/tmp/initial-conditions.txt");
279 280
}

281 282 283 284 285 286 287 288
print "##### Restarting servers\n";
if (system "$TEVC -e $pid/$eid now allservers stop") {
    die "Error running tevc\n";
}
if (system "$TEVC -e $pid/$eid now allservers start") {
    die "Error running tevc\n";
}

289 290 291 292
#
# Start link tracing
#
print "##### Starting link tracing\n";
Mike Hibler's avatar
Mike Hibler committed
293
if (system "$TEVC -e $pid/$eid now planet-tracemon start") {
294 295 296
    die "Error running tevc\n";
}

297 298 299 300 301 302 303 304
#
# Start host tcpdumps
#
print "##### Starting host tcpdumps\n";
if (system "$TEVC -e $pid/$eid now tdhosts start") {
    die "Error running tevc\n";
}

305 306 307
#
# Start up the stubs and monitors
#
308
#my $startarg = "";
309
print "##### Starting stubs";
310 311 312 313 314 315 316
#if ($initelab && $use_magent) {
#    $stub_cmdargs += " /proj/$pid/exp/$eid/tmp/initial-conditions.txt ";
#}
#if ($stub_cmdargs ne $UNKNOWN) {
#    print " with: '$stub_cmd $stub_cmdargs'";
#    $startarg = "'COMMAND=$stub_cmd $stub_cmdargs'";
#}
317
print "\n";
Mike Hibler's avatar
Mike Hibler committed
318
if ($realplab) {
319
    if (system "$TEVC -e $pid/$eid now planetstubs start") {
Mike Hibler's avatar
Mike Hibler committed
320 321 322
	die "Error running tevc\n";
    }
} else {
323
    if (system "$TEVC -e $pid/$eid now plabstubs start") {
Mike Hibler's avatar
Mike Hibler committed
324 325
	die "Error running tevc\n";
    }
326
}
327

328
#$startarg = "";
329
print "##### Starting monitors";
330 331 332 333
#if ($mon_cmdargs ne $UNKNOWN) {
#    print " with: '$mon_cmd $mon_cmdargs'";
#    $startarg = "'COMMAND=$mon_cmd $mon_cmdargs'";
#}
334
print "\n";
335
if (system "$TEVC -e $pid/$eid now monitorgroup start") {
336 337 338
    die "Error running tevc\n";
}

339
#print "##### Starting tcpdumps\n";
340 341 342 343
#if (system "$TEVC -e $pid/$eid now plabtcpdump start") {
#    die "Error running tevc\n";
#}

344 345 346 347 348
print "##### Starting tcpdumps for cross-correlation sanity checks\n";
if (system "$TEVC -e $pid/$eid now sanityCheckDumps start") {
    die "Error running tevc\n";
}

349
print "##### Done\n";
350 351 352 353 354 355 356

exit(0);

#
# Dig the command line out of the logs from the previous run
# (if there was one)
#
Mike Hibler's avatar
Mike Hibler committed
357
sub get_cmdargs($$)
358
{
Mike Hibler's avatar
Mike Hibler committed
359
    my ($ntype,$cmd) = @_;
360
    my $cmdargs = $UNKNOWN;
Mike Hibler's avatar
Mike Hibler committed
361
    my $lfile = ($ntype eq "plab") ? "plabstub" : "monitor";
362 363 364 365 366 367 368 369 370

    my @list = `/bin/ls /proj/$pid/exp/$eid/logs/$eid.*.zip`;
    return $cmdargs
	if ($?);
    chomp(@list);

    my $file = $list[$#list];
    return $cmdargs
	if (! -r $file);
Mike Hibler's avatar
Mike Hibler committed
371
    my @info = `unzip -p $file "${ntype}-1/local/logs/${lfile}-1.status"`;
372 373
    foreach (@info) {
	if (/^COMMAND=(.*)/) {
Mike Hibler's avatar
Mike Hibler committed
374 375
	    if ($1 ne $cmd) {
		$cmdargs = $1;
Mike Hibler's avatar
Mike Hibler committed
376 377 378 379 380 381 382 383

		# XXX filter out broken cmdlines (old bug in program_agent)
		$cmdargs =~ s/\s*}\s*$//;

		$cmdargs =~ s/^$cmd//;
		if ($cmdargs eq "") {
		    $cmdargs = $UNKNOWN;
		}
Mike Hibler's avatar
Mike Hibler committed
384
	    }
385 386 387 388 389 390
	    last;
	}
    }

    return $cmdargs;
}