start-experiment 10.8 KB
Newer Older
1
#!/usr/bin/perl -w
Mike Hibler's avatar
Mike Hibler committed
2
#
3
# Copyright (c) 2006, 2007 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Mike Hibler's avatar
Mike Hibler committed
23
#
24 25

use strict;
26
use Getopt::Std;
27 28

my $TEVC = "/usr/testbed/bin/tevc";
29
my $LOGHOLE = "/usr/testbed/bin/loghole";
30
my $PYTHON = "/usr/local/bin/python";
31
my $PERL = "/usr/bin/perl";
32
my $EVENTSYS = "/usr/testbed/bin/eventsys_control";
33
my $NODELIST = "/usr/testbed/bin/node_list";
34
my $EXPINFO = "/usr/testbed/bin/expinfo";
Mike Hibler's avatar
Mike Hibler committed
35
my $INITCOND = "init-elabnodes.pl";
36

Mike Hibler's avatar
Mike Hibler committed
37
my $realplab = 0;
38
my $initelab = 0;
39
my $sanityChecks = 0;
Mike Hibler's avatar
Mike Hibler committed
40

41
my $UNKNOWN = "<unknown>";
42
my $stub_cmd = "/bin/sh /local/pelab/stub/auto-stub.sh";
43
my $stub_cmdargs = $UNKNOWN;
44
my $mon_cmd = "/bin/sh /local/pelab/monitor/auto-monitor.sh";
Mike Hibler's avatar
Mike Hibler committed
45
my $mon_cmdargs = $UNKNOWN;
46

47 48
sub get_cmdargs($$);

49 50
sub usage()
{
Mike Hibler's avatar
Mike Hibler committed
51
    print "Usage: $0 [-C] [ -S command-line ] [ -s command-args ] pid eid\n".
52 53
	  "  -p              monitor real planetlab nodes\n".
	  "  -i              initialize elab nodes with historic plab data\n". 
54 55
	  "  -N s|m          use new stub (s) or monitor (m)\n".
	  "  -O s|m          use old stub (s) or monitor (m)\n".
56 57 58 59
	  "  -S command-line replace the current stub command arguments\n".
	  "  -s command-args append args to stub command line\n".
	  "  -M command-line replace the current monitor command arguments\n".
	  "  -m command-args append args to monitor command line\n".
60
      "  -t              start tcpdumps for cross-correlation checks\n".
61
	  "  -C              show current command line(s)\n";
62 63
    exit(1);
}
64
my $optlist = "CS:s:M:m:piN:O:t";
65

Mike Hibler's avatar
Mike Hibler committed
66 67 68 69 70 71 72
my $pelabdir;
if ($0 =~ /(.*)\/[^\/]+$/) {
    $pelabdir = $1;
} else {
    $pelabdir = ".";
}

73 74 75 76 77 78 79 80
#
# Parse command arguments.
#
my %options = ();
if (!getopts($optlist, \%options)) {
    print "error: cannot parse options\n";
    usage();
}
81 82 83 84
#
# Require the pid and eid
#
if (@ARGV != 2) {
85
    usage();
86 87
}
my ($pid, $eid) = @ARGV;
88 89 90 91 92 93 94 95 96 97 98

#
# Make sure the experiment is swapped in!
#
my @expinfo = `$EXPINFO $pid $eid`;
if (grep /No such experiment/, @expinfo) {
    die "*** ERROR: Experiment does not exist!\n";
} elsif (! grep(/State: active/,@expinfo)) {
    die "*** ERROR: Experiment is not swapped in!\n";
}

99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
#$stub_cmdargs = get_cmdargs("plab", $stub_cmd);
#$mon_cmdargs = get_cmdargs("elab", $mon_cmd);
#if (defined($options{"S"})) {
#    $stub_cmdargs = $options{"S"};
#}
#if (defined($options{"s"})) {
#    if ($stub_cmdargs eq $UNKNOWN) {
#	$stub_cmdargs = $options{"s"};
#    } else {
#	$stub_cmdargs .= " " . $options{"s"};
#    }
#}
#if (defined($options{"M"})) {
#    $mon_cmdargs = $options{"M"};
#}
#if (defined($options{"m"})) {
#    if ($mon_cmdargs eq $UNKNOWN) {
#	$mon_cmdargs = $options{"m"};
#    } else {
#	$mon_cmdargs .= " " . $options{"m"};
#    }
#}
#if (defined($options{"C"})) {
#    my $cmdline = $stub_cmd;
#    if ($stub_cmdargs ne $UNKNOWN) {
#	$cmdline .= " $stub_cmdargs";
#    }
#    print "Current stub command line: '$cmdline'\n";
#
#    $cmdline = $mon_cmd;
#    if ($mon_cmdargs ne $UNKNOWN) {
#	$cmdline .= " $mon_cmdargs";
#    }
#    print "Current monitor command line: '$cmdline'\n";
#}
Mike Hibler's avatar
Mike Hibler committed
134 135
if (defined($options{"p"})) {
    $realplab = 1;
Mike Hibler's avatar
Mike Hibler committed
136 137 138 139 140
    if ($mon_cmdargs eq $UNKNOWN) {
	$mon_cmdargs = "-p";
    } else {
	$mon_cmdargs .= " " . "-p";
    }
141
}
142 143 144 145 146
if (defined($options{"i"})) {
    $initelab = 1;
    print "WARNING: -i only makes sense with -p\n"
	if (!$realplab);
}
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
#if (defined($options{"N"})) {
#    if ($options{"N"} eq "s") {
#	$stub_cmd = "/bin/sh /local/pelab/magent/auto-magent.sh";
#	$stub_cmdargs = ""
#	    if ($stub_cmdargs eq $UNKNOWN);
#    } elsif ($options{"N"} eq "m") {
#	$mon_cmd = "/bin/sh /local/pelab/dbmonitor/auto-dbmonitor.sh";
#	$mon_cmdargs = ""
#	    if ($mon_cmdargs eq $UNKNOWN);
#    } else {
#	die "Unknown -N type '", $options{"N"}, "'\n";
#    }
#}
#if (defined($options{"O"})) {
#    if ($options{"O"} eq "s") {
#	$stub_cmd = "/bin/sh /local/pelab/stub/auto-stub.sh";
#	$stub_cmdargs = ""
#	    if ($stub_cmdargs eq $UNKNOWN);
#    } elsif ($options{"O"} eq "m") {
#	$mon_cmd = "/bin/sh /local/pelab/monitor/auto-monitor.sh";
#	$mon_cmdargs = ""
#	    if ($mon_cmdargs eq $UNKNOWN);
#    } else {
#	die "Unknown -O type '", $options{"O"}, "'\n";
#    }
#}
173 174 175
if (defined($options{"t"})) {
    $sanityChecks = 1;
}
176

Mike Hibler's avatar
Mike Hibler committed
177 178 179
my $use_magent = ($stub_cmd =~ /magent/) ? 1 : 0;
my $use_simplemodel = ($mon_cmd =~ /dbmonitor/) ? 1 : 0;

180
#
181
# Let's help some poor bozos (Rob) get the right command line args
182 183 184 185 186
#
print "##### Stashing node list for experiment\n";
my @vnodes = split /\s+/,`$NODELIST -v -e $pid,$eid`;
my $hasfakeplab = grep /plab-\d+/, @vnodes;
my $hasrealplab = grep /planet-\d+/, @vnodes;
187
print "HFPL = $hasfakeplab HRPL = $hasrealplab RPL = $realplab\n";
188 189 190 191 192 193
if (!$hasfakeplab && $hasrealplab && !$realplab) {
    die "*** ERROR: Forgot to specify -p!\n";
} elsif ($hasfakeplab && !$hasrealplab && $realplab) {
    die "*** ERROR: Should not use -p!\n";
}

194
#
195
# Make sure any old stubs/monitors are dead
196
#
197
print "##### Stopping old stubs and monitors\n";
Mike Hibler's avatar
Mike Hibler committed
198 199 200 201 202 203 204 205
if ($realplab) {
    if (system "$TEVC -e $pid/$eid now planetstubs stop") {
	die "Error running tevc\n";
    }
} else {
    if (system "$TEVC -e $pid/$eid now plabstubs stop") {
	die "Error running tevc\n";
    }
206 207
}
if (system "$TEVC -e $pid/$eid now monitorgroup stop") {
208 209 210 211 212 213 214
    die "Error running tevc\n";
}

#
# Make sure link logs get cleaned up
#
print "##### Rolling link tracing files \n";
215
if (system "$TEVC -e $pid/$eid now planet-tracemon snapshot") {
216 217
    die "Error running tevc\n";
}
218
if (system "$TEVC -e $pid/$eid now planet-tracemon stop") {
219 220
    die "Error running tevc\n";
}
221

222 223 224 225 226 227 228 229 230
#
# Smack down host tcpdumps
#
print "##### Stopping host tcpdumps\n";
if (system "$TEVC -e $pid/$eid now tdhosts stop") {
    die "Error running tevc\n";
}


231 232 233
#
# Restart the server program

234 235 236 237 238 239 240 241
#
# Clean out the logs so that we can start fresh
#
print "##### Cleaning old logfiles (ignore 'No match' errors)\n";
if (system "$LOGHOLE -e $pid/$eid clean -f -r") {
    die "Error running loghole\n";
}

242 243 244 245
#
# Reset the links so that we remove any delay changes we might have previously
# made
#
246 247 248 249 250 251 252 253
print "##### Resetting all links in the experiment..";
print "clear..";
if (system "$TEVC -e $pid/$eid now elabc clear") {
    die "Error running tevc elabc clear\n";
}
print "reset\n";
if (system "$TEVC -w -e $pid/$eid now elabc reset") {
    die "Error running tevc elabc reset\n";
254 255
}

256 257 258 259 260 261 262 263 264
#
# XXX stash node list for experiment since nodes don't have node_list
# right now.  This is needed by the dbmonitor if it is in use.
#
print "##### Stashing node list for experiment\n";
if (system "$NODELIST -m -e $pid,$eid > /proj/$pid/exp/$eid/tmp/node_list") {
    warn "Could not create node list for dbmonitor; it will fail if used\n";
}

265 266 267 268 269
#
# Restart the event system so the 'create' event, necessary for pelab to work,
# gets fired again
#
print "##### Restarting the event system\n";
270
if (system "$EVENTSYS -e $pid,$eid replay") {
271 272
    die "Error controlling the event system\n";
}
273

274 275 276
print "##### Waiting for event system to start\n";
sleep(10);

Mike Hibler's avatar
Mike Hibler committed
277 278 279 280 281 282 283 284 285 286 287 288
#
# Fetch and store the initial conditions for plab nodes if desired.
# We both stash them in a file and set them directly.  The former is
# read by the monitor at start up to inform the stubs.  The latter is
# for backward compat with the old monitor/stubs that did not do it for
# themselves.
#
if ($realplab && $initelab && !$use_simplemodel) {
    print "##### Stashing initial path conditions for plab nodes\n";
    if (system "$pelabdir/$INITCOND -o /proj/$pid/exp/$eid/tmp/initial-conditions.txt $pid $eid") {
	warn "Could not acquire initial path conditions\n";
    }
289 290 291 292
#    print "##### Initializing Emulab link characteristics\n";
#    if (system "$pelabdir/$INITCOND -C $pid $eid") {
#	warn "Error initializing Emulab links with initial path conditions\n";
#    }
293 294 295
} else {
    # avoid old stale conditions
    unlink("/proj/$pid/exp/$eid/tmp/initial-conditions.txt");
296 297
}

298 299 300 301 302 303 304 305
print "##### Restarting servers\n";
if (system "$TEVC -e $pid/$eid now allservers stop") {
    die "Error running tevc\n";
}
if (system "$TEVC -e $pid/$eid now allservers start") {
    die "Error running tevc\n";
}

306 307 308 309
#
# Start link tracing
#
print "##### Starting link tracing\n";
310
if (system "$TEVC -e $pid/$eid now planet-tracemon start") {
311 312 313
    die "Error running tevc\n";
}

314 315 316 317 318 319 320 321
#
# Start host tcpdumps
#
print "##### Starting host tcpdumps\n";
if (system "$TEVC -e $pid/$eid now tdhosts start") {
    die "Error running tevc\n";
}

322 323 324
#
# Start up the stubs and monitors
#
325
#my $startarg = "";
326
print "##### Starting stubs";
327 328 329 330 331 332 333
#if ($initelab && $use_magent) {
#    $stub_cmdargs += " /proj/$pid/exp/$eid/tmp/initial-conditions.txt ";
#}
#if ($stub_cmdargs ne $UNKNOWN) {
#    print " with: '$stub_cmd $stub_cmdargs'";
#    $startarg = "'COMMAND=$stub_cmd $stub_cmdargs'";
#}
334
print "\n";
Mike Hibler's avatar
Mike Hibler committed
335
if ($realplab) {
336
    if (system "$TEVC -e $pid/$eid now planetstubs start") {
Mike Hibler's avatar
Mike Hibler committed
337 338 339
	die "Error running tevc\n";
    }
} else {
340
    if (system "$TEVC -e $pid/$eid now plabstubs start") {
Mike Hibler's avatar
Mike Hibler committed
341 342
	die "Error running tevc\n";
    }
343
}
344

345
#$startarg = "";
346
print "##### Starting monitors";
347 348 349 350
#if ($mon_cmdargs ne $UNKNOWN) {
#    print " with: '$mon_cmd $mon_cmdargs'";
#    $startarg = "'COMMAND=$mon_cmd $mon_cmdargs'";
#}
351
print "\n";
352
if (system "$TEVC -e $pid/$eid now monitorgroup start") {
353 354 355
    die "Error running tevc\n";
}

356
#print "##### Starting tcpdumps\n";
357 358 359 360
#if (system "$TEVC -e $pid/$eid now plabtcpdump start") {
#    die "Error running tevc\n";
#}

361 362 363 364 365
print "##### Starting tcpdumps for cross-correlation sanity checks\n";
if (system "$TEVC -e $pid/$eid now sanityCheckDumps start") {
    die "Error running tevc\n";
}

366
print "##### Done\n";
367 368 369 370 371 372 373

exit(0);

#
# Dig the command line out of the logs from the previous run
# (if there was one)
#
Mike Hibler's avatar
Mike Hibler committed
374
sub get_cmdargs($$)
375
{
Mike Hibler's avatar
Mike Hibler committed
376
    my ($ntype,$cmd) = @_;
377
    my $cmdargs = $UNKNOWN;
Mike Hibler's avatar
Mike Hibler committed
378
    my $lfile = ($ntype eq "plab") ? "plabstub" : "monitor";
379 380 381 382 383 384 385 386 387

    my @list = `/bin/ls /proj/$pid/exp/$eid/logs/$eid.*.zip`;
    return $cmdargs
	if ($?);
    chomp(@list);

    my $file = $list[$#list];
    return $cmdargs
	if (! -r $file);
Mike Hibler's avatar
Mike Hibler committed
388
    my @info = `unzip -p $file "${ntype}-1/local/logs/${lfile}-1.status"`;
389 390
    foreach (@info) {
	if (/^COMMAND=(.*)/) {
Mike Hibler's avatar
Mike Hibler committed
391 392
	    if ($1 ne $cmd) {
		$cmdargs = $1;
Mike Hibler's avatar
Mike Hibler committed
393 394 395 396 397 398 399 400

		# XXX filter out broken cmdlines (old bug in program_agent)
		$cmdargs =~ s/\s*}\s*$//;

		$cmdargs =~ s/^$cmd//;
		if ($cmdargs eq "") {
		    $cmdargs = $UNKNOWN;
		}
Mike Hibler's avatar
Mike Hibler committed
401
	    }
402 403 404 405 406 407
	    last;
	}
    }

    return $cmdargs;
}