tbprerun.in 13.2 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
5 6
# All rights reserved.
#
7
use strict;
8
use English;
9
use Getopt::Std;
10 11 12

# tbprerun

Leigh B. Stoller's avatar
Leigh B. Stoller committed
13 14 15 16 17
# This is the first program in the
# tbprerun/tbswapin/tbswapout/.../tbend sequences.  It's main purpose
# is to interpret the NS file and create the appropriate entries in
# virt_nodes and virt_lans.  After this script ends successfully the
# NS file is no longer necessary.
18 19 20
#
sub usage()
{
21
    print STDERR "Usage: $0 [-f] -e eid nsfile\n";
22 23
    exit(-1);
}
24 25
my $optlist = "fze:";
my $pideid;
26 27 28
my $force   = 0;
my $zeemode = 0;
my $zeeopt  = "";
29

30 31 32 33
#
# Configure variables
#
my $TBROOT   = "@prefix@";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
34
my $TESTMODE = @TESTMODE@;
35 36
my $TBLOGS   = "@TBLOGSEMAIL@";
my $OPSDBSUPPORT= @OPSDBSUPPORT@;
Mike Hibler's avatar
Mike Hibler committed
37
my $NSVERIFY = @NSVERIFY@;
38
my $vtopgen  = "$TBROOT/bin/vtopgen";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
39 40 41 42
my $wrapper  = "$TBROOT/libexec/assign_wrapper";
my $batchexp = "$TBROOT/bin/batchexp";
my $template = "$TBROOT/lib/ns2ir/elabinelab.ns";
my $endexp   = "$TBROOT/bin/endexp";
43
my $dbcontrol= "$TBROOT/sbin/opsdb_control";
44
my $IMAGESETUP= "$TBROOT/sbin/image_setup";
45

46
# Untaint the path
47
$ENV{'PATH'} = "/usr/bin:/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
48
    ":$TBROOT/libexec/vis:$TBROOT/sbin:$TBROOT/bin";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
49

50 51 52
#
# Testbed Support libraries
#
53
use lib "@prefix@/lib";
54 55
use libdb;
use libtestbed;
56
use libtblog;
57
use User;
58
use Experiment;
59
use Template;
60

61 62 63 64
#
# Turn off line buffering on output
#
$| = 1;
65

66
#
67 68
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
69
#
70
my %options = ();
71
if (! getopts($optlist, \%options)) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
72 73
    usage();
}
74
if (defined($options{"f"})) {
75 76
    $force = 1;
}
77 78 79
if (defined($options{"e"})) {
    $pideid = $options{"e"};
}
80 81 82 83
if (defined($options{"z"})) {
    $zeemode = 1;
    $zeeopt  = "-p";
}
84
if (!defined($pideid) || @ARGV != 1) {
85
    usage();
86
}
87
my ($nsfile) = @ARGV;
88 89

if (! -r $nsfile) {
90
    tbdie("NS File '$nsfile' does not exist!");
91 92
}

93 94 95 96 97 98 99 100
#
# Verify user and get his DB uid and other info for later.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    tbdie("You ($UID) do not exist!");
}

101
# Slowly convert to using Experiment module.
102
my $experiment = Experiment->Lookup($pideid);
103
if (!defined($experiment)) {
104
    tbdie("Could not lookup experiment object for $pideid!");
105
}
106 107 108 109 110 111
# Need these below.
my $pid      = $experiment->pid();
my $eid      = $experiment->eid();
my $state    = $experiment->state();
my $gid      = $experiment->gid();
my $exptidx  = $experiment->idx();
112 113 114 115 116 117 118 119
my $instance;

if ($experiment->IsInstance()) {
    $instance = Template::Instance->LookupByExptidx($experiment->idx());
    if (!defined($instance)) {
	tbdie("Could not lookup instance object for $exptidx!");
    }
}
120 121 122

print "Beginning pre run for $pid/$eid. " . TBTimeStamp() . "\n";
TBDebugTimeStamp("tbprerun started");
123

124 125 126 127 128 129
#
# These are the valid states for running this script.
#
if (!$force &&
    !($state eq EXPTSTATE_PRERUN || $state eq EXPTSTATE_MODIFY_PARSE ||
      $state eq EXPTSTATE_MODIFY_REPARSE)) {
130
    tbdie("Experiment is not in the proper state: $state");
131 132
}

133
#
Chad Barb's avatar
Chad Barb committed
134 135
# Cleanup if something goes wrong.
#
Leigh B. Stoller's avatar
Leigh B. Stoller committed
136
sub cleanup {
137 138
    print STDERR "Cleaning up after errors.\n";
    if ($state eq EXPTSTATE_PRERUN) {
139 140 141
	# Must kill the prerender process before we remove virt state.
	print "Killing the renderer.\n";
	system("prerender -r $pid $eid");
142
    }
143
    print "Removing experiment state.\n";
144
    $experiment->RemoveVirtualState();
145 146
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
147 148 149
#
# Fatal error.
#
150
sub fatal($;$)
Leigh B. Stoller's avatar
Leigh B. Stoller committed
151
{
152 153
    my $parms = {};
    $parms = shift if ref $_[0] eq 'HASH';
Leigh B. Stoller's avatar
Leigh B. Stoller committed
154 155 156
    my ($mesg) = @_;
    
    cleanup();
157
    tbdie($parms, $mesg);
Leigh B. Stoller's avatar
Leigh B. Stoller committed
158 159
}

160 161 162 163
# Must kill any prerender process first!
system("prerender -r $pid $eid");

# Kill old virtual state.
164 165
$experiment->RemoveVirtualState() == 0 or
    tbdie("Could not remove virtual state!");
166

Leigh B. Stoller's avatar
Leigh B. Stoller committed
167 168
# This setups virt_nodes, virt_names including all IP address calculation
# and tb-* handling.
169
print "Running parser ... " . TBTimeStamp() . "\n";
Robert Ricci's avatar
Robert Ricci committed
170
TBDebugTimeStamp("parser started");
171
if (system("parse-ns $zeeopt $pid $gid $eid $nsfile")) {
172 173 174
    fatal({type => 'secondary', severity => SEV_SECONDARY,
	   error => ['ns_parse_failed']},
	  "Parsing failed!");
175
}
Robert Ricci's avatar
Robert Ricci committed
176
TBDebugTimeStamp("parser finished");
177
print "Parser done! " . TBTimeStamp() . "\n";
178

179 180
$experiment->Refresh();

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
#
# XXX Until link agent runs on linux.
#
my $events_result =
    DBQueryFatal("select ev.pid,ev.eid,vl.vnode,vl.vname,vn.osname,o.OS ".
		 "  from eventlist as ev ".
		 "left join event_objecttypes as ev_ob on ".
		 "  ev.objecttype=ev_ob.idx ".
		 "left join virt_lans as vl on vl.vname=ev.vname and ".
		 "  vl.pid=ev.pid and vl.eid=ev.eid ".
		 "left join virt_nodes as vn on vn.pid=ev.pid and ".
		 "  vn.eid=ev.eid and vn.vname=vl.vnode ".
		 "left join os_info as o on o.osname=vn.osname and".
		 "  (o.pid=ev.pid or o.pid='emulab-ops') ".
		 "left join experiments as e on e.pid=ev.pid and ".
		 "  e.eid=ev.eid ".
		 "where ev.pid='$pid' and ev.eid='$eid' and ".
		 "  (vl.uselinkdelay!=0 or e.uselinkdelays!=0 or ".
		 "   e.forcelinkdelays!=0) and ev_ob.type='LINK' and ".
200
		 "  (o.os is NULL or o.os='Linux' or o.os='Fedora')");
201 202 203 204
if ($events_result->num_rows) {
    fatal("Oops, cannot send static events to linkdelay agents on Linux!");
}

205 206 207 208 209 210 211
#
# Only admins can set the sharing mode on nodes.
#
my $query_result =
    DBQueryFatal("select sharing_mode from virt_nodes ".
		 "where pid='$pid' and eid='$eid' and ".
		 "      sharing_mode is not null");
212 213
if ($query_result->numrows &&
    !($this_user->IsAdmin() || $this_user->uid eq "elabman")) {
214 215 216
    fatal("Only testbed admininstrators can set the sharing mode on nodes");
}

217 218 219
#
# Need to know if wrapping an experiment.
#
220 221 222 223 224 225 226 227 228
# If we are, rerun the parser so that we can pass in the proper info
# this time around. This is a side effect of allowing the elabinelab
# bit, and the elabinelab eid, to be specified in the NS file. If we
# did this on the command line, then we could skip this step, but I
# think ease of use is better if it can be in the NS file. Remember,
# the user can clear the elabinelab_eid field, so must rerun to clear
# out the nodes. What if they clear the elabinelab bit too? Ick, not
# going to worry about that.
#
229
if ($experiment->elabinelab()) {
230
    # Inner experiment better exist.
231 232
    if (defined($experiment->elabinelab_eid()) &&
	$experiment->elabinelab_eid() ne "") {
233 234 235 236 237 238 239 240 241 242 243 244
	my $inner_experiment =
	    Experiment->Lookup($pid, $experiment->elabinelab_eid());

	fatal("Inner experiment does not exist!")
	    if (!defined($inner_experiment));

	# Wrong place to do this?
	my %args = ("elabinelab_exptidx" => $inner_experiment->idx());
	
	$experiment->Update(\%args) == 0 or
	    fatal("Could not update elabinelab_exptidx in $experiment");
    }
245 246
    
    # Again, kill virtual state.
247 248
    $experiment->RemoveVirtualState() == 0 or
	tbdie("Could not remove virtual state!");
249 250 251 252

    print "Re-Running parser ... " . TBTimeStamp() . "\n";
    TBDebugTimeStamp("parser started");
    if (system("parse-ns $pid $gid $eid $nsfile")) {
253 254 255
	fatal({type => 'secondary', severity => SEV_SECONDARY,
	       error => ['ns_parse_failed']},
	      "Parsing failed!");
256 257 258 259 260
    }
    TBDebugTimeStamp("parser finished");
    print "Parser done! " . TBTimeStamp() . "\n";
}

261 262 263
#
# Put the nsfile into the DB, now that we know it parses okay.
#
264 265
$experiment->SetNSFile($nsfile) == 0 or
    fatal("Error storing the NS file into the database!");
266

267 268 269 270
#
# In update mode, do not start the renderer until later. If update fails we
# want to try to restore old render info rather then rerunning. 
# 
271
if ($state eq EXPTSTATE_PRERUN && !$zeemode) {
272 273 274 275
    TBDebugTimeStamp("prerender started in background");
    print "Precomputing visualization ...\n";
    system("prerender -t $pid $eid");
}
276

277
#
278
# See if using the new ipassign.
279
#
280 281
if ($experiment->use_ipassign()) {
    my $ipassign_args  = $experiment->ipassign_args();
282

283
    if (! defined($ipassign_args)) {
284 285
	$ipassign_args = "";
    }
286 287 288
    TBDebugTimeStamp("ipassign_wrapper started");
    print "Doing IP assignment ...\n";
    
289
    if (system("ipassign_wrapper $ipassign_args $pid $eid")) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
290
	fatal("ipassign_wrapper failed!");
291
    }
292
}
293

294
#
295
# Fire up the route calculator. 
296 297 298 299
#
if (!$zeemode) {
    TBDebugTimeStamp("static route calculator started");
    print "Setting up static routes (if requested) ... \n";
300

301
    if (system("staticroutes $pid $eid")) {
302 303 304
	fatal({type => 'secondary', severity => SEV_SECONDARY,
	       error => ['static_routes_failed']},
	      "Static route calculation failed!");
305
    }
306 307
}

308 309 310 311
#
# Generate a topo map that is used by the remote nodes to create the
# routes (ddijk) and the /etc/hosts file.
#
312 313 314
TBDebugTimeStamp("gentopofile started");
print "Generating topomap ...\n";

315
if (system("gentopofile $pid $eid")) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
316 317 318
    fatal("gentopofile failed!");
}

319 320 321 322 323 324 325 326
if ($OPSDBSUPPORT) {
    TBDebugTimeStamp("opsdb_control started");
    
    if (system("$dbcontrol addexpdb $pid $eid")) {
	fatal("$dbcontrol addexpdb failed!");
    }
}

327
if (!$experiment->elabinelab()) {
328
    TBDebugTimeStamp("verify-ns started");
Mike Hibler's avatar
Mike Hibler committed
329 330
    if ($NSVERIFY) {
	print "Verifying parse ...\n";
331
    
Mike Hibler's avatar
Mike Hibler committed
332 333 334 335 336 337 338 339 340 341 342 343 344
	if (system("verify-ns $pid $gid $eid $nsfile")) {
	    if (1) {
		SENDMAIL($TBLOGS,
			 "NS Verify failure for $pid/$eid",
			 "'verify-ns' failed for $pid/$eid\n".
			 "\nIf it is not a user error nstb_compat.tcl is ".
			 "probably out of date.\n");
	    }
	    else {
		fatal({type => 'secondary', severity => SEV_SECONDARY,
		       error => ['ns_verify_failed']},
		      "verify-ns failed!");
	    }
345
	}
Mike Hibler's avatar
Mike Hibler committed
346 347
    } else {
	print "NSVERIFY not configured, skipping ...\n";
348 349 350
    }
}

351 352 353 354 355 356 357 358
#
# Load up external image references.
#
system("$IMAGESETUP $pid,$eid");
if ($?) {
    fatal("Could not setup external image references");
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
359 360 361 362
#
# Do an assign_prerun to set the min/max nodes. Generates a top file too.
# This is the only DB state that is modified during a top only run.
#
363
if (!$zeemode) {
364
    my $cmd = "$vtopgen -p $pid $eid";
365

366
    TBDebugTimeStamp("assign prerun started");
367 368 369
    print "Doing a pre-assign: '$cmd' ...\n";

    if (system($cmd)) {
370
	fatal({type => 'secondary', severity => SEV_SECONDARY,
371
	       error => ['vtopgen', undef]},
372
	      "assign prerun failed!");
373
    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
374 375
}

376 377 378 379 380 381 382
#
# If wrapping an experiment, copy over type/fixnode info to wrapper.
# We already made sure the experiment was created with enough nodes (see
# the parser) but now we have to make sure that the type/fixnode stuff will
# correspond to what the wrapped experiment really wants. It is a lot easier
# to do this here then in the parser.
#
383 384 385
if ($experiment->elabinelab() &&
    defined($experiment->elabinelab_eid()) &&
    $experiment->elabinelab_eid() ne "") {
386 387
    my $outer_result;
    my $inner_result;
388 389 390 391 392
    my $inner_experiment =
	Experiment->Lookup($pid, $experiment->elabinelab_eid());

    fatal("Inner experiment does not exist!")
	if (!defined($inner_experiment));
393 394

    if (! ($inner_result =
395 396
	   $inner_experiment->TableLookUp("virt_nodes", "type,fixed"))) {
	fatal("Could not get virt_node info for $inner_experiment!");
397 398
    }
    if (! ($outer_result =
399 400 401
	   $experiment->TableLookUp("virt_nodes", "vname",
				    "inner_elab_role='node'"))) {
	fatal("Could not get virt_node info for $experiment!");
402 403 404 405 406 407 408 409 410 411 412 413 414 415
    }

    #
    # Move over info for each real node in the inner experiment. Delay
    # nodes will be extra nodes in the outer experiment, but those do
    # not currently get type/fix info since the the user has no way to
    # do that in the NS file.
    # 
    while (my ($type,$fixed) = $inner_result->fetchrow_array()) {
	my ($vname) = $outer_result->fetchrow_array();

	fatal("Not enough nodes in outer experiment!")
	    if (!defined($vname));

416 417
	$experiment->TableUpdate("virt_nodes",
				 "type='$type',fixed='$fixed'",
418
				 "vname='$vname'") == 0
419
	    or fatal("Could not update virt_node info for $experiment!");
420 421 422
    }
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
423 424 425 426 427 428 429 430 431
#
# This stats stuff is not done in testmode.
# 
if (! $TESTMODE) {
    #
    # Deal with ElabInElab stuff, which is updated after the parse and prerun.
    # Want to move this to the stats records. Ditto for the security stuff,
    # which was handled above.
    #
432 433 434 435 436 437 438 439 440 441
    my %sets = ();

    $sets{"elabinelab"} = 1
	if ($experiment->elabinelab());
    $sets{"security_level"} = $experiment->security_level()
	if ($experiment->security_level());
    $sets{"elabinelab_exptidx"} = $experiment->elabinelab_exptidx()
	if (defined($experiment->elabinelab_exptidx()));

    if (keys(%sets)) {
442
	$experiment->TableUpdate("experiment_stats", \%sets) == 0 or
443
	    fatal("Could not update experiment_stats info for $experiment!");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
444
    }
445 446
}

447
# Setup env variables.
448 449 450 451 452 453 454 455
if ($experiment->IsInstance()) {
    $instance->InitializeEnvVariables() == 0
	or fatal("Could not initialize environment strings variables");
}
else {
    $experiment->InitializeEnvVariables() == 0
	or fatal("Could not initialize environment strings variables");
}
456

457 458 459 460
print "Writing environment strings ...\n";
$experiment->WriteEnvVariables() == 0
    or fatal("Could not write environment strings for program agents");

461 462 463 464
print "Setting up additional program agent support ...\n";
$experiment->SetupProgramAgents() == 0
    or fatal("Could not setup program agent support");

465 466 467 468
print "Setting up additional network agent support ...\n";
$experiment->SetupNetworkAgents() == 0
    or fatal("Could not setup network agent support");

469 470 471 472
print "Writing program agent info ...\n";
$experiment->WriteProgramAgents() == 0
    or fatal("Could not write program agent info");

Robert Ricci's avatar
Robert Ricci committed
473
TBDebugTimeStamp("tbprerun finished");
474 475
print "Pre run finished. " . TBTimeStamp() . "\n";
exit(0);
476