tbprerun.in 14.2 KB
Newer Older
1
#!/usr/bin/perl -w
Leigh Stoller's avatar
Leigh Stoller committed
2
#
3
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
Leigh Stoller's avatar
Leigh Stoller committed
23
#
24
use strict;
25
use English;
26
use Getopt::Std;
27 28 29

# tbprerun

Leigh Stoller's avatar
Leigh Stoller committed
30 31 32 33 34
# This is the first program in the
# tbprerun/tbswapin/tbswapout/.../tbend sequences.  It's main purpose
# is to interpret the NS file and create the appropriate entries in
# virt_nodes and virt_lans.  After this script ends successfully the
# NS file is no longer necessary.
35 36 37
#
sub usage()
{
38
    print STDERR "Usage: $0 [-f] -e eid nsfile\n";
39 40
    exit(-1);
}
41 42
my $optlist = "fze:";
my $pideid;
43 44 45
my $force   = 0;
my $zeemode = 0;
my $zeeopt  = "";
46

47 48 49 50
#
# Configure variables
#
my $TBROOT   = "@prefix@";
Leigh Stoller's avatar
Leigh Stoller committed
51
my $TESTMODE = @TESTMODE@;
52 53
my $TBLOGS   = "@TBLOGSEMAIL@";
my $OPSDBSUPPORT= @OPSDBSUPPORT@;
54
my $NSVERIFY = @NSVERIFY@;
55
my $vtopgen  = "$TBROOT/bin/vtopgen";
Leigh Stoller's avatar
Leigh Stoller committed
56 57 58 59
my $wrapper  = "$TBROOT/libexec/assign_wrapper";
my $batchexp = "$TBROOT/bin/batchexp";
my $template = "$TBROOT/lib/ns2ir/elabinelab.ns";
my $endexp   = "$TBROOT/bin/endexp";
60
my $dbcontrol= "$TBROOT/sbin/opsdb_control";
61
my $IMAGESETUP= "$TBROOT/sbin/image_setup";
62

63
# Untaint the path
64
$ENV{'PATH'} = "/usr/bin:/bin:$TBROOT/libexec:$TBROOT/libexec/ns2ir" . 
65
    ":$TBROOT/libexec/vis:$TBROOT/sbin:$TBROOT/bin";
Leigh Stoller's avatar
Leigh Stoller committed
66

67 68 69
#
# Testbed Support libraries
#
70
use lib "@prefix@/lib";
71 72
use libdb;
use libtestbed;
73
use libtblog;
74
use User;
75
use Experiment;
76
use Template;
77
use OSImage;
78

79 80 81 82
#
# Turn off line buffering on output
#
$| = 1;
83

84
#
85 86
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
87
#
88
my %options = ();
89
if (! getopts($optlist, \%options)) {
Leigh Stoller's avatar
Leigh Stoller committed
90 91
    usage();
}
92
if (defined($options{"f"})) {
93 94
    $force = 1;
}
95 96 97
if (defined($options{"e"})) {
    $pideid = $options{"e"};
}
98 99 100 101
if (defined($options{"z"})) {
    $zeemode = 1;
    $zeeopt  = "-p";
}
102
if (!defined($pideid) || @ARGV != 1) {
103
    usage();
104
}
105
my ($nsfile) = @ARGV;
106 107

if (! -r $nsfile) {
108
    tbdie("NS File '$nsfile' does not exist!");
109 110
}

111 112 113 114 115 116 117 118
#
# Verify user and get his DB uid and other info for later.
#
my $this_user = User->ThisUser();
if (! defined($this_user)) {
    tbdie("You ($UID) do not exist!");
}

119
# Slowly convert to using Experiment module.
120
my $experiment = Experiment->Lookup($pideid);
121
if (!defined($experiment)) {
122
    tbdie("Could not lookup experiment object for $pideid!");
123
}
124 125 126 127 128 129
# Need these below.
my $pid      = $experiment->pid();
my $eid      = $experiment->eid();
my $state    = $experiment->state();
my $gid      = $experiment->gid();
my $exptidx  = $experiment->idx();
130 131 132 133 134 135 136 137
my $instance;

if ($experiment->IsInstance()) {
    $instance = Template::Instance->LookupByExptidx($experiment->idx());
    if (!defined($instance)) {
	tbdie("Could not lookup instance object for $exptidx!");
    }
}
138 139 140

print "Beginning pre run for $pid/$eid. " . TBTimeStamp() . "\n";
TBDebugTimeStamp("tbprerun started");
141

142 143 144 145 146 147
#
# These are the valid states for running this script.
#
if (!$force &&
    !($state eq EXPTSTATE_PRERUN || $state eq EXPTSTATE_MODIFY_PARSE ||
      $state eq EXPTSTATE_MODIFY_REPARSE)) {
148
    tbdie("Experiment is not in the proper state: $state");
149 150
}

151
#
152 153
# Cleanup if something goes wrong.
#
Leigh Stoller's avatar
Leigh Stoller committed
154
sub cleanup {
155 156
    print STDERR "Cleaning up after errors.\n";
    if ($state eq EXPTSTATE_PRERUN) {
157 158 159
	# Must kill the prerender process before we remove virt state.
	print "Killing the renderer.\n";
	system("prerender -r $pid $eid");
160
    }
161
    print "Removing experiment state.\n";
162
    $experiment->RemoveVirtualState();
163 164
}

Leigh Stoller's avatar
Leigh Stoller committed
165 166 167
#
# Fatal error.
#
168
sub fatal($;$)
Leigh Stoller's avatar
Leigh Stoller committed
169
{
170 171
    my $parms = {};
    $parms = shift if ref $_[0] eq 'HASH';
Leigh Stoller's avatar
Leigh Stoller committed
172 173 174
    my ($mesg) = @_;
    
    cleanup();
175
    tbdie($parms, $mesg);
Leigh Stoller's avatar
Leigh Stoller committed
176 177
}

178 179 180 181
# Must kill any prerender process first!
system("prerender -r $pid $eid");

# Kill old virtual state.
182 183
$experiment->RemoveVirtualState() == 0 or
    tbdie("Could not remove virtual state!");
184

Leigh Stoller's avatar
Leigh Stoller committed
185 186
# This setups virt_nodes, virt_names including all IP address calculation
# and tb-* handling.
187
print "Running parser ... " . TBTimeStamp() . "\n";
188
TBDebugTimeStamp("parser started");
189
if (system("parse-ns $zeeopt $pid $gid $eid $nsfile")) {
190 191 192
    fatal({type => 'secondary', severity => SEV_SECONDARY,
	   error => ['ns_parse_failed']},
	  "Parsing failed!");
193
}
194
TBDebugTimeStamp("parser finished");
195
print "Parser done! " . TBTimeStamp() . "\n";
196

197 198
$experiment->Refresh();

199 200 201 202
#
# XXX Until link agent runs on linux.
#
my $events_result =
203
    DBQueryFatal("select ev.pid,ev.eid,vl.vnode,vl.vname,vn.osname,ov.OS ".
204 205 206 207 208 209 210
		 "  from eventlist as ev ".
		 "left join event_objecttypes as ev_ob on ".
		 "  ev.objecttype=ev_ob.idx ".
		 "left join virt_lans as vl on vl.vname=ev.vname and ".
		 "  vl.pid=ev.pid and vl.eid=ev.eid ".
		 "left join virt_nodes as vn on vn.pid=ev.pid and ".
		 "  vn.eid=ev.eid and vn.vname=vl.vnode ".
211 212
		 "left join os_info as o on ".
		 "  o.osname=substring_index(vn.osname, ':', 1) and".
213
		 "  (o.pid=ev.pid or o.pid='emulab-ops') ".
214 215
		 "left join os_info_versions as ov on ".
		 "     ov.osid=o.osid and ov.vers=o.version ".
216 217 218 219 220
		 "left join experiments as e on e.pid=ev.pid and ".
		 "  e.eid=ev.eid ".
		 "where ev.pid='$pid' and ev.eid='$eid' and ".
		 "  (vl.uselinkdelay!=0 or e.uselinkdelays!=0 or ".
		 "   e.forcelinkdelays!=0) and ev_ob.type='LINK' and ".
221
		 "  (ov.os is NULL or ov.os='Linux' or ov.os='Fedora')");
222 223 224 225
if ($events_result->num_rows) {
    fatal("Oops, cannot send static events to linkdelay agents on Linux!");
}

226 227 228 229 230 231 232 233
#
# Prelim check for deprecated images.
#
if ($experiment->CheckForDeprecatedImages($this_user, 0)) {
    cleanup();
    exit(-1);
}

234 235 236 237 238 239 240
#
# Only admins can set the sharing mode on nodes.
#
my $query_result =
    DBQueryFatal("select sharing_mode from virt_nodes ".
		 "where pid='$pid' and eid='$eid' and ".
		 "      sharing_mode is not null");
241 242
if ($query_result->numrows &&
    !($this_user->IsAdmin() || $this_user->uid eq "elabman")) {
243 244 245
    fatal("Only testbed admininstrators can set the sharing mode on nodes");
}

246 247 248
#
# Need to know if wrapping an experiment.
#
249 250 251 252 253 254 255 256 257
# If we are, rerun the parser so that we can pass in the proper info
# this time around. This is a side effect of allowing the elabinelab
# bit, and the elabinelab eid, to be specified in the NS file. If we
# did this on the command line, then we could skip this step, but I
# think ease of use is better if it can be in the NS file. Remember,
# the user can clear the elabinelab_eid field, so must rerun to clear
# out the nodes. What if they clear the elabinelab bit too? Ick, not
# going to worry about that.
#
258
if ($experiment->elabinelab()) {
259
    # Inner experiment better exist.
260 261
    if (defined($experiment->elabinelab_eid()) &&
	$experiment->elabinelab_eid() ne "") {
262 263 264 265 266 267 268 269 270 271 272 273
	my $inner_experiment =
	    Experiment->Lookup($pid, $experiment->elabinelab_eid());

	fatal("Inner experiment does not exist!")
	    if (!defined($inner_experiment));

	# Wrong place to do this?
	my %args = ("elabinelab_exptidx" => $inner_experiment->idx());
	
	$experiment->Update(\%args) == 0 or
	    fatal("Could not update elabinelab_exptidx in $experiment");
    }
274 275
    
    # Again, kill virtual state.
276 277
    $experiment->RemoveVirtualState() == 0 or
	tbdie("Could not remove virtual state!");
278 279 280 281

    print "Re-Running parser ... " . TBTimeStamp() . "\n";
    TBDebugTimeStamp("parser started");
    if (system("parse-ns $pid $gid $eid $nsfile")) {
282 283 284
	fatal({type => 'secondary', severity => SEV_SECONDARY,
	       error => ['ns_parse_failed']},
	      "Parsing failed!");
285 286 287 288 289
    }
    TBDebugTimeStamp("parser finished");
    print "Parser done! " . TBTimeStamp() . "\n";
}

290 291 292
#
# Put the nsfile into the DB, now that we know it parses okay.
#
293 294
$experiment->SetNSFile($nsfile) == 0 or
    fatal("Error storing the NS file into the database!");
295

296 297 298 299
#
# In update mode, do not start the renderer until later. If update fails we
# want to try to restore old render info rather then rerunning. 
# 
300
if ($state eq EXPTSTATE_PRERUN && !$zeemode) {
301 302 303 304
    TBDebugTimeStamp("prerender started in background");
    print "Precomputing visualization ...\n";
    system("prerender -t $pid $eid");
}
305

306
#
307
# See if using the new ipassign.
308
#
309 310
if ($experiment->use_ipassign()) {
    my $ipassign_args  = $experiment->ipassign_args();
311

312
    if (! defined($ipassign_args)) {
313 314
	$ipassign_args = "";
    }
315 316 317
    TBDebugTimeStamp("ipassign_wrapper started");
    print "Doing IP assignment ...\n";
    
318
    if (system("ipassign_wrapper $ipassign_args $pid $eid")) {
Leigh Stoller's avatar
Leigh Stoller committed
319
	fatal("ipassign_wrapper failed!");
320
    }
321
}
322

323
#
324
# Fire up the route calculator. 
325 326 327 328
#
if (!$zeemode) {
    TBDebugTimeStamp("static route calculator started");
    print "Setting up static routes (if requested) ... \n";
329

330
    if (system("staticroutes $pid $eid")) {
331 332 333
	fatal({type => 'secondary', severity => SEV_SECONDARY,
	       error => ['static_routes_failed']},
	      "Static route calculation failed!");
334
    }
335 336
}

337 338 339 340
#
# Generate a topo map that is used by the remote nodes to create the
# routes (ddijk) and the /etc/hosts file.
#
341 342 343
TBDebugTimeStamp("gentopofile started");
print "Generating topomap ...\n";

344
if (system("gentopofile $pid $eid")) {
Leigh Stoller's avatar
Leigh Stoller committed
345 346 347
    fatal("gentopofile failed!");
}

348 349 350 351 352 353 354 355
if ($OPSDBSUPPORT) {
    TBDebugTimeStamp("opsdb_control started");
    
    if (system("$dbcontrol addexpdb $pid $eid")) {
	fatal("$dbcontrol addexpdb failed!");
    }
}

356 357
# Lets not do this anymore. 
if (0 && !$experiment->elabinelab()) {
358
    TBDebugTimeStamp("verify-ns started");
359 360
    if ($NSVERIFY) {
	print "Verifying parse ...\n";
361
    
362 363 364 365 366 367 368 369 370 371 372 373 374
	if (system("verify-ns $pid $gid $eid $nsfile")) {
	    if (1) {
		SENDMAIL($TBLOGS,
			 "NS Verify failure for $pid/$eid",
			 "'verify-ns' failed for $pid/$eid\n".
			 "\nIf it is not a user error nstb_compat.tcl is ".
			 "probably out of date.\n");
	    }
	    else {
		fatal({type => 'secondary', severity => SEV_SECONDARY,
		       error => ['ns_verify_failed']},
		      "verify-ns failed!");
	    }
375
	}
376 377
    } else {
	print "NSVERIFY not configured, skipping ...\n";
378 379 380
    }
}

381 382 383 384 385 386 387 388
#
# Load up external image references.
#
system("$IMAGESETUP $pid,$eid");
if ($?) {
    fatal("Could not setup external image references");
}

Leigh Stoller's avatar
Leigh Stoller committed
389 390 391 392
#
# Do an assign_prerun to set the min/max nodes. Generates a top file too.
# This is the only DB state that is modified during a top only run.
#
393
if (!$zeemode) {
394
    my $cmd = "$vtopgen -p $pid $eid";
395

396
    TBDebugTimeStamp("assign prerun started");
397 398 399
    print "Doing a pre-assign: '$cmd' ...\n";

    if (system($cmd)) {
400
	fatal({type => 'secondary', severity => SEV_SECONDARY,
401
	       error => ['vtopgen', undef]},
402
	      "assign prerun failed!");
403
    }
Leigh Stoller's avatar
Leigh Stoller committed
404 405
}

406 407 408 409 410 411 412
#
# If wrapping an experiment, copy over type/fixnode info to wrapper.
# We already made sure the experiment was created with enough nodes (see
# the parser) but now we have to make sure that the type/fixnode stuff will
# correspond to what the wrapped experiment really wants. It is a lot easier
# to do this here then in the parser.
#
413 414 415
if ($experiment->elabinelab() &&
    defined($experiment->elabinelab_eid()) &&
    $experiment->elabinelab_eid() ne "") {
416 417
    my $outer_result;
    my $inner_result;
418 419 420 421 422
    my $inner_experiment =
	Experiment->Lookup($pid, $experiment->elabinelab_eid());

    fatal("Inner experiment does not exist!")
	if (!defined($inner_experiment));
423 424

    if (! ($inner_result =
425 426
	   $inner_experiment->TableLookUp("virt_nodes", "type,fixed"))) {
	fatal("Could not get virt_node info for $inner_experiment!");
427 428
    }
    if (! ($outer_result =
429 430 431
	   $experiment->TableLookUp("virt_nodes", "vname",
				    "inner_elab_role='node'"))) {
	fatal("Could not get virt_node info for $experiment!");
432 433 434 435 436 437 438 439 440 441 442 443 444 445
    }

    #
    # Move over info for each real node in the inner experiment. Delay
    # nodes will be extra nodes in the outer experiment, but those do
    # not currently get type/fix info since the the user has no way to
    # do that in the NS file.
    # 
    while (my ($type,$fixed) = $inner_result->fetchrow_array()) {
	my ($vname) = $outer_result->fetchrow_array();

	fatal("Not enough nodes in outer experiment!")
	    if (!defined($vname));

446 447
	$experiment->TableUpdate("virt_nodes",
				 "type='$type',fixed='$fixed'",
448
				 "vname='$vname'") == 0
449
	    or fatal("Could not update virt_node info for $experiment!");
450 451 452
    }
}

Leigh Stoller's avatar
Leigh Stoller committed
453 454 455 456 457 458 459 460 461
#
# This stats stuff is not done in testmode.
# 
if (! $TESTMODE) {
    #
    # Deal with ElabInElab stuff, which is updated after the parse and prerun.
    # Want to move this to the stats records. Ditto for the security stuff,
    # which was handled above.
    #
462 463 464 465 466 467 468 469 470 471
    my %sets = ();

    $sets{"elabinelab"} = 1
	if ($experiment->elabinelab());
    $sets{"security_level"} = $experiment->security_level()
	if ($experiment->security_level());
    $sets{"elabinelab_exptidx"} = $experiment->elabinelab_exptidx()
	if (defined($experiment->elabinelab_exptidx()));

    if (keys(%sets)) {
472
	$experiment->TableUpdate("experiment_stats", \%sets) == 0 or
473
	    fatal("Could not update experiment_stats info for $experiment!");
Leigh Stoller's avatar
Leigh Stoller committed
474
    }
475 476
}

477
# Setup env variables.
478 479 480 481 482 483 484 485
if ($experiment->IsInstance()) {
    $instance->InitializeEnvVariables() == 0
	or fatal("Could not initialize environment strings variables");
}
else {
    $experiment->InitializeEnvVariables() == 0
	or fatal("Could not initialize environment strings variables");
}
486

487 488 489 490
print "Writing environment strings ...\n";
$experiment->WriteEnvVariables() == 0
    or fatal("Could not write environment strings for program agents");

491 492 493 494
print "Setting up additional program agent support ...\n";
$experiment->SetupProgramAgents() == 0
    or fatal("Could not setup program agent support");

495 496 497 498
print "Setting up additional network agent support ...\n";
$experiment->SetupNetworkAgents() == 0
    or fatal("Could not setup network agent support");

499 500 501 502
print "Writing program agent info ...\n";
$experiment->WriteProgramAgents() == 0
    or fatal("Could not write program agent info");

503
TBDebugTimeStamp("tbprerun finished");
504 505
print "Pre run finished. " . TBTimeStamp() . "\n";
exit(0);
506