elabinelab.in 38.9 KB
Newer Older
1
2
3
#!/usr/bin/perl -wT
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2004-2005 University of Utah and the Flux Group.
5
6
7
8
9
10
11
12
# All rights reserved.
#
# TODO: ntpinfo table.
#       Current source directory? From where?
#
use English;
use Getopt::Std;

13
14
15
16
17
# Load the Testbed support stuff.
use lib "@prefix@/lib";
use libdb;
use libtestbed;

18
19
20
21
22
#
# Do things necessary for setting up inner elab experiment. 
#
sub usage()
{
23
    print STDOUT "Usage: elabinelab [-d] [-g] [-u] pid eid\n";
24
    print STDOUT "       elabinelab [-d] [-k | -f] pid eid\n";
25
    print STDOUT "       elabinelab [-d] -r pid eid [node ...]\n";
26
27
28
 
    exit(-1);
}
29
my $optlist  = "dgkfur";
30
my $debug    = 1;
31
my $killmode = 0;
32
my $fwboot   = 0;
33
my $dbgooonly= 0;
34
35
my $update   = 0;
my $remove   = 0;
36

37
38
sub DumpDBGoo();

39
40
41
42
43
44
45
46
47
48
#
# Configure variables
#
my $TB		= "@prefix@";
my $TBOPS       = "@TBOPSEMAIL@";
my $CONTROL	= "@USERNODE@";
my $TBOPSPID    = TBOPSPID();
my $SSH		= "$TB/bin/sshtb";
my $nodereboot  = "$TB/bin/node_reboot";
my $makeconf    = "$TB/sbin/dhcpd_makeconf";
49
my $nodewait    = "$TB/sbin/node_statewait";
50
my $snmpit      = "$TB/bin/snmpit";
51
52
53
54
55
56
57

# Locals
my $elabinelab;
my $elabinelab_eid;

# Protos
sub TearDownEmulab();
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin:/usr/site/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Turn off line buffering on output
#
$| = 1;

#
# We don't want to run this script unless its the real version.
#
if ($EUID != 0) {
    die("*** $0:\n".
	"    Must be root! Maybe its a development version?\n");
}

76
77
78
# Be careful not to exit on transient error
$libdb::DBQUERY_MAXTRIES = 30;

79
80
81
82
83
84
85
# Locals
my $PROJROOT	= PROJROOT();
my $SAVEUID     = $UID;
my $workdir;
my %noderoles	= ();
my $opsnode;
my $bossnode;
Mike Hibler's avatar
Mike Hibler committed
86
87
my $fsnode;
my $routernode;
88
89
90
91
my @expnodes    = ();
my $dbuid;
my $user_name;
my $user_email;
92
93
my $query_result;
my $exptinfo;
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"g"})) {
    $dbgooonly = 1;
}
if (defined($options{"d"})) {
    $debug = 1;
}
109
110
111
if (defined($options{"k"})) {
    $killmode = 1;
}
112
113
114
if (defined($options{"f"})) {
    $fwboot = 1;
}
115
116
117
118
119
120
if (defined($options{"u"})) {
    $update = 1;
}
if (defined($options{"r"})) {
    $remove = 1;
}
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
if (! @ARGV) {
    usage();
}
my ($pid,$eid) = @ARGV;

#
# Untaint the arguments.
#
if ($pid =~ /^([-\w]+)$/) {
    $pid = $1;
}
else {
    die("Tainted argument $pid!\n");
}
if ($eid =~ /^([-\w]+)$/) {
    $eid = $1;
}
else {
    die("Tainted argument $eid!\n");
}
$workdir = TBExptWorkDir($pid, $eid);

#
# Verify user and get his DB uid.
#
if (! UNIX2DBUID($UID, \$dbuid)) {
    die("*** $0:\n".
	"    You do not exist in the Emulab Database.\n");
}

#
# Get email info for user.
#
if (! UserDBInfo($dbuid, \$user_name, \$user_email)) {
    die("*** $0:\n".
	"    Cannot determine your name and email address.\n");
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
158
TBDebugTimeStampsOn();
159

160
161
162
163
164
165
166
167
168
169
170
#
# Get elabinelab status to make sure, and to see if we need to fire off
# an experiment inside once its setup.
#
if (! TBExptIsElabInElab($pid, $eid, \$elabinelab, \$elabinelab_eid)) {
    die("*** $0:\n".
	"    Could not get elabinelab status for experiment $pid/$eid\n");
}
exit(0)
    if (!$elabinelab);

171
172
173
174
175
176
#
# See if the experiment is firewalled
#
my $firewall;
my $firewalled = TBExptFirewall($pid, $eid, \$firewall);

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#
# Presetup; turn off firewall.
#
if ($fwboot) {
    exit(0)
	if (!$firewalled);
    
    print "Turning off firewall rules on $firewall\n";
    $UID = 0;
    system("$SSH -host $firewall ipfw add 1 allow all from any to any");
    if ($?) {
	die("*** $0:\n".
	    "    Error turning off firewall rules ($firewall)!\n");
    }
    exit(0);
}

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
#
# If we are going to start an inner experiment, grab the stuff we need
# from the DB and save it. 
#
if (defined($elabinelab_eid)) {
    $query_result =
	DBQueryFatal("select nsfile from nsfiles ".
		     "where pid='$pid' and eid='$elabinelab_eid'");

    die("*** $0:\n".
	"    No such experiment in DB for $pid/$elabinelab_eid\n")
	if (!$query_result->numrows);

    my ($nsfile) = $query_result->fetchrow_array();
    
    die("*** $0:\n".
	"    No nsfile in DB for $pid/$elabinelab_eid\n")
	if (!defined($nsfile) || $nsfile eq "");

    $query_result =
	DBQueryFatal("select * from experiments ".
		     "where pid='$pid' and eid='$elabinelab_eid'");

    die("*** $0:\n".
	"    No such experiment in DB for $pid/$elabinelab_eid\n")
	if (!$query_result->numrows);

    $exptinfo = $query_result->fetchrow_hashref();
    $exptinfo->{"nsfile"} = $nsfile;
}

#
# Get the role for each node.
#
$query_result =
    DBQueryFatal("select r.node_id,r.inner_elab_role from reserved as r ".
		 "where r.pid='$pid' and r.eid='$eid'");
while (my ($node_id,$role) = $query_result->fetchrow_array()) {
    # Like, the firewall node.
    next
	if (!defined($role));
	
    $noderoles{$node_id} = $role;
    $bossnode = $node_id
Mike Hibler's avatar
Mike Hibler committed
238
239
240
	if ($role eq 'boss' || $role eq 'boss+router');
    $routernode = $node_id
	if ($role eq 'router');
241
    $opsnode = $node_id
Mike Hibler's avatar
Mike Hibler committed
242
243
244
	if ($role eq 'ops' || $role eq 'ops+fs');
    $fsnode = $node_id
	if ($role eq 'fs');
245
246
247
248
249
250
251
252
253
254
    push(@expnodes, $node_id)
	if ($role eq 'node');
}

#
# Tear down an inner emulab.
# 
if ($killmode) {
    exit(TearDownEmulab());
}
255
256
257
258
259
260
elsif ($remove) {
    exit(RemoveNodes());
}
elsif ($update) {
    exit(UpdateEmulab());
}
261
262
263
264
265
266
267

if (1) {
#
# Get elabinelab info. If this is a container for an actual experiment,
# then need to fire off the experiment once the inner emulab is ready to
# go.
# 
Leigh B. Stoller's avatar
Leigh B. Stoller committed
268
TBDebugTimeStamp("Dumping DB state");
269
270
271
272
273
DumpDBGoo();
exit(0)
    if ($dbgooonly);

#
274
# For SSH and SCP below
275
276
#
$UID = 0;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
277

278
#
279
280
# The firewall should be off at this point; called from os_setup with -f.
# 
281

Leigh B. Stoller's avatar
Leigh B. Stoller committed
282
283
284
#
# This is temporary. I think I will switch this over to grabbing the latest
# version from the web server.
285
286
#
my $mkelab = "$TB/etc/rc.mkelab";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
287

288
289
290
if (-e "/proj/$pid/exp/$eid/rc.mkelab") {
    $mkelab = "/proj/$pid/exp/$eid/rc.mkelab";
}
291
292
293
294
print "Copying $mkelab to ${bossnode}/${opsnode}";
print "/${fsnode}"
    if (defined($fsnode));
print "\n";
295
296
system("scp $mkelab ${bossnode}:/tmp");
system("scp $mkelab ${opsnode}:/tmp");
297
298
system("scp $mkelab ${fsnode}:/tmp")
    if (defined($fsnode));
299

300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
if (defined($fsnode)) {
    TBDebugTimeStamp("Setting up fsnode");
    print "Setting up fsnode on $fsnode\n";
    system("$SSH -host $fsnode /tmp/rc.mkelab -s -d > /tmp/fsnode.$$ 2>&1");
    if ($?) {
	$UID = $SAVEUID;
	SENDMAIL("$user_name <$user_email>",
		 "ElabInElab Failure: $pid/$eid",
		 "Error building the fs node ($fsnode)",
		 $TBOPS,
		 "Cc: $TBOPS",
		 ("/tmp/fsnode.$$"));
	system("rm -f /tmp/fsnode.$$ /tmp/opsnode.$$ /tmp/bossnode.$$");
	print STDERR "*** $0:\n".
	    "    Error building the fsnode ($fsnode)!\n";
	exit(($debug ? 0 : -1));
    }
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
318
TBDebugTimeStamp("Setting up opsnode");
319
print "Setting up opsnode on $opsnode\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
320
system("$SSH -host $opsnode /tmp/rc.mkelab -s -d > /tmp/opsnode.$$ 2>&1");
321
322
323
324
if ($?) {
    $UID = $SAVEUID;
    SENDMAIL("$user_name <$user_email>",
	     "ElabInElab Failure: $pid/$eid",
325
	     "Error building the ops node ($opsnode)",
326
327
328
	     $TBOPS,
	     "Cc: $TBOPS",
	     ("/tmp/opsnode.$$"));
329
    system("rm -f /tmp/fsnode.$$ /tmp/opsnode.$$ /tmp/bossnode.$$");
330
331
332
333
    print STDERR "*** $0:\n".
	         "    Error building the opsnode ($opsnode)!\n";
    exit(($debug ? 0 : -1));
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
334
TBDebugTimeStamp("Setting up bossnode");
335
print "Setting up bossnode on $bossnode\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
336
system("$SSH -host $bossnode /tmp/rc.mkelab -s -d > /tmp/bossnode.$$ 2>&1");
337
338
339
340
if ($?) {
    $UID = $SAVEUID;
    SENDMAIL("$user_name <$user_email>",
	     "ElabInElab Failure: $pid/$eid",
341
	     "Error building the boss node ($bossnode)",
342
343
344
	     $TBOPS,
	     "Cc: $TBOPS",
	     ("/tmp/bossnode.$$"));
345
    system("rm -f /tmp/fsnode.$$ /tmp/opsnode.$$ /tmp/bossnode.$$");
346
347
348
349
350
351
    print STDERR "*** $0:\n".
	         "    Error building the bossnode ($bossnode)!\n";
    exit(($debug ? 0 : -1));
}

# Send these log files off now so that we can look at them.
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
if (defined($fsnode)) {
    SENDMAIL("$user_name <$user_email>",
	     "ElabInElab Setup Log: $pid/$eid",
	     "Logs for building fs/ops/boss ($fsnode/$opsnode/$bossnode)",
	     $TBOPS,
	     "Cc: $TBOPS",
	     ("/tmp/fsnode.$$", "/tmp/opsnode.$$", "/tmp/bossnode.$$"));
} else {
    SENDMAIL("$user_name <$user_email>",
	     "ElabInElab Setup Log: $pid/$eid",
	     "Logs for building ops/boss ($opsnode/$bossnode)",
	     $TBOPS,
	     "Cc: $TBOPS",
	     ("/tmp/opsnode.$$", "/tmp/bossnode.$$"));
}
system("rm -f /tmp/fsnode.$$ /tmp/opsnode.$$ /tmp/bossnode.$$");
368
$UID  = $SAVEUID;
369
370

# Run as real user for the next few scripts, which are setuid.
371
$EUID = $UID;
372
373

#
374
375
376
377
378
# Restart DHCPD, but first mark the nodes as being ready to boot inside
# the inner emulab, so that dhcpd_makeconf knows what nodes to change
# the entries for.
#
DBQueryFatal("update reserved set inner_elab_boot=1 ".
379
	     "where pid='$pid' and eid='$eid'");
380

381
382
383
384
385
386
387
print "Regenerating DHCPD config file and restarting daemon.\n";
system("$makeconf -i -r");
if ($?) {
    die("*** $0:\n".
	"    Failed to reconfig/restart DHCPD.\n");
}

388
389
390
391
392
393
394
395
396
397
if (defined($fsnode)) {
    # Reboot fs and wait for it to come back.
    print "Rebooting fsnode ($fsnode).\n";
    TBDebugTimeStamp("Rebooting fsnode");
    system("$nodereboot -w $fsnode");
    if ($?) {
	die("*** $0:\n".
	    "    Error rebooting the fsnode ($fsnode)!\n");
    }
}
398
# Reboot ops and wait for it to come back.
399
print "Rebooting opsnode ($opsnode).\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
400
TBDebugTimeStamp("Rebooting opsnode");
401
402
403
404
405
406
system("$nodereboot -w $opsnode");
if ($?) {
    die("*** $0:\n".
	"    Error rebooting the opsnode ($opsnode)!\n");
}
# Reboot boss and wait for it to come back.
407
print "Rebooting bossnode ($bossnode).\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
408
TBDebugTimeStamp("Rebooting bossnode");
409
410
411
412
413
system("$nodereboot -w $bossnode");
if ($?) {
    die("*** $0:\n".
	"    Error rebooting the bossnode ($bossnode)!\n");
}
414
$EUID = 0;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
415

416
417
418
# Reboot the experimental nodes. They will come up inside the inner elab.
# DO NOT WAIT! They are not going to report ISUP from this point on. 
if (@expnodes) {
419
    print "Rebooting inner experimental nodes.\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
420
    TBDebugTimeStamp("Rebooting experimental nodes");
421
422
    # Run as real user again.
    $EUID = $UID;
423
424
425
426
427
    system("$nodereboot @expnodes");
    if ($?) {
	die("*** $0:\n".
	    "    Error rebooting the expnodes (@expnodes)!\n");
    }
428
    $EUID = 0;
429
430
431
432
433

    #
    # Instead, we ssh into the node and use a utility script to determine
    # when the nodes have rebooted and are in PXEWAIT (part of the inner elab).
    #
434
    # Run as real root for ssh.
435
436
437
438
439
440
441
442
443
444
445
    $UID  = 0;

    print "Waiting for nodes to reboot and join the inner emulab.\n";
    TBDebugTimeStamp("Waiting for inner nodes to reboot");
    system("$SSH -host $bossnode /usr/testbed/sbin/node_statewait -t 180 -a");
    if ($?) {
	print STDERR "*** $0:\n".
	             "    Error waiting for inner nodes to join!\n";
	exit(($debug ? 0 : -1));
    }
    $UID  = $SAVEUID;
446
447
448
449
450
451
452
453
454

    #
    # To avoid confusion later (with swapmod, which wants them to be ISUP),
    # and so the web interface does not show the nodes as down, set the 
    # state to ISUP.
    #
    foreach my $node (@expnodes) {
	TBSetNodeEventState($node, TBDB_NODESTATE_ISUP());
    }
455
}
456
457
458
}

#
459
460
461
462
463
# Fire off inner elab experiment.
# 
if (defined($elabinelab_eid)) {
    # Formatted to make batchexp happy.
    my $nsfilename = "/tmp/$pid-$elabinelab_eid-$$.nsfile";
464
    
465
466
467
468
469
470
471
472
473
474
475
476
477
478
    #
    # Write NS file to temp file so we can send it over.
    #
    open(NS, "> /tmp/$$.ns")
	or die("*** $0:\n".
	       "    Could not write ns code to tmp file!\n");
    print NS $exptinfo->{"nsfile"};
    print NS "\n";
    close(NS);

    #
    # Copy the file over.
    #
    $UID = 0;
479
    print "Sending NS file to inner bossnode ($bossnode).\n";
480
    system("cat /tmp/$$.ns | $SSH -host $bossnode '(cat > $nsfilename)'");
481
482
    if ($?) {
	die("*** $0:\n".
483
	    "    Could not copy ns code to inner boss ($bossnode)!\n");
484
    }
485
486

    #
487
488
    # Now run batchexp on the node as the user. If firewalled, experiment
    # must start async (cause we have to turn the firewall back on). 
489
    #
490
491
492
493
    my $optarg = ($firewalled ? "" : "-w");
	
    print "Starting experiment $pid/$elabinelab_eid on inner emulab.\n";
    TBDebugTimeStamp("Starting inner experiment");
494
    system("$SSH -host $bossnode 'sudo -u $dbuid /usr/testbed/bin/batchexp ".
495
	   "  -q -i $optarg -S \"ElabInElab Experiment\" ".
496
497
498
499
500
	   "  -L \"ElabInElab ElabInElab\" -E \"ElabInElab Experiment\" ".
	   "  -p $pid -e $elabinelab_eid $nsfilename'");
    
    $UID = $SAVEUID;
    unlink("/tmp/$$.ns");
501
502
}

503
#
504
505
506
507
508
509
# Turn the firewall back on.
#
# XXX If this fails, we have to do something much stronger! We do not want
# nodes coming up and starting something if the firewall is not active.
# Maybe hit the panic button from here (turning off the control network).
#
510
511
512
513
514
515
#
if ($firewalled) {
    print "Turning firewall back on\n";
    $UID = 0;
    system("$SSH -host $firewall ipfw delete 1");
    if ($?) {
516
517
518
519
520
521
522
	print STDERR "*** Error turning back on firewall rules ($firewall)!\n".
		     "    Will retry again.\n";
	system("$SSH -host $firewall ipfw delete 1");
	if ($?) {
	    die("*** $0:\n".
		"    Error turning back on firewall rules! Retry failed.\n");
	}
523
524
525
526
    }
    $UID = $SAVEUID;
}

527
528
529
TBDebugTimeStamp("ElabInElab setup done");
exit(0);

530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
#
# Dump parts of the DB that are needed for inner elab to run. The idea
# is to create a set of files named by the table name. Note that mysqld
# cannot write to the project tree cause of directory permissions. Put the
# files into the workdir for now, and them copy them over. 
#
sub DumpDBGoo()
{
    my $statedir = "$workdir/elabinelab";

    if (-d $statedir) {
	system("rm -rf $statedir");
    }
    mkdir($statedir, 0777) or
	die("*** $0:\n".
	    "    Could not mkdir $statedir\n");
    
    chmod(0777, $statedir) or
	die("*** $0:\n".
	    "    Could not chmod $statedir\n");

    #
    # These tables are dumped completely.
    #
    my @FULLTABLES = ("node_types", "interface_types",
		      "interface_capabilities",
		      "switch_paths", "switch_stack_types", "switch_stacks",
		      "node_type_features", "node_types_auxtypes", "osid_map");

    #
    # These tables are dumped by role (node/ops). For each one dump the table
561
562
    # as is, unless its the fs or ops node. For those we want to change the
    # node_id to "fs" or "ops" and their type to ops.
563
564
    #
    my @NODETABLES = ("node_auxtypes", "node_status", "nodes",
Leigh B. Stoller's avatar
Leigh B. Stoller committed
565
		      "node_rusage", "node_hostkeys", "node_idlestats");
566
567
568
569
570
571
572
573
574
575
576
577
578

    #
    # These tables are dumped by project ID.
    #
    my @PROJTABLES = ("project_stats", "projects", "group_stats", "groups");

    #
    # These tables are dumped by user ID (for the project).
    #
    my @USERTABLES = ("users", "user_pubkeys", "user_stats");

    foreach my $table (@FULLTABLES) {
	unlink("$statedir/$table");
579
580
581
582
583
584
585
586
587
588
589
590
	DBQueryWarn("create temporary table temp_${table} ".
		    "select t.* from $table as t")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");

	if ($table eq "node_types") {
	    DBQueryFatal("update temp_${table} set ".
			 "delay_capacity=delay_capacity-1 ".
			 "where delay_capacity>0");
	}

	DBQueryWarn("select * from temp_$table ".
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }

    foreach my $table (@NODETABLES) {
	unlink("$statedir/$table");
	#
	# Create a temporary table.
	#
	DBQueryWarn("create temporary table temp_${table} ".
		    "select t.* from reserved as r ".
		    "left join $table as t on t.node_id=r.node_id ".
		    "left join virt_nodes as v on v.vname=r.vname and ".
		    "     v.pid=r.pid and v.eid=r.eid ".
		    "where r.pid='$pid' and r.eid='$eid' and ".
		    "      t.node_id is not null and ".
Mike Hibler's avatar
Mike Hibler committed
608
		    "      v.inner_elab_role in ('node','fs','ops','ops+fs')")
609
610
611
	    or die("*** $0:\n".
		   "    Could not create temporary table temp_$table\n");
	#
612
613
	# Rename the fs and ops node in each table. For the nodes table,
	# there is a bunch of other stuff to do.
614
	#
615
616
617
	DBQueryFatal("update temp_${table} set node_id='fs' ".
		     "where node_id='$fsnode'")
	    if (defined($fsnode));
618
619
620
621
622
623
	DBQueryFatal("update temp_${table} set node_id='ops' ".
		     "where node_id='$opsnode'");

	if ($table eq "nodes") {
	    DBQueryFatal("update temp_${table} set ".
			 " type='ops', ".
624
			 " phys_nodeid=node_id, ".
625
626
			 " role='ctrlnode', ".
			 " op_mode='OPSNODEBSD' ".
627
			 "where node_id in ('fs','ops')");
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651

	    # Also add the nodes that correspond to the "trunk" wires.
	    DBQueryFatal("insert into temp_${table} ".
			 "select distinct n.* from wires as w ".
			 "left join nodes as n on w.node_id1=n.node_id or ".
			 "     w.node_id2=n.node_id ".
			 "where w.type='Trunk'");
	}
    
	DBQueryWarn("select * from temp_$table ".
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }

    foreach my $table (@PROJTABLES) {
	unlink("$statedir/$table");
	DBQueryWarn("select * from $table ".
		    "where pid='$pid' ".
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
		   "    Could not dump table $table\n");
    }

652
653
654
655
656
657
658
659
660
661
662
663
    #
    # Special case the group and user policy tables. Not sure what to
    # really do about this; should there be any restrictions inside the
    # inner elab?
    #
    unlink("$statedir/group_policies");
    DBQueryWarn("select * from group_policies ".
		"where pid='$pid' or pid='+' or pid='-' ".
		"into outfile '$statedir/group_policies'")
	or die("*** $0:\n".
	       "    Could not dump table group_policies\n");

664
665
    foreach my $table (@USERTABLES) {
	unlink("$statedir/$table");
666
667
668

	DBQueryWarn("create temporary table temp_$table ".
		    "select t.* from group_membership as gm ".
669
670
671
672
		    "left join users as u on u.uid=gm.uid ".
		    "left join $table as t on t.uid=u.uid ".
		    "where gm.pid='$pid' and gm.gid=gm.pid ".
		    " and t.uid is not NULL and ".
673
674
675
676
677
678
679
680
681
682
683
684
685
		    " u.status='" . USERSTATUS_ACTIVE() . "'")
	    or die("*** $0:\n".
		   "    Could not create table temp_$table\n");

	if ($table eq "users") {
	    $creator = ExpLeader($pid, $eid);
	    
	    DBQueryFatal("update temp_${table} set ".
			 " admin=1 ".
			 "where uid='$creator'");
	}

	DBQueryWarn("select * from temp_$table ".
686
687
		    "into outfile '$statedir/$table'")
	    or die("*** $0:\n".
688
		   "    Could not dump table temp_$table\n");
689
690
691
692
693
    }

    # The group_membership is also special.
    DBQueryWarn("select gm.* from group_membership as gm ".
		"left join users as u on u.uid=gm.uid ".
694
695
		"where (gm.pid='$pid' or ".
		"       gm.pid='" . TBOPSPID() . "') and ".
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
		" u.status='" . USERSTATUS_ACTIVE() . "' ".
		"into outfile '$statedir/group_membership'")
	or die("*** $0:\n".
	       "    Could not dump table group_membership\n");

    #
    # Initial images; not that these images are not going to exist inside!
    # 
    DBQueryWarn("select * from images ".
		"where pid='$pid' or (pid='$TBOPSPID' and global=1) ".
		"into outfile '$statedir/images'")
	or die("*** $0:\n".
	       "    Could not dump table images\n");
	    
    DBQueryWarn("create temporary table temp_os_info ".
		"select * from os_info ".
		"where pid='$pid' or (pid='$TBOPSPID' and shared=1)")
	or die("*** $0:\n".
	       "    Could not create table temp_os_info\n");

    # Ack. The MFS paths have a hardcoded "boss" in them, but that is going
    # to resolve incorrectly to an inner control IP, which will not work
    # from the pxeboot kernel since it uses the outer control network.
    # Just remove the host spec; pxeboot will do the right thing.
720
    my $query_result =
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
	DBQueryFatal("select osid,path from temp_os_info ".
		     "where path like '%:%'");
    
    while (my ($osid,$hostpath) = $query_result->fetchrow_array()) {
	my ($host,$path) = $hostpath =~ /^(.*):(.*)$/;

	DBQueryFatal("update temp_os_info set path='$path' where osid='$osid'");
    }

    DBQueryWarn("select * from temp_os_info ".
		"into outfile '$statedir/os_info'")
	or die("*** $0:\n".
	       "    Could not dump table os_info\n");
	    
    DBQueryWarn("select o.* from osidtoimageid as o ".
		"left join images as i on i.imageid=o.imageid ".
		"where i.pid='$pid' or (i.pid='$TBOPSPID' and i.global=1) ".
		"into outfile '$statedir/osidtoimageid'")
	or die("*** $0:\n".
	       "    Could not dump table osidtoimageid\n");

    #
    # interfaces table. Need to tag the interfaces being used as the control
    # network, with the proper tag so they do not say they experimental
    # interfaces in the inner emulab. Use a temp table again.
    #
    DBQueryWarn("create temporary table temp_interfaces ".
		"select t.* from reserved as r ".
		"left join interfaces as t on t.node_id=r.node_id ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"where r.pid='$pid' and r.eid='$eid' and ".
Mike Hibler's avatar
Mike Hibler committed
753
		"      v.inner_elab_role in ('node','ops','fs','ops+fs')")
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
	or die("*** $0:\n".
	       "    Could not create temporary table temp_interfaces\n");

    # First, mark the real control network as "other" to avoid it being
    # thought of as the control network!.
    DBQueryWarn("update temp_interfaces ".
		"set role='" . TBDB_IFACEROLE_OUTER_CONTROL() . "' " .
		"where role='" . TBDB_IFACEROLE_CONTROL() . "'")
	or die("*** $0:\n".
	       "    Could not delete control ifaces from temp_interfaces\n");

    DBQueryWarn("update temp_interfaces set ".
		" role='" . TBDB_IFACEROLE_CONTROL() . "' " .
		"where IP!='' and role='" . TBDB_IFACEROLE_EXPERIMENT() . "'")
	or die("*** $0:\n".
	       "    Could not update roles in temp_interfaces\n");

771
772
773
774
775
776
777
    # And rename the fs/ops nodes as above.
    if (defined($fsnode)) {
	DBQueryWarn("update temp_interfaces set node_id='fs' ".
		    "where node_id='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not fs node_id in temp_interfaces\n");
    }
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
    DBQueryWarn("update temp_interfaces set node_id='ops' ".
		"where node_id='$opsnode'")
	or die("*** $0:\n".
	       "    Could not ops node_id in temp_interfaces\n");

    # Also add the interfaces that correspond to the "trunk" wires.
    DBQueryFatal("insert into temp_interfaces ".
		 "select distinct i.* from wires as w ".
		 "left join interfaces as i on w.node_id1=i.node_id or ".
		 "     w.node_id2=i.node_id ".
		 "where w.type='Trunk'");

    DBQueryWarn("select * from temp_interfaces ".
		"into outfile '$statedir/interfaces'")
	or die("*** $0:\n".
	       "    Could not dump table interfaces\n");

    # And the wires table. Strip out the control wires; not needed.
    DBQueryWarn("create temporary table temp_wires ".
		"select t.* from reserved as r ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"left join wires as t on t.node_id1=r.node_id and ".
		"     t.type='Node' ".
		"where r.pid='$pid' and r.eid='$eid' and ".
Mike Hibler's avatar
Mike Hibler committed
803
		"      v.inner_elab_role in ('node','ops','fs','ops+fs') ")
804
805
806
	or die("*** $0:\n".
	       "    Could not create temporary table temp_wires\n");

807
808
809
810
811
812
813
    # And rename the fs/ops node as above.
    if (defined($fsnode)) {
	DBQueryWarn("update temp_wires set node_id1='fs' ".
		    "where node_id1='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not fs node_id in temp_wires\n");
    }
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
    DBQueryWarn("update temp_wires set node_id1='ops' ".
		"where node_id1='$opsnode'")
	or die("*** $0:\n".
	       "    Could not ops node_id in temp_wires\n");

    # But we need to take out the wires that are being used as the
    # inner control network, or at least mark them as Control.
    $query_result =
	DBQueryWarn("select node_id,card,port from temp_interfaces ".
		    "where role='" . TBDB_IFACEROLE_CONTROL() . "' ");

    while (my ($node_id,$card,$port) = $query_result->fetchrow_array()) {
	DBQueryWarn("update temp_wires set type='Control' ".
		    "where node_id1='$node_id' and card1=$card and ".
		    "      port1=$port");
    }
    # Okay, now add the "trunk" wires in without any alteration.
    DBQueryWarn("insert into temp_wires ".
		"select * from wires where type='Trunk'") 
	or die("*** $0:\n".
	       "    Could not add trunk lines to temp_wires\n");

    DBQueryWarn("select * from temp_wires ".
		"into outfile '$statedir/wires'")
	or die("*** $0:\n".
	       "    Could not dump table wires\n");

    #
842
843
    # Ack, we need to create a reservation for the fs and ops nodes,
    # or else they will look free and it will not be able to check in.
844
845
846
847
848
849
    #
    DBQueryWarn("create temporary table temp_reserved ".
		"select r.* from reserved as r ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"where r.pid='$pid' and r.eid='$eid' ".
Mike Hibler's avatar
Mike Hibler committed
850
		"      and v.inner_elab_role in ('fs','ops','ops+fs')")
851
852
	or die("*** $0:\n".
	       "    Could not create temporary table temp_reserved\n");
853
854
855
856
857
858
859
860
861
    if (defined($fsnode)) {
	DBQueryWarn("update temp_reserved set ".
		    "   node_id='fs', ".
		    "   pid='$TBOPSPID', ".
		    "   eid='opsnodes' ".
		    "where node_id='$fsnode'")
	    or die("*** $0:\n".
		   "    Could not update temporary table temp_reserved\n");
    }
862
863
864
865
866
867
868
869
870
871
872
873
    DBQueryWarn("update temp_reserved set ".
		"   node_id='ops', ".
		"   pid='$TBOPSPID', ".
		"   eid='opsnodes' ".
		"where node_id='$opsnode'")
	or die("*** $0:\n".
	       "    Could not update temporary table temp_reserved\n");
    DBQueryWarn("select * from temp_reserved ".
		"into outfile '$statedir/reserved'")
	or die("*** $0:\n".
	       "    Could not dump table reserved\n");

Mike Hibler's avatar
Mike Hibler committed
874
875
876
877
878
879
880
881
882
883
884
885
    # Copy tiplines table for all nodes so web form gives us a console icon!
    DBQueryWarn("select t.tipname,t.node_id,'',0,0,NULL ".
		"from reserved as r ".
		"left join virt_nodes as v on v.vname=r.vname and ".
		"     v.pid=r.pid and v.eid=r.eid ".
		"left join tiplines as t on t.node_id=r.node_id ".
		"where r.pid='$pid' and r.eid='$eid' and ".
		"      v.inner_elab_role='node' ".
		"into outfile '$statedir/tiplines'")
	or die("*** $0:\n".
	       "    Could not dump table tiplines\n");

886
    #
887
    # Tar up the directory and send it over to (real) ops.
888
889
890
891
892
893
894
895
896
897
898
899
900
    #
    $UID = 0;
    system("tar cf - -C $statedir . | ".
	   "   gzip | $SSH -1 -F /dev/null -host $CONTROL ".
	   "   '(cat > /$PROJROOT/$pid/exp/$eid/dbstate.tar.gz)'");
    if ($?) {
	die("*** $0:\n".
	    "    Could not create dbstate.tar.gz\n");
    }
    $UID = $SAVEUID;
    return 0;
}

901
#
902
903
# Tear down an inner Emulab as cleanly as possible to avoid power cycling
# nodes.
904
905
906
907
908
909
# 
sub TearDownEmulab()
{
    my $tbdir      = "/usr/testbed";
    my $wap        = "$tbdir/sbin/withadminprivs";
    my $nodereboot = "$tbdir/bin/node_reboot";
910
911
912
913
914
915
916
917
918
    my $paniced;

    #
    # If firewalled, check to see if paniced. Right now that means the nodes
    # are going to be powered off, so need to do the clean shutdown dance.
    # 
    if ($firewalled) {
	TBExptGetPanicBit($pid, $eid, \$paniced);
    }
919
920
921
922
923
924
925
926
        
    #
    # We want to rebuild the DHCPD file so that when we reboot the inner nodes
    # they come back to the outer emulab. We cannot just free the nodes, cause
    # then the reload daemon might beat us to it, and end up power cycling the
    # nodes, and that would be bad. So, munge the DB and clear the "role" slot
    # for inner nodes. 
    #
927
    DBQueryFatal("update reserved set inner_elab_role=NULL,inner_elab_boot=0 ".
928
929
930
931
932
933
934
935
936
937
938
939
940
		 "where pid='$pid' and eid='$eid'");

    #
    # XXX Failure at this point will leave things in an inconsistent state
    # cause we have just munged the reserved table. Since we were trying
    # to swap out the experiment, I think this will be okay. Wait and see.
    #
    return 0
	if (!defined($bossnode));

    #
    # Now regen the DHCPD file.
    #
941
    # Run as real user since script is setuid.
942
943
944
945
946
947
948
949
950
951
    $EUID = $UID;
    
    print "Regenerating DHCPD config file and restarting daemon.\n";
    system("$makeconf -i -r");
    if ($?) {
	die("*** $0:\n".
	    "    Failed to reconfig/restart DHCPD.\n");
    }
    $EUID = 0;

952
953
954
955
956
957
958
959
960
    #
    # Kill inner vlans table entries; this is the table that maps
    # inner to outer vlans. We do not care about that anymore since
    # all of the vlans are going to be torn down (using the outer
    # ids).
    #
    DBQueryFatal("delete from elabinelab_vlans ".
		 "where pid='$pid' and eid='$eid'");

961
962
963
964
965
966
967
968
    #
    # If panic set, just return; nodes are going to be powered down.
    #
    if ($firewalled and $paniced) {
	print "Skipping clean shutdown cause panic button pressed.\n";
	return 0;
    }

969
970
971
972
973
974
975
976
977
978
979
980
    #
    # When the nodes reboot, we want them to do something reasonable. We
    # have no idea what is loaded on the disk, so they should go into an
    # MFS and wait, but then a bunch of nodes will all try to load the big
    # MFS at once, and that could wreak havoc. So, clear the boot osids
    # so they go into PXEWAIT. I could use os_select, but clearing all the
    # OSIDs for a node is apparently a bad thing and generates warnings and
    # emails. Why is that? So just clear the DB state until I figure out
    # why that is.
    #
    DBQueryFatal("update nodes set ".
		 "  def_boot_osid='',next_boot_osid='',temp_boot_osid='' ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
981
982
		 "where " .
		    join(" or ", map("node_id='$_'",
983
984
				     ($bossnode, $opsnode,
				      defined($fsnode) ? $fsnode : (),
985
				      @expnodes))));
Leigh B. Stoller's avatar
Leigh B. Stoller committed
986
    
987
988
989
990
991
    #
    # SSH in and kill the inner DHCPD daemon so that it does not reply
    # to rebooting nodes along the inner control network.
    #
    $UID = 0;
992

993
994
995
    print "Killing DHCPD on inner boss ($bossnode)\n";
    system("$SSH -host $bossnode /usr/local/etc/rc.d/2.dhcpd.sh stop");
    if ($?) {
996
997
998
999
1000
	#
	# This error is non-fatal. If DHCPD cannot be killed, then the inner
	# boss is scrogged or never set up properly. Just return and let
	# the nodes get power cycled (if need be). At some point we need a
	# state machine to control this setup stuff.