rc.mkelab 29.3 KB
Newer Older
1
2
3
#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2004, 2005 University of Utah and the Flux Group.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# All rights reserved.
#
# XXX I hardwire IPs into generated /etc/rc.conf and /etc/rc.resolv.
#
# TODO:
#  * Put admin people in local homedirs. 
#
#
use English;
use Getopt::Std;
use Socket;
use IO::Handle;

sub usage()
{
    print "Usage: " .
	scriptname() . " boot|shutdown|reconfig|reset\n";
    exit(1);
}
my $optlist = "ds";
my $action  = "boot";
my $debug   = 0;
27
my $skipit  = 1;	# Temporary until images updated.
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

# Turn off line buffering on output
$| = 1;

# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

# Only root.
if ($EUID != 0) {
    die("*** $0:\n".
	"    Must be root to run this script!\n");
}

# Script specific goo. Put it someplace where prepare script will leave it.
my $LOGFILE    = "/usr/mkelab.debug";
my %elabconfig = ();

#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself. 
# 
use libsetup;
use liblocsetup;
use libtmcc;
use librc;

#
# Not all clients support this.
#
exit(0)
    if (REMOTE() || JAILED() || DELAYHOST());

# Protos.
sub doboot();
sub doshutdown();
sub doreconfig();
sub docleanup();

# Parse command line.
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{'d'})) {
    $debug = 1;
}
if (defined($options{'s'})) {
    $skipit = 1;
}
# Allow default above.
if (@ARGV) {
    $action = $ARGV[0];
}

# More stuff we need below.
my $TBDIR;
my $domain;
my ($pid,$eid,undef) = check_nickname();
my $file    = TMCREATOR();
my $creator = `cat $file`;
chomp($creator);

my $hostname = `hostname`;
chomp($hostname);
91
my ($bossname, $outer_bossip) = tmccbossinfo();
92

93
94
# Cert stuff to give the inner emulab
my $RPCCERT = "/usr/testbed/etc/outer_emulab.pem";
95
#my $RPCPORT = 7778;
96

97
98
99
# This gets turned on/off below
my $WINSUPPORT = 0;

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#
# Find out our domain name, so that we can qualify the localhost entry
#
if ($hostname =~ /[^.]+\.(.+)/) {
    $domain = $1;
}

#
# Find the outer domain for sending email to creator.
#
my $outer_domain;
if ($bossname =~ /[^.]+\.(.+)/) {
    $outer_domain = $1;
}

# Execute the action.
SWITCH: for ($action) {
    /^boot$/i && do {
	doboot();
	last SWITCH;
    };
    /^shutdown$/i && do {
	doshutdown();
	last SWITCH;
    };
    /^reconfig$/i && do {
	doreconfig();
	last SWITCH;
    };
    /^reset$/i && do {
	docleanup();
	last SWITCH;
    };
    fatal("Invalid action: $action\n");
}
exit(0);

# More protos
sub SetupFatal($);
139
sub mysystem($;$);
140
141
142
143
sub SetupOpsNode();
sub SetupBossNode();
sub CreateDefsFile($);
sub SetupSendMail($$);
144
sub GetEmulabSource($);
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200

#
# Boot Action.
#
sub doboot()
{
    my @tmccresults;

    if (tmcc(TMCCCMD_EMULABCONFIG, undef, \@tmccresults) < 0) {
	fatal("Could not get Inner Emulab Config info from server!");
    }
    # If no results then do nothing. No inner elab.
    return 0
	if (! @tmccresults);

    #
    # This is temporary until images are up to date.
    #
    if (! $skipit) {
	print "*** Installing current software first ... \n";
	
	mysystem("cd /users/stoller/testbed/obj-real/tmcd/common; ".
		 "gmake local-install");
	exec($PROGRAM_NAME, ("-s"));
	die("*** $0:\n".
	    "    Could not re-exec script!\n");
    }

    if (!$debug) {
	print "Redirecting output to $LOGFILE\n";

	open(STDERR, ">  $LOGFILE") or die("opening $LOGFILE for STDERR: $!");
	open(STDOUT, ">> $LOGFILE") or die("opening $LOGFILE for STDOUT: $!");

	#
	# Turn off line buffering on output
	#
	STDOUT->autoflush(1);
	STDERR->autoflush(1);
    }
    
    #
    # Turn the tmcc results into a hash first. Then call the boss or ops
    # setup function.
    #
    foreach my $line (@tmccresults) {
	if ($line =~ /^(.*)="(.+)"$/ ||
	    $line =~ /^(.*)=(.+)$/) {
	    $emulabconfig{$1} = $2;
	}
    }

    #
    # XXX To avoid NFS errors while copying goo from outer boss.
    # 
    system("sysctl vfs.nfs.eacces_retry_enable=1 >/dev/null 2>&1");    
201
202
    system("sysctl vfs.nfs.eacces_retry_count=20 >/dev/null 2>&1");

203
204
205
206
207
    # Turn on windows support.
    if (exists($emulabconfig{"WINSUPPORT"}) && $emulabconfig{"WINSUPPORT"}) {
	$WINSUPPORT = 1;
    }

208
209
210
211
212
213
    #
    # Temp hack; make sure control iface in full duplex mode! pc2000 problem.
    #
#    my $outer_controlif = `cat $BOOTDIR/controlif`;
#    chomp($outer_controlif);
#    mysystem("ifconfig $outer_controlif media 100baseTX mediaopt full-duplex");
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242

    if ($emulabconfig{"ROLE"} eq "ops") {
	SetupOpsNode();
    }
    elsif ($emulabconfig{"ROLE"} eq "boss") {
	SetupBossNode();
    }
    print "Done!\n";
    return 0;
}

#
# Setup an ops node.
#
sub SetupOpsNode()
{
    print "Setting up an Ops node ...\n";
    $TBDIR = "/q";

    #
    # Create a ${TBDIR} from the extra slice and put everything there.
    # 
    mysystem("mkdir ${TBDIR}")
	if (! -d "${TBDIR}");
    mysystem("$BINDIR/mkextrafs.pl -f ${TBDIR}");
    mysystem("mkdir ${TBDIR}/testbed");
    mysystem("mkdir ${TBDIR}/testbed/src");
    mysystem("mkdir ${TBDIR}/testbed/obj");

243
    GetEmulabSource("${TBDIR}/testbed/src");
244

245
246
247
248
249
250
    #
    # The mirror tree is copied to temp storage, and then copied into
    # place later.
    #
    if (-e "/proj/$pid/mirror") {
	print "Copying over mirror tree from /proj/$pid/mirror\n";
251
	mysystem("rsync -a --delete /proj/$pid/mirror ${TBDIR}", 3);
252
253
    }

254
255
256
257
258
    #
    # Stash the IP of the outer emulab for tmcc (and script above).
    # We use an IP to avoid DNS issues (there will be a DNS running inside).
    # Ditto for the current router. Need that for later (rc.inelab).
    # 
259
    mysystem("echo '${outer_bossip}' > $ETCDIR/outer_bossnode");
260
261
    mysystem("cp -p $BOOTDIR/routerip $ETCDIR/outer_router");

262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
    #
    # Need outer ip and netmask and iface for hardwired config below.
    #
    if (! -e "$BOOTDIR/myip") {
	SetupFatal("$BOOTDIR/myip does not exist!");
    }
    my $outer_ip = `cat $BOOTDIR/myip`;
    chomp($outer_ip);
    
    if (! -e "$BOOTDIR/mynetmask") {
	SetupFatal("$BOOTDIR/mynetmask does not exist!");
    }
    my $outer_netmask = `cat $BOOTDIR/mynetmask`;
    chomp($outer_netmask);

    if (! -e "$BOOTDIR/controlif") {
	SetupFatal("$BOOTDIR/controlif does not exist!");
    }
    my $outer_controlif = `cat $BOOTDIR/controlif`;
    chomp($outer_controlif);

    #
    # Need outer control router IP below too.
    #
    if (! -e "$BOOTDIR/routerip") {
	SetupFatal("$BOOTDIR/routerip does not exist!");
    }
    my $outer_routerip = `cat $BOOTDIR/routerip`;
    chomp($outer_routerip);

    #
    # We also need the hardwired config for the inner control network.
    # Major kludge; should get it from tmcd data.
    #
    my @ifacelist;
    
    if (getifconfig(\@ifacelist) != 0 || !@ifacelist) {
	SetupFatal("Could not get ifconfig from libsetup!");
    }
    my $inner_controlif = $ifacelist[0]->{IFACE};
    my $inner_ip        = $ifacelist[0]->{IPADDR};
    my $inner_netmask   = $ifacelist[0]->{IPMASK};

305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
    #
    # Run the prepare script to clear out the current accounts and such.
    # From this point on will need to log in as root,
    #
    print "Clearing out existing accounts and such\n";
    mysystem("$BINDIR/prepare");

    #
    # Remove the outer testbed startup script.
    # See new code in dhclient-exit-hooks
    #
    mysystem("rm -f /usr/local/etc/rc.d/testbed.sh");

    #
    # Do this as a separate step cause we need the NFS mounts, but
    # must do the unmounts before running ops-install. 
    # 
322
323
324
    if (!$emulabconfig{OPS_PKG_DIR} || !$emulabconfig{OPS_PKG}) {
	SetupFatal("Could not get package info from Emulab!");
    }
325
    print "Installing the ops metaport.\n";
326
327
    $ENV{"PKG_PATH"} = $emulabconfig{OPS_PKG_DIR};
    mysystem("pkg_add $emulabconfig{OPS_PKG} >/tmp/perrs 2>&1");
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387

    #
    # Clean up a few things on the image and create symlinks into ${TBDIR} for
    # /proj, /users, and /groups. Also allows /share to be created/
    #
    mysystem("umount -A -t nfs");
    # In case umount fails!
    mysystem("cd /; mv -f users users.old");
    mysystem("cd /; mv -f proj proj.old");
    # Groups might not exists
    mysystem("cd /; mv -f groups groups.old")
	if (-d "/groups");
    mysystem("mkdir ${TBDIR}/users ${TBDIR}/proj ${TBDIR}/groups");
    mysystem("ln -s ${TBDIR}/users /users");
    mysystem("ln -s ${TBDIR}/proj /proj");
    mysystem("ln -s ${TBDIR}/groups /groups");

    #
    # Setup a stub /share using slice 2 of the image.
    #
    mysystem("$BINDIR/mkextrafs.pl -f -s 2 /share");

    #
    # Need these for rc.conf.
    # 
    my $bossnode_ip = $emulabconfig{"BOSSIP"};
    my $opsnode_ip  = $emulabconfig{"OPSIP"};

    #
    # Need control network.
    #
    my $control_network = inet_ntoa(inet_aton($opsnode_ip) &
				    inet_aton("255.255.255.0")) . "/24";

    #
    # Need to create an /etc/rc.conf that is more suitable for ops.
    # I took most of this from our real ops node. It will be modified
    # by the ops-install script below.
    #
    print "Creating a new /etc/rc.conf\n";
    open(RC, ">/etc/rc.conf") or
	SetupFatal("Could not open /etc/rc.conf for writing: $!");

    print RC "inetd_enable=\"YES\"\n";
    print RC "sendmail_enable=\"YES\"\n";
    print RC "sshd_enable=\"YES\"\n";

    print RC "ntpdate_enable=\"YES\"\n";
    print RC "ntpdate_flags=\"boss\"\n";
    print RC "xntpd_enable=\"YES\"\n";
    print RC "linux_enable=\"YES\"\n";
    print RC "accounting_enable=\"YES\"\n";

    print RC "nfs_server_enable=\"YES\"\n";
    print RC "nfs_server_flags=\"-u -t -n 8\"\n";
    print RC "nfs_client_enable=\"YES\"\n";
    print RC "mountd_flags=\"-r -p 900\"\n";

    print RC "syslogd_flags=\"-a $control_network\"\n";

388
389
    print RC "network_interfaces=\"$outer_controlif $inner_controlif lo0\"\n";
    print RC "ifconfig_${outer_controlif}=".
390
	"\"inet $outer_ip netmask $outer_netmask\"\n";
391
392
393
394
    print RC "ifconfig_${inner_controlif}=".
	"\"inet $inner_ip netmask $inner_netmask ".
	"media 100baseTX mediaopt full-duplex\"\n";
    print RC "static_routes=\"outerboss\"\n";
395
    print RC "route_outerboss=\"$outer_bossip $outer_routerip\"\n";
396
397
398
399
400

    print RC "defaultrouter=\"$bossnode_ip\"\n";
    print RC "hostname=\"" . $emulabconfig{"OPSNODE"} . "." . $domain . "\"\n";
    close(RC);

401
402
403
404
405
406
407
    #
    # Remove some cruft from /etc/syslog.conf
    #
    mysystem("cat /etc/syslog.conf | grep -v '\@users' > /tmp/syslog.conf");
    mysystem("cp -pf /etc/syslog.conf /etc/syslog.conf.old ; ".
	     "cp /tmp/syslog.conf /etc/syslog.conf");

408
409
410
411
412
413
414
415
416
417
418
    #
    # Create a defs file. Note that this will move to boss at some point.
    #
    CreateDefsFile("${TBDIR}/testbed/src/testbed/defs-elabinelab");

    #
    # Configure an object tree. 
    #
    mysystem("mkdir -p ${TBDIR}/testbed/obj/testbed");
    mysystem("cd ${TBDIR}/testbed/obj/testbed; ".
	     "   ../../src/testbed/configure ".
419
420
	     "      --with-TBDEFS=../../src/testbed/defs-elabinelab ".
	     ($WINSUPPORT ? "--enable-windows" : ""));
421
422
423
424

    #
    # Create the ops node.
    #
425
    my $pkg = "-P $emulabconfig{OPS_PKG}";
426
427
    mysystem("cd ${TBDIR}/testbed/obj/testbed/install; ".
	     "   perl ops-install $pkg -b");
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470

    #
    # And install the ops side.
    #
    mysystem("cd ${TBDIR}/testbed/obj/testbed; gmake ops-install");

    #
    # Lets populate the mail lists with the creator of the experiment so
    # that email goes someplace useful.
    # 
    opendir(DIR, "/etc/mail/lists") or
	SetupFatal("Cannot opendir /etc/mail/lists: $!");
    my @lists = grep { $_ ne "." && $_ ne ".." } readdir(DIR);
    closedir(DIR);

    foreach my $list (@lists) {
	mysystem("echo ${creator}\@${outer_domain} > /etc/mail/lists/$list");
    }

    #
    # Ack! The prepare script above killed the pid file for mountd. This
    # is going to matter later when boss sets up and tries to add accounts.
    # We could reboot ops, but for now its easier if I let both nodes setup
    # before rebooting either one. So, restart mountd so it will create a
    # new pid file in /var/run. Another idea might be that we do not run
    # prepare, or scale it back for inner elab. Needs more thought.
    #
    mysystem("killall mountd");
    mysystem("mountd -r");

    #
    # Need to create a resolv.conf that points to inner boss. This is the
    # last thing we do cause after this, stuff is probably going to stop
    # working properly!
    # 
    print "Creating a new /etc/resolv.conf\n";
    open(RC, ">/etc/resolv.conf") or
	SetupFatal("Could not open /etc/resolv.conf for writing: $!");

    print RC "domain $domain\n";
    print RC "search $domain\n";
    print RC "nameserver $bossnode_ip\n";
    close(RC);
471
472
473
474
475
476
477

    #
    # Hmm, need to run this at startup though.
    # 
    mysystem("echo '/usr/local/etc/emulab/rc/rc.inelab' ".
	     "   >> /etc/rc.local");

478
479
480
481
    # Ack, still waiting for new images.
    mysystem("cp ${TBDIR}/testbed/src/testbed/tmcd/common/rc.inelab ".
	     "   /usr/local/etc/emulab/rc/rc.inelab");

482
483
484
485
486
487
488
489
490
    #
    # Remove source code from ops so that mere users do not get access to it.
    # Something to do with licensing ...
    #
    mysystem("cp -p ${TBDIR}/testbed/src/testbed/defs-elabinelab ".
	     "      ${TBDIR}/testbed/src");
    mysystem("rm -rf ${TBDIR}/testbed/src/testbed");
    mysystem("rm -rf ${TBDIR}/testbed/obj/testbed");

491
492
493
494
495
496
497
    #
    # Copy the mirror tree into place. Do not use rsync.
    #
    if (0 && -e "${TBDIR}/mirror") {
	print "Copying mirror tree into place\n";
	mysystem("cp -Rfp ${TBDIR}/mirror/ /");
    }
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
}

sub SetupBossNode()
{
    print "Setting up a Boss node ...\n";
    $TBDIR = "/usr";

    #
    # Create a ${TBDIR}/testbed from the extra slice and put everything there.
    # 
    mysystem("mkdir ${TBDIR}/testbed")
	if (! -d "${TBDIR}/testbed");
    mysystem("$BINDIR/mkextrafs.pl -f ${TBDIR}/testbed");
    mysystem("mkdir ${TBDIR}/testbed/src");
    mysystem("mkdir ${TBDIR}/testbed/obj");

514
    GetEmulabSource("${TBDIR}/testbed/src");
515
516
517

    print "Copying over initial dbstate from /proj\n";
    mysystem("mkdir ${TBDIR}/testbed/stuff");
518
519
    mysystem("cp -fp /proj/$pid/exp/$eid/dbstate.tar.gz ".
	     "       ${TBDIR}/testbed/stuff");
520
521

    # Copy over creators ssl certificate for XMLRPC. See below.
522
    mysystem("cp -fp ~${creator}/.ssl/emulab.pem ${TBDIR}/testbed/stuff");
523
524
525
526
527

    #
    # Stash the IP of the outer emulab for tmcc (and script above).
    # We use an IP to avoid DNS issues (there will be a DNS running inside).
    # 
528
    mysystem("echo '${outer_bossip}' > $ETCDIR/outer_bossnode");
529
    mysystem("cp -p $BOOTDIR/routerip $ETCDIR/outer_router");
530
    mysystem("cp -p $BOOTDIR/myip $ETCDIR/outer_ipaddr");
531

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
    #
    # Need outer ip and netmask for hardwired config below.
    #
    if (! -e "$BOOTDIR/myip") {
	SetupFatal("$BOOTDIR/myip does not exist!");
    }
    my $outer_ip = `cat $BOOTDIR/myip`;
    chomp($outer_ip);
    
    if (! -e "$BOOTDIR/mynetmask") {
	SetupFatal("$BOOTDIR/mynetmask does not exist!");
    }
    my $outer_netmask = `cat $BOOTDIR/mynetmask`;
    chomp($outer_netmask);

547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
    #
    # Need outer control router IP below too.
    #
    if (! -e "$BOOTDIR/routerip") {
	SetupFatal("$BOOTDIR/routerip does not exist!");
    }
    my $outer_routerip = `cat $BOOTDIR/routerip`;
    chomp($outer_routerip);

    #
    # And we need the name of the control interface for natd below.
    #
    if (! -e "$BOOTDIR/controlif") {
	SetupFatal("$BOOTDIR/controlif does not exist!");
    }
    my $outer_controlif = `cat $BOOTDIR/controlif`;
    chomp($outer_controlif);

565
566
567
568
569
570
571
572
573
574
575
576
577
    #
    # We also need the hardwired config for the inner control network.
    # Major kludge; should get it from tmcd data.
    #
    my @ifacelist;
    
    if (getifconfig(\@ifacelist) != 0 || !@ifacelist) {
	SetupFatal("Could not get ifconfig from libsetup!");
    }
    my $inner_controlif = $ifacelist[0]->{IFACE};
    my $inner_ip        = $ifacelist[0]->{IPADDR};
    my $inner_netmask   = $ifacelist[0]->{IPMASK};

578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
    #
    # Run the prepare script to clear out the current accounts and such.
    # From this point on will need to log in as root,
    #
    print "Clearing out existing accounts and such\n";
    mysystem("$BINDIR/prepare");
  
    #
    # Remove the outer testbed startup script.
    # See new code in dhclient-exit-hooks
    #
    mysystem("rm -f /usr/local/etc/rc.d/testbed.sh");

    #
    # Create a bigger /var/db/mysql (before installing mysql!)
    #
    mysystem("mkdir /var/db/mysql")
	if (! -d "/var/db/mysql");
    mysystem("$BINDIR/mkextrafs.pl -f -s 2 /var/db/mysql");

    #
    # Do this as a separate step cause we need the NFS mounts, but
    # must do the unmounts before running ops-install. 
    # 
602
603
604
    if (!$emulabconfig{BOSS_PKG_DIR} || !$emulabconfig{BOSS_PKG}) {
	SetupFatal("Could not get package info from Emulab!");
    }
605
    print "Installing the boss metaport.\n";
606
607
    $ENV{"PKG_PATH"} = $emulabconfig{BOSS_PKG_DIR};
    mysystem("pkg_add -f -v $emulabconfig{BOSS_PKG} >/tmp/perrs 2>&1");
608
609
610
611

    #
    # Extra stuff to fix up php package stuff. Ask Rob!
    #
612
#    mysystem("tar xzf ${TBDIR}/testbed/stuff/extras.tar.gz -C /");
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634

    #
    # We no longer need anything from NFS, and we need to unmount everything
    # so we can mount new NFS filesystems in their proper places. 
    # 
    mysystem("umount -A -t nfs");
    
    #
    # Need to create an /etc/rc.conf that is more suitable for boss.
    # I took most of this from our real ops node. It will be modified
    # by the ops-install script below.
    #
    print "Creating a new /etc/rc.conf\n";
    open(RC, ">/etc/rc.conf") or
	SetupFatal("Could not open /etc/rc.conf for writing: $!");

    print RC "kern_securelevel_enable=\"NO\"\n";
    print RC "sendmail_enable=\"YES\"\n";
    print RC "sshd_enable=\"YES\"\n";

    print RC "ntpdate_enable=\"YES\"\n";
    # Points to outer boss
635
    print RC "ntpdate_flags=\"${outer_bossip}\"\n";
636
637
638
639
640
641
642
    print RC "linux_enable=\"YES\"\n";
    print RC "accounting_enable=\"YES\"\n";

    print RC "nfs_server_enable=\"YES\"\n";
    print RC "nfs_server_flags=\"-u -t -n 8\"\n";
    print RC "nfs_client_enable=\"YES\"\n";

643
644
    print RC "network_interfaces=\"$outer_controlif $inner_controlif lo0\"\n";
    print RC "ifconfig_${outer_controlif}=".
645
	"\"inet $outer_ip netmask $outer_netmask\"\n";
646
647
648
649
    print RC "ifconfig_${inner_controlif}=".
	"\"inet $inner_ip netmask $inner_netmask ".
	"media 100baseTX mediaopt full-duplex\"\n";
    print RC "static_routes=\"outerboss\"\n";
650
    print RC "route_outerboss=\"$outer_bossip $outer_routerip\"\n";
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671

    #
    # Use natd so that internal control network can talk to outside world.
    # Maybe make an option?
    #
    print RC "firewall_enable=\"YES\"\n";
    print RC "firewall_type=\"open\"\n";
    print RC "natd_interface=\"${outer_controlif}\"\n";
    print RC "natd_enable=\"YES\"\n";
    print RC "natd_flags=\"-use_sockets -unregistered_only -same_ports ".
	"-dynamic -log_facility local6\"\n";

    # Points to outer control router.
    print RC "defaultrouter=\"$outer_routerip\"\n";
    print RC "hostname=\"" . $emulabconfig{"BOSSNODE"} . "." . $domain . "\"\n";
    # We act as the router for the inner ops and inner nodes.
    print RC "gateway_enable=\"YES\"\n";

    print RC "check_quotas=\"NO\"\n";
    close(RC);

672
673
674
675
676
677
678
    #
    # Remove some cruft from /etc/syslog.conf
    #
    mysystem("cat /etc/syslog.conf | grep -v '\@users' > /tmp/syslog.conf");
    mysystem("cp -pf /etc/syslog.conf /etc/syslog.conf.old ; ".
	     "cp /tmp/syslog.conf /etc/syslog.conf");

679
680
681
682
683
684
685
686
687
688
689
    #
    # Create a defs file. Note that this will move to boss at some point.
    #
    CreateDefsFile("${TBDIR}/testbed/src/testbed/defs-elabinelab");

    #
    # Configure an object tree. 
    #
    mysystem("mkdir -p ${TBDIR}/testbed/obj/testbed");
    mysystem("cd ${TBDIR}/testbed/obj/testbed; ".
	     "   ../../src/testbed/configure ".
690
691
	     "      --with-TBDEFS=../../src/testbed/defs-elabinelab ".
	     ($WINSUPPORT ? "--enable-windows" : ""));
692
693
694
695

    #
    # Create the boss node. This will also install the software.
    #
696
    my $pkg = "-P $emulabconfig{BOSS_PKG}";
697
    mysystem("cd ${TBDIR}/testbed/obj/testbed/install; ".
698
	     "   perl boss-install $pkg -b -w ElabInElab");
699
700
701
702
703
704
705
706
707
708
709
710
711
712

    #
    # The above script wiped out the outer emulab root ssh pub keys from ops.
    # We want to add them back so we can ssh into the node from outer Emulab.
    #
    mysystem("cat /root/.ssh/authorized_keys | ssh " . $emulabconfig{"OPSIP"} .
	     " '(cat >> /root/.ssh/authorized_keys)'");

    #
    # Unpack the tftpboot directory. It would be nice if this was part
    # of boss install too. 
    #
    print "Copying over tftpboot tar file from web server and unpacking\n";
    mysystem("wget -q -O ${TBDIR}/testbed/stuff/tftpboot.tar.gz ".
713
	     "http://www.emulab.net/downloads/tftpboot-latest.tar.gz");
714
715
716
717
718
719

    mysystem("tar xzf ${TBDIR}/testbed/stuff/tftpboot.tar.gz ".
	     "    -C /tftpboot");

    # Its the generic stuff; must localize.
    mysystem("cd /tftpboot; mv pxeboot.emu-sio pxeboot.emu; ".
720
	     "              mv freebsd47 freebsd ; mv frisbee47 frisbee");
721

722
723
724
725
    # Create the compressed versions of the files
    mysystem("cd /tftpboot/frisbee/boot; ./prepare; ".
	     "cd /tftpboot/freebsd/boot; ./prepare");

726
727
728
729
730
    #
    # Copy the creators ssl certificate into place. This allows the
    # inner boss to invoke the XMLRPC server on the outer boss for
    # doing things like power control, vlan setup, etc.
    #
731
    mysystem("cp -p ${TBDIR}/testbed/stuff/emulab.pem $RPCCERT");
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755

    #
    # Set up a bunch of DB stuff.  This part will eventually be optional,
    # resulting in a naked setup that will need to be configured the rest of
    # the way by hand. 
    #
    #
    # Unpack the initial DB contents and load it into the DB.
    #
    mysystem("mkdir /tmp/dbstate.$$");
    mysystem("tar xzf ${TBDIR}/testbed/stuff/dbstate.tar.gz ".
	     "    -C /tmp/dbstate.$$");
    
    opendir(DIR, "/tmp/dbstate.$$") or
	SetupFatal("Cannot opendir /tmp/dbstate.$$: $!");
    my @tables = grep { $_ ne "." && $_ ne ".." } readdir(DIR);
    closedir(DIR);

    foreach my $table (@tables) {
	mysystem("echo \"load data infile '/tmp/dbstate.$$/$table' ".
		 "replace into table $table\" | mysql tbdb");
    }

    #
756
757
    # This script does a bunch of stuff with the above DB state, like
    # create the initial project, create subgroups, users, etc. 
758
759
    #
    mysystem("sudo -u elabman /usr/testbed/sbin/withadminprivs ".
760
	     "     /usr/testbed/sbin/elabinelab_bossinit $pid");
761
762
763
764
765
766
767
768
769
770

    #
    # Need to regen the dhcpd config file after loading the DB above.
    #
    mysystem("/usr/testbed/sbin/dhcpd_makeconf -i");

    #
    # Ditto for named config.
    #
    mysystem("/usr/testbed/sbin/named_setup");
771
772
773
774
775
776

    #
    # Tack the frisbee mcast addr ipfw rule onto end of /etc/rc.local.
    #
    mysystem("echo 'ipfw add 10 allow udp from any to 224.0.0.0/4' ".
	     "   >> /etc/rc.local");
777
778
779
780
781
782

    #
    # Hmm, need to run this at startup though.
    #
    mysystem("echo '/usr/local/etc/emulab/rc/rc.inelab' ".
	     "   >> /etc/rc.local");
783
784
785
786

    # Ack, still waiting for new images.
    mysystem("cp ${TBDIR}/testbed/src/testbed/tmcd/common/rc.inelab ".
	     "   /usr/local/etc/emulab/rc/rc.inelab");
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
}

#
# Create a defs file by starting with the stub file, and turning it into
# a real defs file. We should probably do this on the boss side, but its
# easier to localize here for now. 
# 
sub CreateDefsFile($)
{
    my ($defsfile) = @_;
    
    print "Creating defs file from stub defs file\m";
    
    my $bossnode_ip     = $emulabconfig{"BOSSIP"};
    my $opsnode_ip      = $emulabconfig{"OPSIP"};
    my $control_network = inet_ntoa(inet_aton($opsnode_ip) &
				    inet_aton("255.255.255.0"));
    my $dynrange_low    = inet_ntoa(inet_aton($control_network) |
				    inet_aton("0.0.0.200"));
    my $dynrange_high   = inet_ntoa(inet_aton($control_network) |
				    inet_aton("0.0.0.249"));
808
809
    my ($a,$b,$c,$d)    = ($bossnode_ip =~ /(\d+).(\d+).(\d+).(\d+)/);
    my $frismcastaddr   = "235.${d}.${c}";
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843

    open(INDEFS, $defsfile) or
	SetupFatal("Could not open stub defs-elabinelab: $!");
    open(OUTDEFS, "> /tmp/defs-elabinelab") or
	SetupFatal("Could not open new defs-elabinelab: $!");

    while (<INDEFS>) {
	my $key;
	my $val;
	
	if ($_ =~ /^([-\w]*)="(.+)"$/ ||
	    $_ =~ /^([-\w]*)=(.+)$/) {
	    $key = $1;
	    $val = $2;

	    #
	    # Look for things that include "changeme". Emails are special.
	    #
	    if ($val =~ /^(.*)\@(changeme)$/) {
		my $opsnode = $emulabconfig{"OPSNODE"}; 
		print OUTDEFS "${key}=${1}\@${opsnode}.${domain}\n";
		next;
	    }
	    if (! ($val =~ /changeme/)) {
		print OUTDEFS $_;
		next;
	    }
	
	    SWITCH: for ($key) {
		/^BOSSNODE$/ && do {
		    my $bossnode = $emulabconfig{"BOSSNODE"}; 
		    print OUTDEFS "BOSSNODE=${bossnode}.${domain}\n";
		    last SWITCH;
		};
844
845
846
847
		/^OUTERBOSS_NODENAME$/ && do {
		    print OUTDEFS "OUTERBOSS_NODENAME=${bossname}\n";
		    print OUTDEFS "OUTERBOSS_SSLCERTNAME=$RPCCERT\n";
		    # Debugging
848
849
850
		    if (defined($RPCPORT)) {
			print OUTDEFS "OUTERBOSS_XMLRPCPORT=$RPCPORT\n";
		    }
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
		    last SWITCH;
		};
		/^USERNODE$/ && do {
		    my $opsnode = $emulabconfig{"OPSNODE"}; 
		    print OUTDEFS "USERNODE=${opsnode}.${domain}\n";
		    last SWITCH;
		};
		/^FSNODE$/ && do {
		    my $opsnode = $emulabconfig{"OPSNODE"}; 
		    print OUTDEFS "FSNODE=${opsnode}.${domain}\n";
		    last SWITCH;
		};
		/^OURDOMAIN$/ && do {
		    print OUTDEFS "OURDOMAIN=${domain}\n";
		    last SWITCH;
		};
		/^WWWHOST$/ && do {
		    my $bossnode = $emulabconfig{"BOSSNODE"}; 
		    print OUTDEFS "WWWHOST=${bossnode}.${domain}\n";
		    last SWITCH;
		};
		/^THISHOMEBASE$/ && do {
		    print OUTDEFS "THISHOMEBASE=MyEmulab.Net\n";
		    last SWITCH;
		};
		/^TESTBED_NETWORK$/ && do {
		    print OUTDEFS "TESTBED_NETWORK=$control_network\n";
		    last SWITCH;
		};
		/^TESTBED_NETMASK$/ && do {
		    print OUTDEFS "TESTBED_NETMASK=255.255.255.0\n";
		    last SWITCH;
		};
		/^BOSSNODE_IP$/ && do {
		    print OUTDEFS "BOSSNODE_IP=$bossnode_ip\n";
		    last SWITCH;
		};
		/^USERNODE_IP$/ && do {
		    print OUTDEFS "USERNODE_IP=$opsnode_ip\n";
		    last SWITCH;
		};
		/^CONTROL_ROUTER_IP$/ && do {
		    print OUTDEFS "CONTROL_ROUTER_IP=$bossnode_ip\n";
		    last SWITCH;
		};
		/^CONTROL_NETWORK$/ && do {
		    print OUTDEFS "CONTROL_NETWORK=$control_network\n";
		    last SWITCH;
		};
		/^CONTROL_NETMASK$/ && do {
		    print OUTDEFS "CONTROL_NETMASK=255.255.255.0\n";
		    last SWITCH;
		};
		/^PRIVATE_NETWORK$/ && do {
		    print OUTDEFS "PRIVATE_NETWORK=$control_network\n";
		    last SWITCH;
		};
		/^PRIVATE_ROUTER$/ && do {
		    print OUTDEFS "PRIVATE_ROUTER=$bossnode_ip\n";
		    last SWITCH;
		};
		/^PRIVATE_NETMASK$/ && do {
		    print OUTDEFS "PRIVATE_NETMASK=255.255.255.0\n";
		    last SWITCH;
		};
		/^PUBLIC_NETWORK$/ && do {
		    print OUTDEFS "PUBLIC_NETWORK=$control_network\n";
		    last SWITCH;
		};
		/^PUBLIC_ROUTER$/ && do {
		    print OUTDEFS "PUBLIC_ROUTER=$bossnode_ip\n";
		    last SWITCH;
		};
		/^PUBLIC_NETMASK$/ && do {
		    print OUTDEFS "PUBLIC_NETMASK=255.255.255.0\n";
		    last SWITCH;
		};
928
929
930
931
		/^NAMED_FORWARDERS$/ && do {
		    print OUTDEFS "NAMED_FORWARDERS=\"${outer_bossip}\"\n";
		    last SWITCH;
		};
932
933
934
935
936
		/^DHCPD_DYNRANGE$/ && do {
		    print OUTDEFS "DHCPD_DYNRANGE=".
			"\"$dynrange_low $dynrange_high\"\n";
		    last SWITCH;
		};
937
938
		/^FRISEBEEMCASTADDR$/ && do {
		    print OUTDEFS "FRISEBEEMCASTADDR=\"$frismcastaddr\"\n";
939
		    print OUTDEFS "FRISEBEEMCASTPORT=\"6000\"\n";
940
941
		    last SWITCH;
		};
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
		
		print OUTDEFS $_;
	    }
	}
	else {
		print OUTDEFS $_;
	}
    }
    close(INDEFS);
    close(OUTDEFS);
    mysystem("cat /tmp/defs-elabinelab");
    mysystem("mv -f /tmp/defs-elabinelab $defsfile");
}

#
# Print error and exit.
#
sub SetupFatal($)
{
    my ($msg) = @_;

    die("*** $0:\n".
	"    $msg\n");
}

#
# Send email. This should come from a library. 
#
sub SetupSendMail($$)
{
    my ($isfatal, $msg) = @_;

    if (! open(MAIL, "|/usr/sbin/sendmail -t")) {
	die("*** $0:\n".
	    "    SENDMAIL: Could not start sendmail: $!\n".
	    "    $msg\n");
    }
    print MAIL "From: ${creator}\@${hostname}\n";
    print MAIL "To: ${creator}\@${outer_domain}\n";
    if ($isfatal) {
	print MAIL "Subject: ElabInElab setup failure on $hostname\n";
    }
    else {
	print MAIL "Subject: ElabInElab setup completed on $hostname\n";
    }
    print MAIL "\n";
    print MAIL "$msg\n";
    print MAIL "\n";

    if (open(IN, "$LOGFILE")) {
	print MAIL "\n--------- $LOGFILE --------\n";

	while (<IN>) {
	    print MAIL "$_";
	}
	close(IN);
    }
    
    print MAIL "\n";
For faster browsing, not all history is shown. View entire blame