emulab-cnet.pl 17.6 KB
Newer Older
1
2
#!/usr/bin/perl -w
#
Mike Hibler's avatar
Mike Hibler committed
3
# Copyright (c) 2000-2015 University of Utah and the Flux Group.
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
use strict;
use Getopt::Std;
use English;
use Data::Dumper;
use POSIX qw(setsid);
Mike Hibler's avatar
Mike Hibler committed
29
use POSIX ":sys_wait_h";
30
use Socket;
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

#
# Invoked by xmcreate script to configure the control network for a vnode.
#
# NOTE: vmid should be an integer ID.
#
sub usage()
{
    print "Usage: emulab-cnet ".
	"vmid host_ip vnode_name vnode_ip (online|offline)\n";
    exit(1);
}

#
# Turn off line buffering on output
#
$| = 1;

# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself. 
# 
use libsetup;
use libtmcc;
58
use libutil;
59
use libtestbed;
60
use libgenvnode;
61
use libvnode;
62

Mike Hibler's avatar
Mike Hibler committed
63
64
my $lockdebug = 0;

65
66
67
68
69
70
#
# Configure.
#
my $TMCD_PORT	= 7777;
my $SLOTHD_PORT = 8509;
my $EVPROXY_PORT= 16505;
71
72
73
# where all our config files go
my $VMS         = "/var/emulab/vms";
my $VMDIR       = "$VMS/vminfo";
74
my $IPTABLES	= "/sbin/iptables";
75
my $ARPING      = "/usr/bin/arping";
76
my $CAPTURE     = "/usr/local/sbin/capture-nossl";
77
78
# For testing.
my $VIFROUTING  = ((-e "$ETCDIR/xenvifrouting") ? 1 : 0);
79
80

usage()
81
    if (@ARGV < 5);
82

83
84
85
86
87
my $vmid      = shift(@ARGV);
my $host_ip   = shift(@ARGV);
my $vnode_id  = shift(@ARGV);
my $vnode_ip  = shift(@ARGV);
my $vnode_mac = shift(@ARGV);
88
89
90
91
92

# The caller (xmcreate) puts this into the environment.
my $vif         = $ENV{'vif'};
my $XENBUS_PATH = $ENV{'XENBUS_PATH'};
my $bridge      = `xenstore-read "$XENBUS_PATH/bridge"`;
93
94
95
# Need this for capture.
my $LOGPATH     = "$VMDIR/$vnode_id";

96
#
97
# Well, this is interesting; we are called with the XEN store
98
# gone and so not able to find the bridge. vif-bridge does the same
99
100
101
# thing and just ignores it! So if we cannot get it, default to what
# currently think is the control network bridge, so that vif-bridge
# does not leave a bunch of iptables rules behind. 
102
103
104
105
106
107
#
if ($?) {
    $bridge = "xenbr0";
    # For vif-bridge
    $ENV{"bridge"} = $bridge;
}
108
109
chomp($bridge);

110
111
112
113
114
115
116
117
118
119
120
#
# We need the domid below; we can figure that out from the XENBUS_PATH.
#
my $domid;
if ($XENBUS_PATH =~ /vif\/(\d*)\//) {
    $domid = $1;
}
else {
    die("Could not determine domid from $XENBUS_PATH\n");
}

121
122
123
124
125
126
127
my ($bossdomain) = tmccbossinfo();
die("Could not get bossname from tmcc!")
    if (!defined($bossdomain));
if ($bossdomain =~ /^[-\w]+\.(.*)$/) {
    $bossdomain = $1;
}

128
# We need these IP addresses.
129
my $boss_ip = `host boss.${bossdomain} | grep 'has address'`;
130
131
132
if ($boss_ip =~ /has address ([0-9\.]*)$/) {
    $boss_ip = $1;
}
133
my $ops_ip = `host ops.${bossdomain} | grep 'has address'`;
134
135
136
if ($ops_ip =~ /has address ([0-9\.]*)$/) {
    $ops_ip = $1;
}
137
my $fs_ip = `host fs.${bossdomain} | grep 'has address'`;
138
139
140
if ($fs_ip =~ /has address ([0-9\.]*)$/) {
    $fs_ip = $1;
}
141
142
143
144
145
146
147
148
149
150
151
my $PCNET_IP_FILE   = "$BOOTDIR/myip";
my $PCNET_MASK_FILE = "$BOOTDIR/mynetmask";
my $PCNET_GW_FILE   = "$BOOTDIR/routerip";

my $cnet_ip   = `cat $PCNET_IP_FILE`;
my $cnet_mask = `cat $PCNET_MASK_FILE`;
my $cnet_gw   = `cat $PCNET_GW_FILE`;
chomp($cnet_ip);
chomp($cnet_mask);
chomp($cnet_gw);
my $network   = inet_ntoa(inet_aton($cnet_ip) & inet_aton($cnet_mask));
152

153
my ($jail_network,$jail_netmask) = findVirtControlNet();
154
155
# XXX InstaGeni Rack Hack. Hack until I decide on a better way
my $fs_jailip = "172.17.253.254";
156

157
# Each container gets a tmcc proxy running on another port.
158
# If this changes, look at firewall handling in libvnode_xen.
159
160
161
162
163
164
165
166
167
168
169
170
my $local_tmcd_port = $TMCD_PORT + $vmid;

# Need this too.
my $outer_controlif = `cat $BOOTDIR/controlif`;
chomp($outer_controlif);

#
# We setup a bunch of iptables rules when a container goes online, and
# then clear them when it goes offline.
#
sub Online()
{
Mike Hibler's avatar
Mike Hibler committed
171
    my @rules;
172
173
    mysystem2("ifconfig $vif txqueuelen 256");

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
    if ($VIFROUTING) {
	#
	# When using routing instead of bridging, we have to restart
	# dhcp *after* the vif has been created so that dhcpd will
	# start listening on it. 
	#
	if (TBScriptLock("dhcpd", 0, 900) != TBSCRIPTLOCK_OKAY()) {
	    print STDERR "Could not get the dhcpd lock after a long time!\n";
	    return -1;
	}
	restartDHCP();
	TBScriptUnlock();

	#
	# And this clears the arp caches.
	#
	mysystem("$ARPING -c 4 -A -I $bridge $vnode_ip");
    }

Mike Hibler's avatar
Mike Hibler committed
193
194
    @rules = ();

195
    # Prevent dhcp requests from leaving the physical host.
Mike Hibler's avatar
Mike Hibler committed
196
197
198
199
200
    push(@rules,
	 "-A FORWARD -o $bridge -m pkttype ".
	 "--pkt-type broadcast ".
	 "-m physdev --physdev-in $vif --physdev-is-bridged ".
	 "--physdev-out $outer_controlif -j DROP");
201

202
    #
203
204
205
206
207
    # We turn on antispoofing. In bridge mode, vif-bridge adds a rule
    # to allow outgoing traffic. But vif-route does this wrong, so we
    # do it here. We also need an incoming rule since in route mode,
    # incoming packets go throught the FORWARD table, which is set to
    # DROP for antispoofing.
208
    #
209
210
211
212
    # Everything goes through the per vnode INCOMING/OUTGOING tables
    # which are set up in libvnode_xen. If firewalling is not on, then
    # these chains just accept everything. 
    #
213
    if ($VIFROUTING) {
Mike Hibler's avatar
Mike Hibler committed
214
215
216
217
218
	push(@rules,
	     "-A FORWARD -i $vif -s $vnode_ip ".
	     "-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
	push(@rules,
	     "-A FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}");
219
220
221
222
223
224
225
226

	#
	# Another wrinkle. We have to think about packets coming from
	# the container and addressed to the physical host. Send them
	# through OUTGOING chain for filtering, rather then adding
	# another chain. We make sure there are appropriate rules in
	# the OUTGOING chain to protect the host.
	# 
Mike Hibler's avatar
Mike Hibler committed
227
228
229
	push(@rules,
	     "-A INPUT -i $vif -s $vnode_ip ".
	     "-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
230
231
232
233
234
235
236

	#
	# This rule effectively says that if the packet was not filtered 
	# by the INCOMING chain during forwarding, it must be okay to
	# output to the container; we do not want it to go through the
	# dom0 rules.
	#
Mike Hibler's avatar
Mike Hibler committed
237
238
	push(@rules,
	     "-A OUTPUT -o $vif -j ACCEPT");
239
240
241
242
243
244
245
246
    }
    else {
	#
	# Bridge mode. vif-bridge stuck some rules in that we do not
	# want, so insert some new rules ahead of them to capture the
	# packets we want to filter. But we still have to allow the
	# DHCP request packets through.
	#
Mike Hibler's avatar
Mike Hibler committed
247
248
249
	push(@rules,
	     "-I FORWARD -m physdev --physdev-is-bridged ".
	     "--physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}");
250
	    
Mike Hibler's avatar
Mike Hibler committed
251
252
253
	push(@rules,
	     "-I FORWARD -m physdev --physdev-is-bridged ".
	     "--physdev-out $vif -j INCOMING_${vnode_id}");
254
255
256
257
258
259
260
261
262
263
264
265

	#
	# Another wrinkle. We have to think about packets coming from
	# the container and addressed to the physical host. Send them
	# through OUTGOING chain for filtering, rather then adding
	# another chain. We make sure there are appropriate rules in
	# the OUTGOING chain to protect the host.
	#
	# XXX: We cannot use the input interface or bridge options, cause
	# if the vnode_ip is unroutable, the packet appears to come from
	# eth0, according to iptables logging. WTF!
	# 
Mike Hibler's avatar
Mike Hibler committed
266
267
	push(@rules,
	     "-A INPUT -s $vnode_ip -j OUTGOING_${vnode_id}");
268

Mike Hibler's avatar
Mike Hibler committed
269
270
	push(@rules,
	     "-A OUTPUT -d $vnode_ip -j ACCEPT");
271
    }
Mike Hibler's avatar
Mike Hibler committed
272
273
274
275
276

    # Apply the rules
    DoIPtables(@rules) == 0 or
	return -1;

277
278
279
280
281
    # Start a tmcc proxy (handles both TCP and UDP)
    my $tmccpid = fork();
    if ($tmccpid) {
	# Give child a chance to react.
	sleep(1);
Mike Hibler's avatar
Mike Hibler committed
282
283
284
285
286
287
288
289
290
291
292

	# Make sure it is alive.
	if (waitpid($tmccpid, &WNOHANG) == $tmccpid) {
	    print STDERR "$vnode_id: tmcc proxy failed to start\n";
	    return -1;
	}

	if (open(FD, ">/var/run/tmccproxy-$vnode_id.pid")) {
	    print FD "$tmccpid\n";
	    close(FD);
	}
293
294
295
296
297
298
299
300
301
302
    }
    else {
	POSIX::setsid();
	
	exec("$BINDIR/tmcc.bin -d -t 15 -n $vnode_id ".
	       "  -X $host_ip:$local_tmcd_port -s $boss_ip -p $TMCD_PORT ".
	       "  -o $LOGDIR/tmccproxy.$vnode_id.log");
	die("Failed to exec tmcc proxy"); 
    }

303
    # Start a capture for the serial console.
304
    if (-e "$CAPTURE") {
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
	my $tty;

	#
	# XXX HVMs can use an emulated UART as their console.
	# Check for that.
	#
	if (!mysystem2("xenstore-read /local/domain/$domid/hvmloader >/dev/null 2>&1")) {
	    $tty = `xenstore-read /local/domain/$domid/serial/0/tty 2>/dev/null`;
	    $tty = undef if ($?);
	}
	if (!$tty) {
	    $tty = `xenstore-read /local/domain/$domid/console/tty 2>/dev/null`;
	    $tty = undef if ($?);
	}
	if ($tty) {
320
	    chomp($tty);
321
322
323
	    $tty =~ s,\/dev\/,,;

	    # unlink so that we know when capture is ready.
324
	    my $acl = "$LOGPATH/$vnode_id.acl";
325
326
	    unlink($acl)
		if (-e $acl);
327
328
329
330
	    # Remove old log file before start.
	    my $logfile = "$LOGPATH/$vnode_id.log";
	    unlink($logfile)
		if (-e $logfile);
331
	    mysystem2("$CAPTURE -C -i -l $LOGPATH $vnode_id $tty");
332
333
	    #
	    # We need to tell tmcd about it. But do not hang, use timeout.
334
335
	    # Also need to wait for the acl file, since capture is running
	    # in the background. 
336
337
	    #
	    if (! $?) {
338
339
		for (my $i = 0; $i < 10; $i++) {
		    sleep(1);
340
341
		    last
			if (-e $acl && -s $acl);
342
		}
343
		if (! (-e $acl && -s $acl)) {
344
345
346
347
348
349
		    print STDERR "$acl does not exist\n";
		}
		else {
		    mysystem2("$BINDIR/tmcc.bin -n $vnode_id -t 5 ".
			      "   -f $acl tiplineinfo");
		}
350
351
352
353
	    }
	}
    }

Mike Hibler's avatar
Mike Hibler committed
354
355
    @rules = ();

356
    # Reroute tmcd calls to the proxy on the physical host
Mike Hibler's avatar
Mike Hibler committed
357
358
359
360
    push(@rules,
	 "-t nat -A PREROUTING -j DNAT -p tcp ".
	 "--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$local_tmcd_port");
361

Mike Hibler's avatar
Mike Hibler committed
362
363
364
365
    push(@rules,
	 "-t nat -A PREROUTING -j DNAT -p udp ".
	 "--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$local_tmcd_port");
366
367

    # Reroute evproxy to use the local daemon.
Mike Hibler's avatar
Mike Hibler committed
368
369
370
371
    push(@rules,
	 "-t nat -A PREROUTING -j DNAT -p tcp ".
	 "--dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$EVPROXY_PORT");
372
373
374
375
    
    #
    # GROSS! source-nat all traffic destined the fs node, to come from the
    # vnode host, so that NFS mounts work. We do this for non-shared nodes.
376
377
378
379
380
381
382
383
384
    # Shared nodes do the mounts normally from inside the guest. The reason
    # for this distinction is that on a shared host, we ask vif-bridge to
    # turn on antispoofing so that the guest cannot use an IP address other
    # then what we assign. On a non-shared node, the user can log into the
    # physical host and pick any IP they want, but as long as the NFS server
    # is exporting only to the physical IP, they won't be able to mount
    # any directories outside their project. The NFS server *does* export
    # filesystems to the guest IPs if the guest is on a shared host.
    # 
385
    if (!SHAREDHOST()) {
Mike Hibler's avatar
Mike Hibler committed
386
387
388
389
	push(@rules,
	     "-t nat -A POSTROUTING -j SNAT ".
	     "--to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
	     "-o $bridge");
390
    }
391
392
393
394

    # 
    # If the source is from the vnode, headed to the local control 
    # net, no need for any NAT; just let it through.
395
396
397
398
399
400
    #
    # On a remote node (pcpg) we are not bridged to the control
    # network, and so we route to the control network, and then
    # rely on the SNAT rule below. 
    #
    if (!REMOTEDED()) {
Mike Hibler's avatar
Mike Hibler committed
401
402
403
	push(@rules,
	     "-t nat -A POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d $network/$cnet_mask");
404

405
406
407
	#
	# Do not rewrite multicast (frisbee) traffic. Client throws up.
	# 
Mike Hibler's avatar
Mike Hibler committed
408
409
410
	push(@rules,
	     "-t nat -A POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d 224.0.0.0/4");
411
412
413
414

	#
	# Ditto the apod packet.
	#
Mike Hibler's avatar
Mike Hibler committed
415
416
417
	push(@rules,
	     "-t nat -A POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6");
418

419
420
421
422
	#
	# Boss/ops/fs specific rules in case the control network is
	# segmented like it is in Utah.
	#
Mike Hibler's avatar
Mike Hibler committed
423
424
425
	push(@rules,
	     "-t nat -A POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d $boss_ip,$ops_ip");
426
    }
427

428
    # 
429
430
431
    # Ditto for the jail network. On a remote node, the only
    # jail network in on our node, and all of them are bridged
    # togther anyway. 
432
    # 
Mike Hibler's avatar
Mike Hibler committed
433
434
435
    push(@rules,
	 "-t nat -A POSTROUTING -j ACCEPT ".
	 "-s $vnode_ip -d $jail_network/$jail_netmask");
436

437
438
439
440
441
442
    # 
    # Otherwise, setup NAT so that traffic leaving the vnode on its 
    # control net IP, that has been routed out the phys host's
    # control net iface, is NAT'd to the phys host's control
    # net IP, using SNAT.
    # 
Mike Hibler's avatar
Mike Hibler committed
443
444
445
446
    push(@rules,
	 "-t nat -A POSTROUTING ".
	 "-s $vnode_ip -o $outer_controlif ".
	 "-j SNAT --to-source $host_ip");
447
    
Mike Hibler's avatar
Mike Hibler committed
448
449
450
451
    # Apply the rules
    DoIPtables(@rules) == 0 or
	return -1;

452
453
454
455
456
    return 0;
}

sub Offline()
{
Mike Hibler's avatar
Mike Hibler committed
457
458
459
460
    my @rules;

    @rules = ();

461
    # dhcp
Mike Hibler's avatar
Mike Hibler committed
462
463
464
465
466
    push(@rules,
	 "-D FORWARD -o $bridge -m pkttype ".
	 "--pkt-type broadcast ".
	 "-m physdev --physdev-in $vif --physdev-is-bridged ".
	 "--physdev-out $outer_controlif -j DROP");
467
468

    # See above. 
469
    if ($VIFROUTING) {
Mike Hibler's avatar
Mike Hibler committed
470
471
472
473
474
475
476
477
478
479
	push(@rules,
	     "-D FORWARD -i $vif -s $vnode_ip ".
	     "-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
	push(@rules,
	     "-D FORWARD -o $vif -d $vnode_ip -j INCOMING_${vnode_id}");
	push(@rules,
	     "-D INPUT -i $vif -s $vnode_ip ".
	     "-m mac --mac-source $vnode_mac -j OUTGOING_${vnode_id}");
	push(@rules,
	     "-D OUTPUT -o $vif -j ACCEPT");
480
481
482
	
    }
    else {
Mike Hibler's avatar
Mike Hibler committed
483
484
485
486
487
488
489
490
491
492
	push(@rules,
	     "-D FORWARD -m physdev --physdev-is-bridged ".
	     "--physdev-in $vif -s $vnode_ip -j OUTGOING_${vnode_id}");
	push(@rules,
	     "-D FORWARD -m physdev --physdev-is-bridged ".
	     "--physdev-out $vif -j INCOMING_${vnode_id}");
	push(@rules,
	     "-D INPUT -s $vnode_ip -j OUTGOING_${vnode_id}");
	push(@rules,
	     "-D OUTPUT -d $vnode_ip -j ACCEPT");
493
    }
494
495
496

    # tmcc
    # Reroute tmcd calls to the proxy on the physical host
Mike Hibler's avatar
Mike Hibler committed
497
498
499
500
501
502
503
504
505
506
507
508
509
    push(@rules,
	 "-t nat -D PREROUTING -j DNAT -p tcp ".
	 "--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$local_tmcd_port");
    push(@rules,
	 "-t nat -D PREROUTING -j DNAT -p udp ".
	 "--dport $TMCD_PORT -d $boss_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$local_tmcd_port");

    # Apply the rules
    if (DoIPtables(@rules) != 0) {
	print STDERR "WARNING: could not remove iptables rules\n";
    }
510
511
512
513
514
515
516

    if (-e "/var/run/tmccproxy-$vnode_id.pid") {
	my $pid = `cat /var/run/tmccproxy-$vnode_id.pid`;
	chomp($pid);
	mysystem2("/bin/kill $pid");
    }

517
518
    if (-e "$LOGPATH/$vnode_id.pid") {
	my $pid = `cat $LOGPATH/$vnode_id.pid`;
519
520
521
522
	chomp($pid);
	mysystem2("/bin/kill $pid");
    }

Mike Hibler's avatar
Mike Hibler committed
523
524
    @rules = ();

525
    if (!SHAREDHOST()) {
Mike Hibler's avatar
Mike Hibler committed
526
527
528
529
	push(@rules,
	     "-t nat -D POSTROUTING -j SNAT ".
	     "--to-source $host_ip -s $vnode_ip -d $fs_ip,$fs_jailip ".
	     "-o $bridge");
530
531
    }

Mike Hibler's avatar
Mike Hibler committed
532
533
534
    push(@rules,
	 "-t nat -D POSTROUTING -j ACCEPT ".
	 "-s $vnode_ip -d $jail_network/$jail_netmask");
535

536
    if (!REMOTEDED()) {
Mike Hibler's avatar
Mike Hibler committed
537
538
539
	push(@rules,
	     "-t nat -D POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d $network/$cnet_mask");
540

Mike Hibler's avatar
Mike Hibler committed
541
542
543
	push(@rules,
	     "-t nat -D POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d $boss_ip,$ops_ip");
544
	
Mike Hibler's avatar
Mike Hibler committed
545
546
547
	push(@rules,
	     "-t nat -D POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -d 224.0.0.0/4");
548
	
Mike Hibler's avatar
Mike Hibler committed
549
550
551
	push(@rules,
	     "-t nat -D POSTROUTING -j ACCEPT ".
	     "-s $vnode_ip -m icmp --protocol icmp --icmp-type 6/6");
552
    }
553

Mike Hibler's avatar
Mike Hibler committed
554
555
556
    push(@rules,
	 "-t nat -D POSTROUTING ".
	 "-s $vnode_ip -o $outer_controlif -j SNAT --to-source $host_ip");
557

558
    # evproxy
Mike Hibler's avatar
Mike Hibler committed
559
560
561
562
563
564
565
566
567
    push(@rules,
	 "-t nat -D PREROUTING -j DNAT -p tcp ".
	 "--dport $EVPROXY_PORT -d $ops_ip -s $vnode_ip ".
	 "--to-destination $host_ip:$EVPROXY_PORT");

    # Apply the rules
    if (DoIPtablesNoFail(@rules) != 0) {
	print STDERR "WARNING: could not remove iptables rules\n";
    }
568
569
570
571

    return 0;
}

Mike Hibler's avatar
Mike Hibler committed
572
573
574
575
576
577
578
579
#
# Run the Xen vif-* script under our iptables lock.
#
sub Runscript($@)
{
    my ($vnode_ip, @args) = @_;
    my $rv = 0;

580
581
582
583
584
585
    #
    # Oh jeez, iptables is about the dumbest POS I've ever seen;
    # it fails if you run two at the same time. So we have to
    # serialize the calls. Rather then worry about each call, just
    # take a big lock here. 
    #
Mike Hibler's avatar
Mike Hibler committed
586
587
    TBDebugTimeStamp("$vnode_id emulab-cnet: grabbing iptables lock")
	if ($lockdebug);
588
    if (TBScriptLock("iptables", 0, 300) != TBSCRIPTLOCK_OKAY()) {
589
	print STDERR "Could not get the iptables lock after a long time!\n";
Mike Hibler's avatar
Mike Hibler committed
590
	return -1;
591
    }
Mike Hibler's avatar
Mike Hibler committed
592
593
    TBDebugTimeStamp("  got iptables lock")
	if ($lockdebug);
594
595
596

    #
    # First run the xen script to do the bridge interface. We do this
Mike Hibler's avatar
Mike Hibler committed
597
    # inside the lock since vif-bridge/vif-route do some iptables stuff.
598
    #
599
600
601
602
603
604
605
606
    # vif-bridge/vif-route has bugs that cause it to leave iptables
    # rules behind. If we put this stuff into the environment, they
    # will work properly.
    #
    $ENV{"ip"} = $vnode_ip;
    if ($VIFROUTING) {
	$ENV{"netdev"} = "xenbr0";
	$ENV{"gatewaydev"} = "xenbr0";
Mike Hibler's avatar
Mike Hibler committed
607
	mysystem2("/etc/xen/scripts/vif-route-emulab @args");
608
609
    }
    else {
Mike Hibler's avatar
Mike Hibler committed
610
	mysystem2("/etc/xen/scripts/vif-bridge @args");
611
    }
Mike Hibler's avatar
Mike Hibler committed
612
613
614
    $rv = $?;
    TBDebugTimeStamp("  releasing iptables lock")
	if ($lockdebug);
615
    TBScriptUnlock();
Mike Hibler's avatar
Mike Hibler committed
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631

    return $rv;
}

my $rval = 0;
if (@ARGV) {
    my $op = $ARGV[0];

    TBDebugTimeStampsOn();
    TBDebugTimeStampWithDate("$vnode_ip emulab-cnet $op: called");

    # for online and add, run script at beginning
    if ($op ne "offline" && Runscript($vnode_ip, @ARGV)) {
	$rval = -1;
    }
    elsif ($op eq "online") {
632
	$rval = Online();
633
634
    }
    elsif ($op eq "offline") {
635
	$rval = Offline();
Mike Hibler's avatar
Mike Hibler committed
636
637
638
639
	# for offline, run script at the end
	if (!$rval) {
	    $rval = Runscript($vnode_ip, @ARGV);
	}
640
    }
Mike Hibler's avatar
Mike Hibler committed
641
642

    TBDebugTimeStampWithDate("$vnode_id: emulab-cnet $op: done, rval=$rval");
643
}
Mike Hibler's avatar
Mike Hibler committed
644
exit($rval);