All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

setup-ovs-node.sh 19.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
#!/bin/sh

#
# This sets up openvswitch networks (on neutron, the external and data
# networks).  The networkmanager and compute nodes' physical interfaces
# have to get moved into br-ex and br-int, respectively -- on the
# moonshots, that's eth0 and eth1.  The controller is special; it doesn't
# get an openvswitch setup, and gets eth1 10.0.0.3/8 .  The networkmanager
# is also special; it gets eth1 10.0.0.1/8, but its eth0 moves into br-ex,
# and its eth1 moves into br-int.  The compute nodes get IP addrs from
# 10.0.1.1/8 and up, but setup-ovs.sh determines that.
#

set -x

# Gotta know the rules!
if [ $EUID -ne 0 ] ; then
    echo "This script must be run as root" 1>&2
    exit 1
fi

# Grab our libs
. "`dirname $0`/setup-lib.sh"

25 26
logtstart "ovs-node"

27 28 29 30 31
#
# Figure out which interfaces need to go where.  We already have 
# $EXTERNAL_NETWORK_INTERFACE from setup-lib.sh , and it and its configuration
# get applied to br-ex .  So, we need to find which interface corresponds to
# DATALAN on this node, if any, and move it (and its configuration OR its new
32
# new DATAIP iff USE_EXISTING_IPS was set) to br-int
33 34 35 36 37
#
EXTERNAL_NETWORK_BRIDGE="br-ex"
#DATA_NETWORK_INTERFACE=`ip addr show | grep "inet $MYIP" | sed -e "s/.*scope global \(.*\)\$/\1/"`
DATA_NETWORK_BRIDGE="br-data"
INTEGRATION_NETWORK_BRIDGE="br-int"
38 39 40 41 42

#
# If this is the controller, we don't have to do much network setup; just
# setup the data network with its IP.
#
43
#if [ "$HOSTNAME" = "$CONTROLLER" ]; then
44
#    if [ ${USE_EXISTING_IPS} -eq 0 ]; then
45 46 47 48
#	ifconfig ${DATA_NETWORK_INTERFACE} $DATAIP netmask 255.0.0.0 up
#    fi
#    exit 0;
#fi
49

50 51 52 53 54
#
# Grab our control net info before we change things around.
#
if [ ! -f $OURDIR/ctlnet.vars ]; then
    ctlip="$MYIP"
55 56
    ctlmac=`ip -o link show ${EXTERNAL_NETWORK_INTERFACE} | sed -n -e 's/^.*link\/ether \([0-9a-fA-F:]*\) .*$/\1/p'`
    ctlstrippedmac=`echo $ctlmac | sed -e 's/://g'`
57
    ctlnetmask=`ifconfig ${EXTERNAL_NETWORK_INTERFACE} | sed -n -e 's/^.*mask[: ]*\([0-9]*.[0-9]*.[0-9]*.[0-9]*\).*$/\1/ip'`
58 59
    ctlgw=`ip route show default | sed -n -e 's/^default via \([0-9]*.[0-9]*.[0-9]*.[0-9]*\).*$/\1/p'`
    ctlnet=`ip route show dev ${EXTERNAL_NETWORK_INTERFACE} | sed -n -e 's/^\([0-9]*.[0-9]*.[0-9]*.[0-9]*\/[0-9]*\) .*$/\1/p'`
60
    ctlprefix=`echo $ctlnet | cut -d/ -f2`
61 62

    echo "ctlip=\"$ctlip\"" > $OURDIR/ctlnet.vars
63 64
    echo "ctlmac=\"$ctlmac\"" >> $OURDIR/ctlnet.vars
    echo "ctlstrippedmac=\"$ctlstrippedmac\"" >> $OURDIR/ctlnet.vars
65 66 67
    echo "ctlnetmask=\"$ctlnetmask\"" >> $OURDIR/ctlnet.vars
    echo "ctlgw=\"$ctlgw\"" >> $OURDIR/ctlnet.vars
    echo "ctlnet=\"$ctlnet\"" >> $OURDIR/ctlnet.vars
68
    echo "ctlprefix=\"$ctlprefix\"" >> $OURDIR/ctlnet.vars
69 70 71 72
else
    . $OURDIR/ctlnet.vars
fi

73 74 75
#
# Otherwise, first we need openvswitch.
#
76
maybe_install_packages openvswitch-common openvswitch-switch
77 78

# Make sure it's running
79 80 81 82
service_restart openvswitch
service_restart openvswitch-switch
service_enable openvswitch
service_enable openvswitch-switch
83 84 85 86

#
# Setup the external network
#
87 88
ovs-vsctl add-br ${EXTERNAL_NETWORK_BRIDGE}
ovs-vsctl add-port ${EXTERNAL_NETWORK_BRIDGE} ${EXTERNAL_NETWORK_INTERFACE}
89 90 91
#ethtool -K $EXTERNAL_NETWORK_INTERFACE gro off

#
92
# Now move the $EXTERNAL_NETWORK_INTERFACE and default route config to ${EXTERNAL_NETWORK_BRIDGE}
93
#
94 95
DNSDOMAIN=`hostname | cut -d. -f4-100`
DNSSERVER=`cat /etc/resolv.conf | grep nameserver | head -1 | awk '{ print $2 }'`
96

97 98 99 100 101 102 103 104 105 106
#
# If we're Mitaka or greater, we have to always re-add our anti-ARP
# spoofing flows on each boot.  See setup-network-plugin-openvswitch.sh
# and the bottom of this script.
#
readdflows=""
if [ $OSVERSION -gt $OSLIBERTY ] ; then
    readdflows='up for line in `cat /etc/neutron/ovs-default-flows/br-ex`; do ovs-ofctl add-flow br-ex $line ; done'
fi

107 108 109 110
#
# We need to blow away the Emulab config -- no more dhcp
# This would definitely break experiment modify, of course
#
111 112
if [ $DISTRIB_MAJOR -lt 18 ]; then
    cat <<EOF > /etc/network/interfaces
113 114 115
#
# Openstack Network Node in Cloudlab/Emulab/Apt/Federation
#
116 117

# The loopback network interface
118
auto lo
119 120
iface lo inet loopback

121
auto ${EXTERNAL_NETWORK_BRIDGE}
122
iface ${EXTERNAL_NETWORK_BRIDGE} inet static
123 124 125
    address $ctlip
    netmask $ctlnetmask
    gateway $ctlgw
126 127
    dns-search $DNSDOMAIN
    dns-nameservers $DNSSERVER
128
    up echo "${EXTERNAL_NETWORK_BRIDGE}" > /var/run/cnet
129
    up echo "${EXTERNAL_NETWORK_INTERFACE}" > /var/emulab/boot/controlif
130
$readdflows
131

132
auto ${EXTERNAL_NETWORK_INTERFACE}
133 134 135
iface ${EXTERNAL_NETWORK_INTERFACE} inet static
    address 0.0.0.0
EOF
136 137 138
else
    mv /etc/udev/rules.d/99-emulab-networkd.rules \
        /etc/udev/rules.d/99-emulab-networkd.rules.NO
David Johnson's avatar
David Johnson committed
139
    systemctl disable emulab-udev-settle.service
140 141 142 143 144 145
    cat <<EOF >/etc/systemd/system/testbed-pre-static-control-network.service
[Unit]
Description=Testbed Static Control Network Services
After=network.target network-online.target local-fs.target
Wants=network.target
Before=testbed.service
146
Before=pubsubd.service
147 148 149 150

[Service]
Type=oneshot
RemainAfterExit=yes
David Johnson's avatar
David Johnson committed
151
ExecStart=$OURDIR/testbed-pre-static-control-network.sh
152 153 154 155 156 157
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
WantedBy=testbed.service
158
WantedBy=pubsubd.service
159
EOF
David Johnson's avatar
David Johnson committed
160
    cat <<EOF >$OURDIR/testbed-pre-static-control-network.sh
161 162 163 164 165 166 167
#!/bin/sh

#
# These are just the things we cannot do via hook from systemd-networkd,
# that were previously done in /etc/network/interfaces via "up" hook.
#
echo "${EXTERNAL_NETWORK_BRIDGE}" > /var/run/cnet
168
echo "${EXTERNAL_NETWORK_INTERFACE}" > /var/emulab/boot/controlif
169
EOF
David Johnson's avatar
David Johnson committed
170
    chmod 755 $OURDIR/testbed-pre-static-control-network.sh
David Johnson's avatar
David Johnson committed
171 172
    systemctl daemon-reload
    systemctl enable testbed-pre-static-control-network.service
173
    cat <<'EOF' >/etc/systemd/system/openvswitch-post-control-network.service
David Johnson's avatar
David Johnson committed
174 175 176 177 178
[Unit]
Description=Testbed OpenVswitch Static Control Network Flows
After=network.target network-online.target local-fs.target openvswitch-switch.service
Wants=network.target openvswitch-switch.service
Before=testbed.service
179
Before=pubsubd.service
David Johnson's avatar
David Johnson committed
180 181 182 183 184 185 186 187 188 189 190

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh -c 'for line in `cat /etc/neutron/ovs-default-flows/br-ex`; do ovs-ofctl add-flow br-ex $line ; done'
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
WantedBy=testbed.service
191
WantedBy=pubsubd.service
David Johnson's avatar
David Johnson committed
192 193 194
EOF
    systemctl daemon-reload
    systemctl enable openvswitch-post-control-network.service
195 196 197 198 199 200 201 202 203 204 205 206
    cat <<EOF >/etc/systemd/network/${EXTERNAL_NETWORK_BRIDGE}.network
[Match]
Name=${EXTERNAL_NETWORK_BRIDGE}

[Network]
Description=OpenStack External Network Bridge
DHCP=no
Address=$ctlip/$ctlprefix
Gateway=$ctlgw
DNS=$DNSSERVER
Domains=$DNSDOMAIN
IPForward=yes
David Johnson's avatar
David Johnson committed
207 208 209 210 211 212 213 214
EOF
    cat <<EOF >/etc/systemd/network/${EXTERNAL_NETWORK_INTERFACE}.network
[Match]
Name=${EXTERNAL_NETWORK_INTERFACE}

[Network]
Description=OpenStack External Network Bridge Physical Interface
DHCP=no
215 216
EOF
fi
217 218

ifconfig ${EXTERNAL_NETWORK_INTERFACE} 0 up
219 220
ifconfig ${EXTERNAL_NETWORK_BRIDGE} $ctlip netmask $ctlnetmask up
route add default gw $ctlgw
221

222
service_restart openvswitch-switch
223

224 225
# Also restart slothd so it listens on the new control iface.
echo "${EXTERNAL_NETWORK_BRIDGE}" > /var/run/cnet
226
echo "${EXTERNAL_NETWORK_INTERFACE}" > /var/emulab/boot/controlif
227 228 229 230 231
/usr/local/etc/emulab/rc/rc.slothd stop
pkill slothd
sleep 1
/usr/local/etc/emulab/rc/rc.slothd start

232 233 234 235
#
# Add the management network config if necessary (if not, it's already a VPN)
#
if [ ! -z "$MGMTLAN" ]; then
236 237
    if [ $DISTRIB_MAJOR -lt 18 ]; then
	cat <<EOF >> /etc/network/interfaces
238

239
auto ${MGMT_NETWORK_INTERFACE}
240 241 242
iface ${MGMT_NETWORK_INTERFACE} inet static
    address $MGMTIP
    netmask $MGMTNETMASK
243 244
    up mkdir -p /var/run/emulab
    up echo "${MGMT_NETWORK_INTERFACE} $MGMTIP $MGMTMAC" > /var/run/emulab/interface-done-$MGMTMAC
245
EOF
246 247 248 249 250 251 252 253 254 255 256
    else
	cat <<EOF >/etc/systemd/network/${MGMT_NETWORK_INTERFACE}.network
[Match]
Name=${MGMT_NETWORK_INTERFACE}

[Network]
Description=OpenStack Management Network
DHCP=no
Address=$MGMTIP/$MGMTPREFIX
IPForward=yes
EOF
David Johnson's avatar
David Johnson committed
257
	cat <<EOF >>$OURDIR/testbed-pre-static-control-network.sh
258 259 260 261 262

mkdir -p /var/run/emulab
echo "${MGMT_NETWORK_INTERFACE} $MGMTIP $MGMTMAC" > /var/run/emulab/interface-done-$MGMTMAC
EOF
    fi
David Johnson's avatar
David Johnson committed
263
    if [ -n "$MGMTVLANDEV" ]; then
264 265
	if [ $DISTRIB_MAJOR -lt 18 ]; then
	    cat <<EOF >> /etc/network/interfaces
David Johnson's avatar
David Johnson committed
266 267
    vlan-raw-device ${MGMTVLANDEV}
EOF
268 269 270 271 272 273 274 275 276
	else
	    cat <<EOF >/etc/systemd/network/${MGMT_NETWORK_INTERFACE}.netdev
[NetDev]
Name=${MGMT_NETWORK_INTERFACE}
Kind=vlan

[VLAN]
Id=$MGMTVLANTAG
EOF
277 278
	    if [ ! -e /etc/systemd/network/${MGMTVLANDEV}.network ]; then
		cat <<EOF >/etc/systemd/network/${MGMTVLANDEV}.network
279 280 281 282 283 284 285
[Match]
Name=${MGMTVLANDEV}

[Network]
DHCP=no
VLAN=${MGMT_NETWORK_INTERFACE}
EOF
286 287 288 289 290
	    else
		cat <<EOF >>/etc/systemd/network/${MGMTVLANDEV}.network
VLAN=${MGMT_NETWORK_INTERFACE}
EOF
	    fi
291
	fi
David Johnson's avatar
David Johnson committed
292
    fi
293 294
fi

295
#
David Johnson's avatar
David Johnson committed
296
# Make sure we have the integration bridge
297
#
298
ovs-vsctl add-br ${INTEGRATION_NETWORK_BRIDGE}
299

David Johnson's avatar
David Johnson committed
300
#
301
# (Maybe) Setup the flat data networks
David Johnson's avatar
David Johnson committed
302
#
303 304 305
for lan in $DATAFLATLANS ; do
    # suck in the vars we'll use to configure this one
    . $OURDIR/info.$lan
David Johnson's avatar
David Johnson committed
306

307 308 309 310
    ovs-vsctl add-br ${DATABRIDGE}

    ovs-vsctl add-port ${DATABRIDGE} ${DATADEV}
    ifconfig ${DATADEV} 0 up
311 312
    if [ $DISTRIB_MAJOR -lt 18 ]; then
	cat <<EOF >> /etc/network/interfaces
313

314 315
auto ${DATABRIDGE}
iface ${DATABRIDGE} inet static
316 317
    address $DATAIP
    netmask $DATANETMASK
318 319
    up mkdir -p /var/run/emulab
    up echo "${DATABRIDGE} $DATAIP $DATAMAC" > /var/run/emulab/interface-done-$DATAMAC
David Johnson's avatar
David Johnson committed
320

321 322
auto ${DATADEV}
iface ${DATADEV} inet static
David Johnson's avatar
David Johnson committed
323 324
    address 0.0.0.0
EOF
325 326
	if [ -n "$DATAVLANDEV" ]; then
	    cat <<EOF >> /etc/network/interfaces
David Johnson's avatar
David Johnson committed
327 328
    vlan-raw-device ${DATAVLANDEV}
EOF
329 330 331 332 333 334 335 336 337 338 339
	fi
    else
	cat <<EOF >/etc/systemd/network/${DATABRIDGE}.network
[Match]
Name=${DATABRIDGE}

[Network]
Description=OpenStack Data Flat Lan $DATABRIDGE Network
DHCP=no
Address=$DATAIP/$DATAPREFIX
IPForward=yes
David Johnson's avatar
David Johnson committed
340 341 342 343 344 345 346 347
EOF
	cat <<EOF >/etc/systemd/network/${DATADEV}.network
[Match]
Name=${DATADEV}

[Network]
Description=OpenStack Data Flat Lan $DATABRIDGE Network Physical Interface
DHCP=no
348
EOF
David Johnson's avatar
David Johnson committed
349
	cat <<EOF >>$OURDIR/testbed-pre-static-control-network.sh
350 351 352 353 354 355 356 357 358 359 360 361 362

mkdir -p /var/run/emulab
echo "${DATABRIDGE} $DATAIP $DATAMAC" > /var/run/emulab/interface-done-$DATAMAC
EOF
	if [ -n "$DATAVLANDEV" ]; then
	    cat <<EOF >/etc/systemd/network/${DATADEV}.netdev
[NetDev]
Name=${DATADEV}
Kind=vlan

[VLAN]
Id=$DATAVLANTAG
EOF
363 364
	    if [ ! -e /etc/systemd/network/${DATAVLANDEV}.network ]; then
		cat <<EOF >/etc/systemd/network/${DATAVLANDEV}.network
365 366 367 368 369 370 371
[Match]
Name=${DATAVLANDEV}

[Network]
DHCP=no
VLAN=${DATADEV}
EOF
372 373 374 375 376
	    else
		cat <<EOF >/etc/systemd/network/${DATAVLANDEV}.network
VLAN=${DATADEV}
EOF
	    fi
377
	fi
David Johnson's avatar
David Johnson committed
378 379
    fi

380
    ifconfig ${DATABRIDGE} $DATAIP netmask $DATANETMASK up
David Johnson's avatar
David Johnson committed
381
    # XXX!
382 383
    #route add -net 10.0.0.0/8 dev ${DATA_NETWORK_BRIDGE}
done
David Johnson's avatar
David Johnson committed
384

385 386 387 388 389 390 391 392 393 394
#
# (Maybe) Setup the VLAN data networks.
# Note, these are for the case where we're giving openstack the chance
# to manage these networks... so we delete the emulab-created vlan devices,
# create an openvswitch switch for the vlan device, and just add the physical
# device as a port.  Simple.
#
for lan in $DATAVLANS ; do
    # suck in the vars we'll use to configure this one
    . $OURDIR/info.$lan
David Johnson's avatar
David Johnson committed
395

396 397 398 399 400 401 402 403 404
    ifconfig $DATADEV down
    vconfig rem $DATADEV

    # If the bridge exists, we've already done it (we might have multiplexed
    # (trunked) more than one vlan across this physical device).
    ovs-vsctl br-exists ${DATABRIDGE}
    if [ $? -ne 0 ]; then
	ovs-vsctl add-br ${DATABRIDGE}
	ovs-vsctl add-port ${DATABRIDGE} ${DATAVLANDEV}
David Johnson's avatar
David Johnson committed
405
    fi
406 407 408

    grep "^auto ${DATAVLANDEV}$" /etc/network/interfaces
    if [ ! $? -eq 0 ]; then
409 410
	if [ $DISTRIB_MAJOR -lt 18 ]; then
	    cat <<EOF >> /etc/network/interfaces
411 412 413 414 415 416 417 418 419
auto ${DATAVLANDEV}
iface ${DATAVLANDEV} inet static
    #address 0.0.0.0
    up mkdir -p /var/run/emulab
    # Just touch it, don't put iface/inet/mac into it; the vlans atop this
    # device are being used natively by openstack.  So just let Emulab setup
    # to not setup any of these vlans.
    up touch /var/run/emulab/interface-done-$DATAPMAC
EOF
420
	else
David Johnson's avatar
David Johnson committed
421 422 423 424 425 426 427
	    cat <<EOF >/etc/systemd/network/${DATAVLANDEV}.network
[Match]
Name=${DATAVLANDEV}

[Network]
UseDHCP=no
EOF
David Johnson's avatar
David Johnson committed
428
	    cat <<EOF >>$OURDIR/testbed-pre-static-control-network.sh
429 430 431 432 433 434 435 436

mkdir -p /var/run/emulab
# Just touch it, don't put iface/inet/mac into it; the vlans atop this
# device are being used natively by openstack.  So just let Emulab setup
# to not setup any of these vlans.
touch /var/run/emulab/interface-done-$DATAPMAC
EOF
	fi
437
    fi
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
done

#else
#    ifconfig ${DATA_NETWORK_INTERFACE} $DATAIP netmask 255.0.0.0 up
#
#    cat <<EOF >> /etc/network/interfaces
#
#auto ${DATA_NETWORK_INTERFACE}
#iface ${DATA_NETWORK_INTERFACE} inet static
#    address $DATAIP
#    netmask $DATANETMASK
#EOF
#    if [ -n "$DATAVLANDEV" ]; then
#	cat <<EOF >> /etc/network/interfaces
#    vlan-raw-device ${DATAVLANDEV}
#EOF
#    fi
#fi
456

457 458 459
#
# Set the hostname for later after reboot!
#
460 461
hostname=`hostname`
echo $hostname > /etc/hostname
462

463
service_restart openvswitch-switch
464 465 466 467 468 469

ip route flush cache

# Just wait a bit
#sleep 8

470 471 472 473 474 475 476
# Some services (neutron-ovs-cleanup) might lookup the hostname prior to
# network being up.  We have to handle this here once at startup; then
# again later in the rc.hostnames hook below.
echo $ctlip $hostname >> /tmp/hosts.tmp
cat /etc/hosts >> /tmp/hosts.tmp
mv /tmp/hosts.tmp /etc/hosts

477 478 479 480 481 482 483 484 485 486
grep -q DYNRUNDIR /etc/emulab/paths.sh
if [ $? -eq 0 ]; then
    echo "*** Hooking Emulab rc.hostnames boot script..."
    mkdir -p $OURDIR/bin
    touch $OURDIR/bin/rc.hostnames-openstack
    chmod 755 $OURDIR/bin/rc.hostnames-openstack
    cat <<EOF >$OURDIR/bin/rc.hostnames-openstack
#!/bin/sh

cp -p $OURDIR/mgmt-hosts /var/run/emulab/hosts.head
David Johnson's avatar
David Johnson committed
487 488 489 490 491

# Some services (neutron-ovs-cleanup) might lookup the hostname prior to
# network being up.
echo $ctlip $hostname >> /var/run/emulab/hosts.head

492 493
exit 0
EOF
494 495 496 497

    mkdir -p /usr/local/etc/emulab/run/rcmanifest.d
    touch /usr/local/etc/emulab/run/rcmanifest.d/0.openstack-rcmanifest
    cat <<EOF >> /usr/local/etc/emulab/run/rcmanifest.d/0.openstack-rcmanifest
498 499 500 501 502 503 504
HOOK SERVICE=rc.hostnames ENV=boot WHENCE=every OP=boot POINT=pre FATAL=0 FILE=$OURDIR/bin/rc.hostnames-openstack ARGV="" 
EOF
else
    echo "*** Nullifying Emulab rc.hostnames and rc.ifconfig services!"
    mv /usr/local/etc/emulab/rc/rc.hostnames /usr/local/etc/emulab/rc/rc.hostnames.NO
    mv /usr/local/etc/emulab/rc/rc.ifconfig /usr/local/etc/emulab/rc/rc.ifconfig.NO
fi
505

506 507 508 509 510 511 512 513 514 515 516 517
if [ ! ${HAVE_SYSTEMD} -eq 0 ] ; then
    # Maybe this is helpful too
    update-rc.d networking remove
    update-rc.d networking defaults
    # This seems to block systemd from doing its job...
    systemctl disable ifup-wait-emulab-cnet.service
    systemctl mask ifup-wait-emulab-cnet.service
    systemctl stop ifup-wait-emulab-cnet.service
    #
    # XXX: fixup a systemd/openvswitch bug
    # https://bugs.launchpad.net/ubuntu/+source/openvswitch/+bug/1448254
    #
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
    #
    # Also, if our init is systemd, fixup the openvswitch service to
    # come up and go down before remote-fs.target .  Somehow,
    # openvswitch-switch always goes down way, way before the rest of
    # the network is brought down.  remote-fs.target seems to be one of
    # the last services to be killed before the network target is
    # brought down, and if there's an NFS mount, NFS might require
    # communication with the remote server to umount the mount.  This
    # affects us because there are Emulab/Cloudlab NFS mounts over the
    # control net device, and we bridge the control net device into the
    # br-ex openvswitch bridge.  To complete the story, once the
    # openvswitch-switch daemon goes down, you have about 30 seconds
    # before the bridge starts acting really flaky... it appears to go
    # down and quit forwarding traffic for awhile, then will pop back to
    # life periodically for 10-second chunks.  So, we hackily "fix" this
    # by making the openswitch-nonetwork service dependent on
    # remote-fs.target ... and since that target is one of the last to
    # go down before the real network is brought down, this seems to
    # work.  Ugh!  So to fix that, we also add the remote-fs.target
    # Before dependency to the "patch" listed in the above bug report.
    #
539 540 541 542 543 544
    cat <<EOF >/lib/systemd/system/openvswitch-nonetwork.service
    [Unit]
Description=Open vSwitch Internal Unit
PartOf=openvswitch-switch.service
DefaultDependencies=no
Wants=network-pre.target openvswitch-switch.service
545 546
Before=network-pre.target remote-fs.target
After=local-fs.target
547 548 549 550 551 552 553 554 555

[Service]
Type=oneshot
RemainAfterExit=yes
EnvironmentFile=-/etc/default/openvswitch-switch
ExecStart=/usr/share/openvswitch/scripts/ovs-ctl start \
          --system-id=random $OPTIONS
ExecStop=/usr/share/openvswitch/scripts/ovs-ctl stop
EOF
556

557
    systemctl enable openvswitch-switch
558
    systemctl daemon-reload
559 560
fi

561 562 563 564 565 566 567 568 569 570 571
#
# Install a basic ARP reply filter that prevents us from sending ARP replies on
# the control net for anything we're not allowed to use (i.e., we can reply for
# ourselves, and any public addresses we're allowed to use).  Really, we only
# need the public address part on the network manager, but may as well let
# any node reply as any public address we're allowed to use).
#

# Cheat and use our IPADDR/NETMASK instead of NETWORK/NETMASK below...
OURNET=`ip addr show br-ex | sed -n -e 's/.*inet \([0-9\.\/]*\) .*/\1/p'`
# Grab the port that corresponds to our
David Johnson's avatar
David Johnson committed
572
OURPORT=`ovs-ofctl show br-ex | sed -n -e "s/[ \t]*\([0-9]*\)(${EXTERNAL_NETWORK_INTERFACE}.*\$/\1/p"`
573

574 575 576 577 578 579 580 581 582
#
# Ok, make the anti-ARP spoofing rules live, and also place them in the right
# place to be picked up by our neutron openvswitch agent so that when it
# remove_all_flows() it also installs our "system" defaults.
#
mkdir -p /etc/neutron/ovs-default-flows
FF=/etc/neutron/ovs-default-flows/br-ex
touch ${FF}

583 584 585 586 587 588 589 590 591 592 593 594 595 596
#
# Huge hack.  Somewhere in Mitaka, something starts removing the first
# flow rule from the table (and that is the rule allowing our control
# net iface ARP replies to go out!).  So, put a simple rule at the head
# of the line that simply allows ARP replies from the local control net
# default gateway to arrive on our control net iface.  This rule is of
# course eclipsed by the "Allow any inbound ARP replies on the control
# network" rule below -- thus it is safe to allow this arbitrary process
# to delete.
#
FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=${ctlgw},in_port=${OURPORT},actions=NORMAL"
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF

597
FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=${ctlip},actions=NORMAL"
598 599 600
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF

601
for addr in $PUBLICADDRS ; do
602 603 604
    FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=${addr},actions=NORMAL"
    ovs-ofctl add-flow br-ex "$FLOW"
    echo "$FLOW" >> $FF
605 606
done
# Allow any inbound ARP replies on the control network.
607 608 609 610
FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=${OURNET},in_port=${OURPORT},actions=NORMAL"
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF

611
# Drop any other control network addr ARP replies on the br-ex switch.
612 613 614
FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=${OURNET},actions=drop"
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF
615

616 617 618 619
# Also, drop Emulab vnode control network addr ARP replies on br-ex!
FLOW="dl_type=0x0806,nw_proto=0x2,arp_spa=172.16.0.0/12,actions=drop"
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF
620

621 622 623 624 625 626 627 628 629 630 631 632 633 634
#
# A final hack.  These days (i.e. Pike), the neutron-openvswitch-agent
# is very aggressive to delete the default NORMAL flow on the br-ex
# bridge.  This causes problems for testbed.service on reboot, because
# connectivity effectively flaps as the NORMAL flow gets deleted and
# added.  So, we make a default NORMAL flow with our cookie, so it
# effectively won't be deleted.  Once the agent has initialized, its
# cookie will replace ours for this priority=0,actions=NORMAL flow, but
# that is fine.
#
FLOW="priority=0,actions=NORMAL"
ovs-ofctl add-flow br-ex "$FLOW"
echo "$FLOW" >> $FF

635 636
logtend "ovs-node"

637
exit 0