auto-pelab.ns 20.7 KB
Newer Older
Mike Hibler's avatar
Mike Hibler committed
1 2 3 4 5 6
#
# EMULAB-COPYRIGHT
# Copyright (c) 2006 University of Utah and the Flux Group.
# All rights reserved.
#

7 8 9 10 11
source tb_compat.tcl

set ns [new Simulator]
$ns rtproto Static

12 13 14
##########
# Beginning of user-settable options

15
#
16 17 18
# If set to 1, doesn't allocate any real Plab nodes (or fake ones either).
# Attributes for the elab-* machines are taken from DB data for nodes in
# the plabnodes list.
19
#
20 21 22 23
# This option requires that you have nodes listed in plabnodes and forces
# use of the dbmonitor.
#
set no_plab 0
24

25
#
26 27
# This control how many _pairs_ of PCs there are. ie. 2 gets you 2 monitors,
# plus matching stubs
28 29 30 31
#
set num_pcs 2

#
32
# If set to 1, we create a fake PlanetLab inside Emulab
33
#
34
set fake_plab 0
35

36 37 38 39
#
# If set to 1, we grab real PlanetLab nodes. Both this and fake_plab can be
# set at the same time
#
40
set real_plab 1
41

42
#
43 44 45 46 47 48 49 50 51 52 53 54
# Run FreeBSD 6.1 on elab_ nodes.  Forces use of dbmonitor.
#
set do_bsd 0

#
# If set to 1, we trace all links (real/fake plab, elab) at the delay nodes.
#
set trace 1

#
# If set to 1, we trace on all end nodes instead of (or in addition to)
# all delay nodes above.
55
#
56 57 58 59 60
# XXX this option won't do anything right now.  If there are delay nodes
# involved for local LAN/clouds it is ignored, and as long as tracing is
# enabled, plab nodes will get end_node tracing anyway.
#
set trace_end_nodes 0
61

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
#
# If you want to get specific real planetlab nodes, ask for them here by site
# or by emulab node ID. 
# NOTES:
#   An empty list for either selection method turns it off
#   Do not use both site and node selection at the same time!
#   The list must be at least as long as num_pcs above, It can be longer, but
#     extra elements will be ignored
#   I recommend using the sites option, rather than the nodes option, since that
#     may get you less-loaded nodes and is more robust to crashed nodes
#   You can get a list of node IDs and sites at:
#     https://www.emulab.net/nodecontrol_list.php3?showtype=widearea
#   Site names are case-sensitive!
#

# Empty list
set plabsites {}
# Example site list
#set plabsites {UCB Stanford Cornell Jerusalem Taiwan}

# Empty list
set plabnodes {}
# Example node list
85
#set plabnodes {plab518 plab541 plab628 plab736 plab360}
86

87 88 89 90
#
# Where to grab your tarball of pelab software from. To make this tarball:
#   Go to your testbed source tree
#   Run 'cvs up'!!!
91
#   Run 'gmake' in pelab/stub and pelab/libnetmon (on a DEVBOX node)
92 93 94 95 96 97 98
#   From the root of your source tree, run:
#       tar czvf /proj/tbres/my-pelab.tar.gz pelab
#   Of course, don't name it 'my-plab.tar.gz'! You can put this file in
#       a subdirectory if you want, but it must be somewhere in
#       /proj/tbres/
#   Put the path to your tarball in this variable
#
99
set pelab_tar "/proj/tbres/CHANGEME.tar.gz"
100
#set pelab_tar "/proj/tbres/mike/pelab-bin.tar.gz"
101

102

103 104 105 106 107 108 109
#
# When using a fake plab, these are the parameters for the fake Internet
# 'cloud' connecting the plab nodes. For now, all nodes have the same
# parameters - hopefully this will change in a later version
#
set cloud_delay "30ms"
set cloud_bw "1.5Mbps"
110 111 112 113 114 115 116 117

#
# When using a fake plab, these are the parameters for the 'control'
# delay (ie. the latency for the elab nodes to reach the plab nodes
#
set control_delay "0ms"
set control_bw "100Mbps"

118 119 120 121
#
# These are the initial conditions for the 'elabc' cloud, the Emulab side of
# a pelab experiment
#
Robert Ricci's avatar
Robert Ricci committed
122 123
set ecloud_delay "0ms"
set ecloud_bw "100Mbps"
124

125 126 127 128 129
#
# Hardare type to use for PCs inside of emulab
#
set hwtype "pc"

130 131 132 133
#
# Server and client to use for automated testing. If set, will automatically
# be started by the 'start-experiment' script
#
134
if {$do_bsd} {
135
    set sstr "/usr/local/etc/emulab/emulab-iperf -s "
136
    # NOTE: No client support for now, you'll have to run the client yourself
137
    set cstr "/usr/local/etc/emulab/emulab-iperf -t 60 -c "
138
} else {
139
    set sstr "/usr/bin/iperf -s "
140
    # NOTE: No client support for now, you'll have to run the client yourself
141
    set cstr "/usr/bin/iperf -t 60 -c "
142
}
143 144 145 146 147 148 149
set serverprog $sstr
set clientprog $cstr

#
# Application to run automatically on all nodes (planet, plab or elab)
#
set allnodesprog "/bin/true"
150

151 152
#
# If non-zero, uses the new stub (magent) instead of the old one
153 154
# If set to 1, uses the new stub as a receiver
# If set to 2, uses a separate iperfd to receive traffic
155
#
156
set use_magent 1
157

158 159 160 161
#
# If non-zero, uses the DB-based "monitor" to control the cloud shaping
#
set use_dbmonitor 0
162 163 164
if {$do_bsd} {
    set use_dbmonitor 1
}
165

166
#
167 168 169 170
# If dbmonitor is set, these are the intervals at which the latency and BW
# data are sampled by bgmon on the plab nodes (zero means use the default).
# The smaller (shorter interval) of the two, is used as the interval at which
# dbmonitor polls the DB to get data recorded by bgmon.
171
#
172 173 174 175 176 177
set dbmonitor_latinterval 10
set dbmonitor_bwinterval 0

# Get these into the environment for dbmonitor to read
set opt(DBMONITOR_LATINTERVAL) $dbmonitor_latinterval
set opt(DBMONITOR_BWINTERVAL) $dbmonitor_bwinterval
178

179 180 181 182 183
#
# If non-zero, limits the number of slots in the queues for the fake PlanetLab
#
set limit_fake_queue_slots 0

184 185
#
# Use this to set a unique port so that you don't collide with others on the
186
# same node. Only supported when use_magent is set
187 188 189
#
set stub_port 3149

190 191 192
# XXX for internal use only
set no_stub 0

193 194 195 196 197 198
#
# Enforce no_plab requirements
#
if {$no_plab} {
    set fake_plab 0
    set real_plab 0
199 200 201 202
# XXX don't force use of dbmonitor; may want to just use init_elabnodes.
#    set use_dbmonitor 1
    set use_magent 0
    set no_stub 1
203 204
}

205 206 207 208 209 210 211 212
#
# Enforce USE_DBMONITOR requirements
#
if {$use_dbmonitor} {
    set use_magent 0
    set no_stub 1
}

213 214 215
# End of user-settable options
##########

216 217 218
set pid ${GLOBALS::pid}
set eid ${GLOBALS::eid}

219
set delay_os FBSD54-DNODE
220

221 222 223 224 225 226 227
if {$do_bsd} {
    set node_os FBSD61-STD
} else {
    # XXX -UPDATE for now.  Contains bug fixes (progagent command lines).
    set node_os PLAB-DEVBOX-UPDATE
    #set node_os PLAB-DEVBOX
}
228 229 230

tb-set-delay-os $delay_os

231 232 233
set stub_peer_port $stub_port
set stub_command_port [expr $stub_port + 1]

234 235 236 237
#
# Tarballs and RPMs we install on all nodes
#
set tarfiles "/local $pelab_tar"
238 239 240 241 242 243 244
set plab_rpms "/proj/tbres/auto-pelab/libpcap-0.8.3-3.i386.rpm /proj/tbres/auto-pelab/iperf-2.0.2-1.1.fc2.rf.i386.rpm"
if {$do_bsd} {
    set elab_rpms ""
} else {
    set elab_rpms "/proj/tbres/auto-pelab/libpcap-0.8.3-3.i386.rpm /proj/tbres/auto-pelab/iperf-2.0.2-1.1.fc2.rf.i386.rpm"
}

245 246 247 248 249 250
if {$do_bsd} {
    set elabshell "/bin/sh -T"
} else {
    set elabshell "/bin/sh"
}

251
if {$use_magent} {
252
    set stubcommand "/bin/sh /local/pelab/magent/auto-magent.sh --peerserverport=$stub_peer_port --monitorserverport=$stub_command_port"
253 254 255 256
    # XXX woeful, and hopefully temporary, hack
    if {$use_magent == 2} {
	set opt(MAGENT_NORECV) 1
    }
257 258 259
} else {
    set stubcommand "/bin/sh /local/pelab/stub/auto-stub.sh"
}
260 261 262

# we don't run dbmonitor on the nodes anymore
if {0 && $use_dbmonitor} {
263
    set moncommand "$elabshell /local/pelab/dbmonitor/auto-dbmonitor.sh"
264
} else {
265
    set moncommand "$elabshell /local/pelab/monitor/auto-monitor.sh --stub-port=$stub_command_port"
266
}
267

268 269 270 271 272 273
#
# So far, nothing I tried can get the auto-* scripts to return 0
# when interrupted
#
set ecode 15

274 275
set elan_string ""
set plan_string ""
276
set inet_string ""
277 278

set stublist {}
279 280
set planetstublist {}
set plabstublist {}
281
set monitorlist {}
282 283 284
set planetservers {}
set serverlist {}
set clientlist {}
Mike Hibler's avatar
Mike Hibler committed
285
set tflist {}
286
set anlist {}
287 288 289 290

#
# Create all of the nodes
#
Mike Hibler's avatar
Mike Hibler committed
291
set tfix 1
292
set anix 1
293 294
for {set i 1} {$i <= $num_pcs} {incr i} {

295 296
    if {$real_plab} {
        set planet($i) [$ns node]
297
        tb-set-hardware $planet($i) pcplab
298
        append inet_string "$planet(${i}) "
299
        set planetstub($i) [$planet($i) program-agent -expected-exit-code $ecode -command $stubcommand]
300 301
        lappend stublist $planetstub($i)
        lappend planetstublist $planetstub($i)
302

303
        tb-set-node-tarfiles $planet($i) $tarfiles
304
        tb-set-node-rpms $planet($i) $plab_rpms
305

306
        set tfupdate($tfix) [$planet($i) program-agent -command "sudo /usr/local/etc/emulab/update -it"]
Mike Hibler's avatar
Mike Hibler committed
307 308
	lappend tflist $tfupdate($tfix)
	incr tfix
309 310 311
	set allnodes($anix) [$planet($i) program-agent -command $allnodesprog]
	lappend anlist $allnodes($anix)
	incr anix
312 313 314 315 316 317 318 319

        if {[llength $plabsites] > 0} {
            set why_doesnt_tcl_have_concat "*&"
            append why_doesnt_tcl_have_concat [lindex $plabsites [expr $i - 1]]
            $planet($i) add-desire $why_doesnt_tcl_have_concat 1.0
        } elseif {[llength $plabnodes] > 0} {
            tb-fix-node $planet($i) [lindex $plabnodes [expr $i - 1]]
        }
320
    }
321

322 323
    if {$fake_plab} {
        set plab($i) [$ns node]
324
        tb-set-node-os $plab($i) $node_os
325 326
        tb-set-hardware $plab($i) $hwtype
        append plan_string "$plab(${i}) "
327
        set plabstub($i) [$plab($i) program-agent -expected-exit-code $ecode -command $stubcommand]
328 329 330 331
        lappend stublist $plabstub($i)
        lappend plabstublist $plabstub($i)

        tb-set-node-tarfiles $plab($i) $tarfiles
332
        tb-set-node-rpms $plab($i) $plab_rpms
333
        set tfupdate($tfix) [$plab($i) program-agent -command "sudo /usr/local/etc/emulab/update -it"]
Mike Hibler's avatar
Mike Hibler committed
334 335
	lappend tflist $tfupdate($tfix)
	incr tfix
336 337 338
	set allnodes($anix) [$plab($i) program-agent -command $allnodesprog]
	lappend anlist $allnodes($anix)
	incr anix
339
    }
340 341

    set elab($i) [$ns node]
342
    tb-set-node-os $elab($i) $node_os
343 344
    tb-set-hardware $elab($i) $hwtype
    append elan_string "$elab(${i}) "
345
    set monitor($i) [$elab($i) program-agent -expected-exit-code $ecode -command $moncommand]
346 347
    lappend monitorlist $monitor($i)

348 349
    set server($i) [$elab($i) program-agent -expected-exit-code $ecode -command $serverprog]
    set client($i) [$elab($i) program-agent -expected-exit-code $ecode -command $clientprog]
350 351 352 353
    lappend serverlist $server($i)
    lappend clientlist $client($i)

    tb-set-node-tarfiles $elab($i) $tarfiles
354 355 356
    if {$elab_rpms != ""} {
        tb-set-node-rpms $elab($i) $elab_rpms
    }
357
    set tfupdate($tfix) [$elab($i) program-agent -command "sudo /usr/local/etc/emulab/update -it"]
Mike Hibler's avatar
Mike Hibler committed
358 359
    lappend tflist $tfupdate($tfix)
    incr tfix
360 361 362
    set allnodes($anix) [$elab($i) program-agent -command $allnodesprog]
    lappend anlist $allnodes($anix)
    incr anix
363

364 365 366 367 368 369
    #
    # If the user has given a plabnodes list, we set up $opt variables so that
    # even if they are not actually using PlanetLab nodes, the simple model can
    # still see the mapping between elab and plab nodes
    #
    if {[llength $plabnodes] > 0} {
370
        set opt(pelab-elab-$i-mapping) [lindex $plabnodes [expr $i - 1]]
371
    }
372 373
}

374 375
#
# Run the DB monitor on ops
376
# Intervals are passed via the environment.
377 378 379 380
#
if {$use_dbmonitor} {
    set dbmonitor [new Program $ns]
    $dbmonitor set node "ops"
381
    $dbmonitor set command "/usr/testbed/sbin/dbmonitor.pl $pid $eid"
382 383 384 385
    $dbmonitor set expected-exit-code $ecode
    set monitorlist $dbmonitor
}

386 387 388
#
# Set up groups to make it easy for us to start/stop program agents
#
389 390 391 392 393 394 395 396
if {$stublist != {}} {
    set stubgroup [$ns event-group $stublist]
    if {$real_plab} {
        set planetstubs [$ns event-group $planetstublist]
    }
    if {$fake_plab} {
        set plabstubs [$ns event-group $plabstublist]
    }
397
}
398
set monitorgroup [$ns event-group $monitorlist]
399 400 401

set allservers [$ns event-group $serverlist]
set allclients [$ns event-group $clientlist]
402

Mike Hibler's avatar
Mike Hibler committed
403
set tfhosts [$ns event-group $tflist]
404
set anhosts [$ns event-group $anlist]
405 406

set tracelist {}
Mike Hibler's avatar
Mike Hibler committed
407

408 409 410 411
#
# Real Internet cloud for real plab nodes
#
if {$real_plab} {
412
    set realinternet [$ns make-lan "$inet_string" 100Mbps 0ms]
413
    tb-set-lan-protocol $realinternet ipv4
414
    if {$trace} {
415
	$realinternet trace header "tcp and port $stub_peer_port"
416 417 418 419 420
	if {$trace_end_nodes} {
	    $realinternet trace_endnode 1
	}
	lappend tracelist $realinternet
    }
421 422
}

423
#
424
# Fake 'Internet' cloud for fake plab nodes
425 426
#
if {$fake_plab} {
Robert Ricci's avatar
Robert Ricci committed
427
    set plabc [$ns make-cloud "$plan_string" $cloud_bw $cloud_delay]
428
    tb-set-ip-lan $plab(1) $plabc 10.1.0.1
429 430 431 432 433 434 435
    if {$trace} {
	$plabc trace
	if {$trace_end_nodes} {
	    $plabc trace_endnode 1
	}
	lappend tracelist $plabc
    }
436 437 438 439 440 441
    if {$limit_fake_queue_slots} {
        for {set i 1} {$i <= $num_pcs} {incr i} {
            set fakequeues($i) [[$ns lanlink $plabc $plab($i)] queue]
            $fakequeues($i) set limit_ $limit_fake_queue_slots
        }
    }
442 443 444 445 446
}

#
# Lan which will be controlled by the monitor
#
Robert Ricci's avatar
Robert Ricci committed
447
set elabc [$ns make-cloud "$elan_string" $ecloud_bw $ecloud_delay]
448
tb-set-ip-lan $elab(1) $elabc 10.0.0.1
449 450 451 452 453 454 455
if {$trace} {
    $elabc trace
    if {$trace_end_nodes} {
	$elabc trace_endnode 1
    }
    lappend tracelist $elabc
}
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470

#
# We don't want the sync server to end up out there on some plab node
#
tb-set-sync-server $elab(1)

#
# Set up a fake Internet link between the PlanetLab and Emulab sides
# when using fake plab nodes
#
if {$fake_plab} {
    set erouter [$ns node]
    set prouter [$ns node]

    set elabcontrol [$ns make-lan "$elan_string $erouter" 100Mbps 0ms]
471
    set plabcontrol [$ns make-lan "$plan_string $prouter" 100Mbps 0ms]
472 473

    set internet [$ns duplex-link $erouter $prouter $control_bw $control_delay DropTail]
474 475 476 477 478 479 480
    if {$trace} {
	$internet trace
	if {$trace_end_nodes} {
	    $internet trace_endnode 1
	}
	lappend tracelist $internet
    }
481 482

    tb-set-ip-lan $elab(1) $elabcontrol 192.168.0.1
483
    tb-set-ip-lan $plab(1) $plabcontrol 192.168.1.1
484 485 486 487 488

    tb-set-ip-link $erouter $internet 192.168.254.1
    tb-set-ip-link $prouter $internet 192.168.254.2
}

489
# for one-off ops commands
490 491 492 493 494 495 496 497 498 499 500 501 502
set opsagent [new Program $ns]
$opsagent set node "ops"
$opsagent set command "/bin/true"

#
# The set of traced links
# XXX still cannot use an event-group of link tracing events in a sequence
#
#set tracegroup [$ns event-group $tracelist]

#
# Build up the event sequences to start and stop an actual run.
#
503
if {$no_plab || $real_plab} {
504 505 506 507 508
  set start [$ns event-sequence]
    $start append "$ns log \"Starting REAL plab experiment\""

    # stop stubs and monitors
    $start append "$ns log \"##### Stopping stubs and monitors...\""
509
    if {!$no_stub && $real_plab} {
510 511
	$start append "$planetstubs stop"
    }
512 513 514
    $start append "$monitorgroup stop"

    # stop servers
515 516 517 518
    if {$serverprog != ""} {
	$start append "$ns log \"##### Stopping servers...\""
	$start append "$allservers stop"
    }
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536

    # stop link logging
    $start append "$ns log \"##### Roll link trace logs...\""
    foreach link $tracelist {
        $start append "$link trace snapshot"
        $start append "$link trace stop"
    }

    # clean out log files
    # XXX original script passed --root, cleanlogs does not--may be a problem.
    $start append "$ns log \"##### Cleaning logs...\""
    $start append "$ns cleanlogs"

    # reset shaping characteristics for all nodes
    $start append "$ns log \"##### Resetting links...\""
    $start append "$elabc clear"
    $start append "$elabc reset"

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
    if {$real_plab} {
	# distinguish between real/fake runs
	# XXX I'm thinkin...we can do better than this!
	$start append "$opsagent run -command \"cp /dev/null /proj/$pid/exp/$eid/tmp/real_plab\""

	# save off node list
	$start append "$ns log \"##### Creating node list...\""
	$start append "$opsagent run -command \"/usr/testbed/bin/node_list -m -e $pid,$eid > /proj/$pid/exp/$eid/tmp/node_list\""

	# initialize path characteristics
	$start append "$ns log \"##### Initialize emulation node path characteristics...\""
	$start append "$elabc create"
	$start append "$opsagent run -command \"/usr/testbed/bin/init-elabnodes.pl  -o /proj/$pid/exp/$eid/tmp/initial-conditions.txt $pid $eid\""
    } else {
	$start append "$elabc create"
    }
553 554 555 556 557 558 559 560 561 562 563

    # restart link tracing
    # XXX cleanlogs has unlinked the current trace log, so we have to
    # snapshot to bring it back into existence
    $start append "$ns log \"##### Starting link tracing...\""
    foreach link $tracelist {
        $start append "$link trace snapshot"
        $start append "$link trace start"
    }

    # restart servers
564 565 566 567
    if {$serverprog != ""} {
	$start append "$ns log \"##### Starting server...\""
	$start append "$allservers start"
    }
568 569 570

    # restart stubs and monitors
    $start append "$ns log \"##### Starting stubs and monitors...\""
571
    if {!$no_stub && $real_plab} {
572 573
	$start append "$planetstubs start"
    }
574 575 576 577 578 579 580 581 582 583 584 585
    $start append "$monitorgroup start"

    # gather up the data and inform the user
    $start append "$ns log \"##### Experiment run started!\""
    # XXX cannot do a report here as that will cause the logs to be
    #     deleted before the next loghole sync

  set stop [$ns event-sequence]
    $stop append "$ns log \"Stopping REAL plab experiment\""

    # stop stubs and monitors
    $stop append "$ns log \"##### Stopping stubs and monitors...\""
586
    if {!$no_stub && $real_plab} {
587 588
	$stop append "$planetstubs stop"
    }
589 590 591
    $stop append "$monitorgroup stop"

    # stop servers
592 593 594 595
    if {$serverprog != ""} {
	$stop append "$ns log \"##### Stopping servers...\""
	$stop append "$allservers stop"
    }
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624

    # stop link logging and save logs
    $stop append "$ns log \"##### Stop link tracing...\""
    foreach link $tracelist {
        $stop append "$link trace snapshot"
        $stop append "$link trace stop"
    }

    # reset shaping characteristics for all nodes
    $stop append "$ns log \"##### Resetting links...\""
    $stop append "$elabc clear"
    $stop append "$elabc reset"

    # gather up the data and inform the user
    $stop append "$ns log \"##### Experiment run stopped!\""
    $stop append "$ns report"
}

#
# Build up the event sequences to start and stop an actual run on FAKE
# plab nodes.  The only difference here (besides the names) is that these
# sequences use plabstubs rather than planetstubs.
#
if {$fake_plab} {
  set start_fake [$ns event-sequence]
    $start_fake append "$ns log \"Starting FAKE plab experiment\""

    # stop stubs and monitors
    $start_fake append "$ns log \"##### Stopping stubs and monitors...\""
625 626 627
    if {!$no_stub} {
	$start_fake append "$plabstubs stop"
    }
628 629 630
    $start_fake append "$monitorgroup stop"

    # stop servers
631 632 633 634
    if {$serverprog != ""} {
	$start_fake append "$ns log \"##### Stopping servers...\""
	$start_fake append "$allservers stop"
    }
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670

    # stop link logging
    $start_fake append "$ns log \"##### Roll link trace logs...\""
    foreach link $tracelist {
        $start_fake append "$link trace snapshot"
        $start_fake append "$link trace stop"
    }

    # clean out log files
    # XXX original script passed --root, cleanlogs does not--may be a problem.
    $start_fake append "$ns log \"##### Cleaning logs...\""
    $start_fake append "$ns cleanlogs"

    # reset shaping characteristics for all nodes
    $start_fake append "$ns log \"##### Resetting links...\""
    $start_fake append "$elabc clear"
    $start_fake append "$elabc reset"

    # distinguish between real/fake runs
    # XXX I'm thinkin...we can do better than this!
    $start_fake append "$opsagent run -command \"rm -f /proj/$pid/exp/$eid/tmp/real_plab\""

    # save off node list
    $start_fake append "$ns log \"##### Creating node list...\""
    $start_fake append "$opsagent run -command \"/usr/testbed/bin/node_list -m -e $pid,$eid > /proj/$pid/exp/$eid/tmp/node_list\""

    # restart link tracing
    # XXX cleanlogs has unlinked the current trace log, so we have to
    # snapshot to bring it back into existence
    $start_fake append "$ns log \"##### Starting link tracing...\""
    foreach link $tracelist {
        $start_fake append "$link trace snapshot"
        $start_fake append "$link trace start"
    }

    # restart servers
671 672 673 674
    if {$serverprog != ""} {
	$start_fake append "$ns log \"##### Starting server...\""
	$start_fake append "$allservers start"
    }
675 676 677

    # restart stubs and monitors
    $start_fake append "$ns log \"##### Starting stubs and monitors...\""
678 679 680
    if {!$no_stub} {
	$start_fake append "$plabstubs start"
    }
681 682 683 684 685 686 687 688 689 690 691 692
    $start_fake append "$monitorgroup start"

    # gather up the data and inform the user
    $start_fake append "$ns log \"##### Experiment run started!\""
    # XXX cannot do a report here as that will cause the logs to be
    #     deleted before the next loghole sync

  set stop_fake [$ns event-sequence]
    $stop_fake append "$ns log \"Stopping FAKE plab experiment\""

    # stop stubs and monitors
    $stop_fake append "$ns log \"##### Stopping stubs and monitors...\""
693 694 695
    if {!$no_stub} {
	$stop_fake append "$plabstubs stop"
    }
696 697 698
    $stop_fake append "$monitorgroup stop"

    # stop servers
699 700 701 702
    if {$serverprog != ""} {
	$stop_fake append "$ns log \"##### Stopping servers...\""
	$stop_fake append "$allservers stop"
    }
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720

    # stop link logging and save logs
    $stop_fake append "$ns log \"##### Stop link tracing...\""
    foreach link $tracelist {
        $stop_fake append "$link trace snapshot"
        $stop_fake append "$link trace stop"
    }

    # reset shaping characteristics for all nodes
    $stop_fake append "$ns log \"##### Resetting links...\""
    $stop_fake append "$elabc clear"
    $stop_fake append "$elabc reset"

    # gather up the data and inform the user
    $stop_fake append "$ns log \"##### Experiment run stopped!\""
    $stop_fake append "$ns report"
}

721
$ns run