tbadb_serv 41.7 KB
Newer Older
1
#!/usr/bin/perl -wT
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24

#
# Copyright (c) 2016 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
25
package TBADB::Server;
26 27 28

use strict;
use English;
29 30
use DB_File;
use IO::Socket::INET;
31
use base qw(Net::Server::Fork);
32

33 34 35
# Drag in Emulab clientside path stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

36
use libtestbed;
37
use libtmcc qw(tmccbossinfo);
38 39
use libjsonrpc;
use tbadb_rpc;
40

41
# Function prototypes for non-object procedures.
42 43 44 45
sub rpc_checkimage($);
sub rpc_loadimage($);
sub rpc_captureimage($);
sub rpc_reboot($);
46
sub rpc_forward($);
47
sub rpc_unforward($);
Kirk Webb's avatar
Kirk Webb committed
48
sub rpc_nodewait($);
49 50
sub rpc_ping($);
sub rpc_exit($);
51 52
sub send_error($$$);
sub do_lru_cleanup();
53
sub unpack_bundle($$);
54 55 56 57
sub enter_fastboot($);
sub load_android_image($$$);
sub reboot_android($);
sub wait_for_android($);
58
sub setup_android_forward($$;$$);
59
sub remove_android_forward($;$);
60
sub check_adb();
61

62
# RPC function dispatch table
63
my %DISPATCH = (
64 65
    'lockimage'    => \&rpc_lockimage,
    'unlockimage'  => \&rpc_unlockimage,
66 67 68 69
    'checkimage'   => \&rpc_checkimage,
    'loadimage'    => \&rpc_loadimage,
    'captureimage' => \&rpc_captureimage,
    'reboot'       => \&rpc_reboot,
70
    'forward'      => \&rpc_forward,
71
    'unforward'    => \&rpc_unforward,
Kirk Webb's avatar
Kirk Webb committed
72
    'nodewait'     => \&rpc_nodewait,
73 74 75 76
    'ping'         => \&rpc_ping,
    'exit'         => \&rpc_exit,
);

77 78
# Constants
my $DEF_LOGLEVEL = 2;
79 80
my $PIDFILE = "/var/run/tbadb_serv.pid";
my $LOGFILE = "$LOGDIR/tbadb_serv.log";
81 82
my $MAPFILE = "$ETCDIR/tbadbmap";
my $FWDDBFILE = "$BOOTDIR/tbadbfwd.db";
83
my $ADB = "/usr/bin/adb";
84
my $FASTBOOT = "/usr/bin/fastboot";
85
my $TOUCH = "/usr/bin/touch";
86
my $IPTABLES = "/sbin/iptables";
87
my $HOST = "/usr/bin/host";
88 89 90
my $FILE = "/usr/bin/file";
my $RM = "/bin/rm";
my $UNZIP = "/usr/bin/unzip";
91 92
my $PS = "/bin/ps";
my $GREP = "/bin/grep";
93
my $IMAGE_CACHE = "/z/tbadb_img_cache";
94
my $IMAGE_SYSDIR = "$IMAGE_CACHE/PNSYSTEM";
95 96
my $WM_HIGH = 50 * 1000 * 1000 * 1000;  # 50 GB
my $WM_LOW  = 40 * 1000 * 1000 * 1000;  # 40 GB
97
my $ADBD_LISTENPORT = 5555;
98 99
my $MINPORT = 8001;
my $MAXPORT = 8100;
100
my $HOUSEKEEPING_INTERVAL = 60;
101
my $FASTBOOT_TMO = 15;
Kirk Webb's avatar
Kirk Webb committed
102
my $ANDROID_BOOT_TMO = 90;
103 104 105
my $IMGLOCK_TMO = 300;
my $LRULOCK_TMO = 60;
my $UNPACKLOCK_TMO = 30;
106
my $FWDLOCK_TMO = 30;
107

108 109 110 111 112 113 114 115 116
# Android partition info
my @ANDROID_PARTITIONS = (
    ["recovery", undef, 0],
    ["boot", undef, 1],
    ["system", undef, 1],
    ["userdata", "$IMAGE_SYSDIR/empty-userdata.img", 1],
    ["cache", "$IMAGE_SYSDIR/empty-cache.img", 1],
);

117 118
# Global variables
my %NMAP = ();
119
my %FWDPORTS = ();
120 121
my $RPCIN;
my $RPCOUT;
122 123 124 125
my $g_imglock;
my $debug = 0;

# Invoke the parent Net::Server class' run routine.
126
TBADB::Server->run({
127
    port       => TBADB_PORT,
128 129 130 131 132 133 134 135
    log_file   => $LOGFILE,
    log_level  => $DEF_LOGLEVEL,
    ipv        => 4,
    pid_file   => $PIDFILE,
    user       => "root",
    group      => "root",
    background => 1,
    setsid     => 1,
136
    no_client_stdout => 1,
137 138
    max_dequeue => 1,
    check_for_dequeue => $HOUSEKEEPING_INTERVAL,
139
});
140

141
##############################################################################
142
#
143
# Our Net::Server subclass override methods follow
144
#
145 146 147 148 149 150 151 152 153 154 155 156 157

#
# Add custom options to Net::Server object
#
sub options($$) {
    my ($self, $template) = @_;
    my $prop = $self->{'server'};

    # Let parent class setup its options.
    $self->SUPER::options($template);

    $prop->{'debug'} ||= undef;
    $template->{'debug'} = \ $prop->{'debug'};
158
}
159 160 161 162

#
# Do some overriding of config variables when debugging is requested
#
163
sub post_configure {
164 165 166 167 168 169
    my $self = shift;
    my $prop = $self->{'server'};

    # Don't go into the background if debugging was requested.  
    # Increase log level.
    if ($prop->{'debug'}) {
170
	warn "tbadb_serv: debug mode requested.  Staying in foreground.\n";
171 172 173 174
	$debug = $prop->{'debug'};  # XXX global var for non-OO funcs.
	$prop->{'background'} = 0;
	$prop->{'setsid'} = 0;
	$prop->{'log_level'} = 4;
175
	$prop->{'log_file'} = undef;
176
    }
177 178

    $self->SUPER::post_configure(@_);
179 180 181
}

#
182 183 184
# Do a bit of post-processing/checking after internal Net::Server
# configuration stage (which includes going into the background,
# setting up logging, etc.)
185 186 187 188 189 190 191
#
sub post_configure_hook($) {
    my $self = shift;
    my $prop = $self->{'server'};

    # Get IP for boss server and add it to the allowed list.
    my (undef, $bossip) = tmccbossinfo();
192
    die "tbadb_serv: Could not get IP address for boss server!\n"
193 194 195 196
	if (!$bossip || $bossip !~ /^(\d+\.\d+\.\d+\.\d+)$/);
    push @{$prop->{'cidr_allow'}}, "${1}/32";

    # Only allow root to run the script.
197
    die("tbadb_serv: May only be run as root!\n")
198 199 200
	if ($UID != 0);

    # Read in the node_id -> serial number map (used by non-OO funcs).
201
    die "tbadb_serv: Cannot run without serial number mapping file: $MAPFILE\n"
202 203
	if (!-r $MAPFILE);
    open(MFILE, "<$MAPFILE")
204
	or die "tbadb_serv: Can't open map file: $MAPFILE: $!\n";
205 206 207 208
    while (my $ln = <MFILE>) {
	chomp $ln;
	next if (!$ln || $ln =~ /^\s*#.*$/);
	if ($ln !~ /^\s*([-\w]+)\s+([a-zA-Z0-9]+)\s*$/) {
209
	    warn "tbadb_serv: malformed node mapping line: $ln\n";
210 211 212 213
	    next;
	}
	$NMAP{$1} = $2;
	$NMAP{$2} = $1;
214
    }
215
    close(MFILE);
216

217 218 219
    # Check/fix adb server and forwarding rules.
    die "tbadb_serv: Startup checks failed!\n"
	if !check_adb();
220 221 222 223 224 225 226 227 228
}

#
# Connection data handler called in the child worker processes forked by
# the parent Net::Server::Fork process.
#
sub process_request($) {
    my $self = shift;

229 230 231
    # Setup the RPC handles;
    $RPCIN  = $self->{'server'}->{'client'};
    $RPCOUT = $RPCIN;
232 233

    # Send "hello" To let remote end know we are ready.
234
    die "tbadb_serv: Could not send hello to caller. Terminating connection.\n"
235
	if !SendRPCData($RPCOUT, EncodeResult(-1, { HELLO => 1 }));
236 237 238 239

    while (1) {
	# Get PDU
	my $pdu;
240
	my $rcode = RecvRPCData($RPCIN, \$pdu);
241
	if ($rcode == -1) {
242
	    warn "tbadb_serv: timed out waiting for RPC data. Terminating connection\n";
243 244 245
	    exit 1;
	}
	elsif ($rcode == 0) {
246
	    warn "tbadb_serv: EOF from RPC pipe. Terminating connection\n";
247 248
	    exit 1;
	}
249
    
250 251 252
	# Decode PDU
	my $data = DecodeRPCData($pdu);
	if (!$data) {
253
	    warn "tbadb_serv: unable to decode RPC data. Terminating connection.\n";
254 255
	    exit 1;
	}
256

257 258 259 260
	# Dispatch function calls
	my $func = $data->{FUNCTION};
	if (defined($func)) {
	    if (!exists($DISPATCH{$func})) {
261
		warn "tbadb_serv: Unkown RPC: $func. Sending error and terminating connection.\n";
262 263 264 265 266
		send_error($data->{FID}, RPCERR_BADFUNC, "Unknown function: $func");
		exit 1;
	    }
	    $DISPATCH{$func}->($data);
	} else {
267
	    warn "tbadb_serv: Received RPC data that was not a function call. Terminating connection.\n";
268 269 270 271 272
	    exit 1;
	}
    }
}

273 274 275 276 277 278 279
#
# Periodically do housekeeping tasks.  This includes babysitting the
# adb forwarding setup (ensuring persistence).
#
sub run_dequeue($) {
    my $self = shift;

280 281
    warn "tbadb_serv: ADB server check failed!\n"
	if (!check_adb());
282 283
}

284 285 286 287
##############################################################################
#
# RPC dispatch functions follow.
#
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
sub rpc_lockimage($) {
    my ($data) = @_;
    my $proj    = $data->{ARGS}->{IMG_PROJ};
    my $srcname = $data->{ARGS}->{IMG_NAME};

    # Arg checking and untainting.
    if (!$proj || !$srcname) {
	warn "tbadb_serv::rpc_lockimage: Argument(s) missing from RPC!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Argument(s) missing.");
	exit 1;
    }
    if ($proj !~ /^([-\w]+)$/) {
	warn "tbadb_serv::rpc_lockimage: Malformed project argument!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
	exit 1;
    }
    $proj = $1;
    if ($srcname !~ /^([-\.\w]+)$/) {
	warn "tbadb_serv::rpc_lockimage: Malformed image name argument!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed image name argument.");
	exit 1;
    }
    $srcname = $1;

    warn "tbadb_serv::rpc_lockimage: Locking image: $proj/$srcname\n";
    my $lockfile = "/tmp/${proj}-${srcname}.imglock";
    my $start = time();
    while (1) {
	last
	    if (sysopen(LOCK, $lockfile, O_RDWR|O_CREAT|O_EXCL));
	if (time() - $start > $IMGLOCK_TMO) {
	    warn "tbadb_serv::rpc_lockimage: timed out trying to get image lock for $proj/$srcname!\n";
	    send_error($data->{FID}, RPCERR_BADARGS, "Timed out waiting to acquire image lock.");
	    exit 1;
	}
	sleep 5;
    }
    close(LOCK);

    # Send success result back to caller.
    warn "tbadb_serv::rpc_lockimage: finished locking image: $proj/$srcname\n";
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
    return;
}


sub rpc_unlockimage($) {
    my ($data) = @_;
    my $proj    = $data->{ARGS}->{IMG_PROJ};
    my $srcname = $data->{ARGS}->{IMG_NAME};

    # Arg checking and untainting.
    if (!$proj || !$srcname) {
	warn "tbadb_serv::rpc_unlockimage: Argument(s) missing from RPC!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Argument(s) missing.");
	exit 1;
    }
    if ($proj !~ /^([-\w]+)$/) {
	warn "tbadb_serv::rpc_unlockimage: Malformed project argument!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
	exit 1;
    }
    $proj = $1;
    if ($srcname !~ /^([-\.\w]+)$/) {
	warn "tbadb_serv::rpc_unlockimage: Malformed image name argument!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed image name argument.");
	exit 1;
    }
    $srcname = $1;

    warn "tbadb_serv::rpc_unlockimage: Unlocking image: $proj/$srcname\n";
    my $lockfile = "/tmp/${proj}-${srcname}.imglock";
    if (!unlink($lockfile)) {
	warn "tbadb_serv::rpc_unlockimage: Could not remove image lock file: $lockfile: $!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Could not remove image lock file.");
	exit 1;
    }

    # Send success result back to caller.
    warn "tbadb_serv::rpc_unlockimage: finished unlocking image: $proj/$srcname\n";
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
    return;
}

372 373 374 375 376 377 378 379

sub rpc_checkimage($) {
    my ($data) = @_;
    my $proj    = $data->{ARGS}->{IMG_PROJ};
    my $srcname = $data->{ARGS}->{IMG_NAME};
    my $srctime = $data->{ARGS}->{IMG_TIME};
    my $srcsize = $data->{ARGS}->{IMG_SIZE};

380
    # Arg checking and untainting.
381
    if (!$proj || !$srcname || !$srctime || !defined($srcsize)) {
382
	warn "tbadb_serv::rpc_checkimage: Argument(s) missing from RPC!\n";
383 384 385
	send_error($data->{FID}, RPCERR_BADARGS, "Argument(s) missing.");
	exit 1;
    }
386
    if ($proj !~ /^([-\w]+)$/) {
387
	warn "tbadb_serv::rpc_checkimage: Malformed project argument!\n";
388 389 390 391 392
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
	exit 1;
    }
    $proj = $1;
    if ($srcname !~ /^([-\.\w]+)$/) {
393
	warn "tbadb_serv::rpc_checkimage: Malformed image name argument!\n";
394 395 396 397
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed image name argument.");
	exit 1;
    }
    $srcname = $1;
398

399 400
    # Caller should have called the "lockimage" RPC already before calling
    # checkimage().  XXX: Maybe we should require a token to prove this.
401
    warn "tbadb_serv::rpc_checkimage: Image check requested for $proj/$srcname\n";
402
    my $projdir = "$IMAGE_CACHE/$proj";
403 404
    if (!-d $projdir) {
	if (!mkdir($projdir, 0750)) {
405
	    warn "tbadb_serv::rpc_checkimage: Failed to create project image directory: $projdir: $!\n";
406 407 408 409 410 411 412 413 414 415 416 417 418
	    send_error($data->{FID}, RPCERR_INTERNAL, "Internal file handling error.");
	    exit 1;
	}
    }

    my $lrufile = "$projdir/.$srcname.lru";
    my $imgfile = "$projdir/$srcname";
    if (-f $imgfile) {
	# Was the cached image placed (created) more recently than the
	# timestamp provided for the upstream version? Is the size different?
	my @imstats = stat($imgfile);
	my $imtime = $imstats[9];
	my $imsize = $imstats[7];
419 420 421 422
	warn "rpc_checkimage: UPSTREAM stats: $srcname, $srctime, $srcsize\n"
	    if $debug;
	warn "rpc_checkimage: LOCAL stats:    $srcname, $imtime, $imsize\n"
	    if $debug;
423
	if ($imtime >= $srctime && $imsize == $srcsize) {
424
	    if (!SendRPCData($RPCOUT, EncodeResult($data->{FID}, { NEED_IMG => 0 }))) {
425
		warn "tbadb_serv::rpc_checkimage: Error sending RPC result. Exiting!\n";
426 427 428 429 430 431
		exit 1;
	    }
	    return;
	} else {
	    # Delete older existing image to make way for new version.
	    if (!unlink($imgfile)) {
432
		warn "tbadb_serv::rpc_checkimage: Could not remove old image: $imgfile\n";
433 434 435 436 437 438 439 440 441
		send_error($data->{FID}, RPCERR_INTERNAL, "Failed to remove older version of image.");
		exit 1;
	    }
	}
    }

    # Touch image's LRU file (even though the image may not have been
    # transferred yet; presumably it will be shortly).
    if (system($TOUCH, "$lrufile") != 0) {
442
	warn "tbadb_serv::rpc_checkimage: Could not update timestamp on $lrufile!\n";
443 444 445 446 447 448
	send_error($data->{FID}, RPCERR_INTERNAL, "Internal file handling error.");
	exit 1;
    }

    # Check to see if we are over quota, and LRU prune if so.
    if (!do_lru_cleanup()) {
449
	warn "tbadb_serv::rpc_checkimage: Error cleaning up cache. Exiting!\n";
450 451 452 453 454 455 456 457 458
	send_error($data->{FID}, RPCERR_INTERNAL, 
		   "Failed while cleaning up image cache.");
	exit 1;
    }

    # If we need/want to enforce some concurrency limits from this side,
    # we could send back a "WAIT" result here, which would tell the caller
    # to wait for some amount of time (maybe we specify), then call again.

459
    # Tell caller that we need the image.
460
    if (!SendRPCData($RPCOUT,
461 462 463
	     EncodeResult($data->{FID}, 
			  { NEED_IMG => 1, 
			    REMOTE_PATH => "$projdir" }))) {
464
	warn "tbadb_serv::rpc_checkimage: Error sending RPC result. Exiting!\n";
465 466 467 468 469 470 471 472
	exit 1;
    }
    return;
}


sub rpc_loadimage($) {
    my ($data) = @_;
473 474 475
    my $node_id     = $data->{ARGS}->{NODE_ID};
    my $bundle_name = $data->{ARGS}->{IMG_NAME};
    my $proj        = $data->{ARGS}->{IMG_PROJ};
476

477
    # Check and untaint arguments
478
    if (!$bundle_name || !$node_id || !$proj) {
479
	warn "tbadb_serv::rpc_loadimage: missing RPC arguments!\n";
480 481 482
	send_error($data->{FID}, RPCERR_BADARGS, "Missing arguments.");
	exit 1;
    }
483
    if ($proj !~ /^([-\w]+)$/) {
484
	warn "tbadb_serv::rpc_loadimage: Malformed project argument!\n";
485 486 487 488
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
	exit 1;
    }
    $proj = $1;
489 490 491 492 493 494 495 496 497 498
    if ($bundle_name !~ /^([-\.\w]+)$/) {
	warn "tbadb_serv::rpc_loadimage: Malformed image bundle argument!\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed image bundle argument.");
	exit 1;
    }
    $bundle_name = $1;
    my $bundle_path = "$IMAGE_CACHE/$proj/$bundle_name";
    if (!-f $bundle_path) {
	warn "tbadb_serv::rpc_loadimage: no such bundle image: $bundle_path\n";
	send_error($data->{FID}, RPCERR_BADARGS, "No such image bundle.");
499 500 501
	exit 1;
    }
    if ($node_id !~ /^([-\w]+)$/) {
502
	warn "tbadb_serv::rpc_loadimage: Malformed node_id argument!\n";
503 504 505 506
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
	exit 1;
    }
    $node_id = $1;
507 508 509 510 511 512
    if (!exists($NMAP{$node_id})) {
	warn "tbadb::rpc_loadimage: unknown/bad node_id: $node_id\n";
	send_error($data->{FID}, RPCERR_BADARGS, "Unknown/bad node_id.");
	exit 1;
    }
    my $serial = $NMAP{$node_id};
513

514
    # Load image on to unit and report success/fail to caller.
515 516 517 518 519 520 521
    warn "tbadb_serv::rpc_loadimage: loading image $proj/$bundle_name on to $node_id\n";

    # Step 1: Unpack image bundle (if necessary). This may block.
    my $bundle_staging_dir = "$bundle_path.work";
    if (!unpack_bundle($bundle_path, $bundle_staging_dir)) {
	warn "tbadb_serv::rpc_loadimage: Could not unpack image bundle: $bundle_path\n";
	send_error($data->{FID}, RPCERR_INTERNAL, "Bundle unpack failed.");
522 523
	exit 1;
    }
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
    
    # Step 2: Make sure all required image files are present.
    my @todo_imgs = ();
    foreach my $partinfo (@ANDROID_PARTITIONS) {
	my ($imgpart, $defaultpath, $required) = @{$partinfo};
	my $imgpath = "$bundle_staging_dir/${imgpart}.img";
	if (!-r $imgpath) {
	    if (defined($defaultpath) && -r $defaultpath) {
		$imgpath = $defaultpath;
	    } 
	    elsif ($required) {
		warn "tbadb_serv::rpc_loadimage: ${imgpart}.img missing from bundle!\n";
		send_error($data->{FID}, RPCERR_BADARGS, "${imgpart}.img missing from bundle.");
		exit 1;
	    }
	    else {
		next;
	    }
	}
	push @todo_imgs, [$imgpart, $imgpath];
544 545
    }

546
    # Step 3: Reboot the device into fastboot mode.
547
    if (!enter_fastboot($node_id)) {
548
	warn "tbadb_serv::rpc_loadimage: failed to boot $node_id into fastboot!\n";
549 550 551
	send_error($data->{FID}, RPCERR_NODE_ERR, "Node failed to load into fastboot.");
	exit 1;
    }
552 553 554 555 556 557 558 559 560 561 562

    # Step 4: Reload the partitions based on the images we setup above.
    foreach my $imgdata (@todo_imgs) {
	my ($imgpart, $imgpath) = @{$imgdata};
	warn "tbadb_serv::rpc_loadimage: loading $imgpart partition on $node_id\n"
	    if $debug;
	if (!load_android_image($node_id, $imgpart, $imgpath)) {
	    warn "tbadb_serv::rpc_loadimage: failed to load $imgpart on $node_id!\n";
	    send_error($data->{FID}, RPCERR_NODE_ERR, "Failed to load $imgpart.");
	    exit 1;
	}
563 564
    }

565
    # Step 5: reboot into newly loaded image
566
    if (!reboot_android($node_id)) {
567
	warn "tbadb_serv::rpc_loadimage: newly loaded image failed to boot!\n";
568 569 570 571 572
	send_error($data->{FID}, RPCERR_NODE_ERR, "Failed to boot newly loaded image.");
	exit 1;
    }

    # Send success result back to caller.
573
    warn "tbadb::rpc_loadimage: finished loading $proj/$bundle_name on $node_id\n";
574
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
575 576 577 578 579 580 581 582
    return;
}


sub rpc_captureimage($) {
    my ($data) = @_;

    # XXX: fill me in!
583
    warn "tbadb_serv::rpc_captureimage: Called...\n";
584 585 586 587 588 589 590 591
    send_error($data->{FID}, RPCERR_NOTIMPL, "captureimage not implemented yet.");
    return;
}


sub rpc_reboot($) {
    my ($data) = @_;

592 593
    my $node_id = $data->{ARGS}->{NODE_ID};
    my $dowait  = $data->{ARGS}->{WAIT};
594
    my $fastboot = $data->{ARGS}->{FASTBOOT};
595 596 597

    # Do a bit of arg checking.
    $dowait = defined($dowait) && int($dowait) ? 1 : 0;
598
    $fastboot = defined($fastboot) && int($fastboot) ? 1 : 0;
599
    if (!$node_id) {
600
	warn "tbadb_serv::rpc_reboot: No node_id provided in RPC args!\n";
601 602 603
	send_error($data->{FID}, RPCERR_BADARGS, "node_id missing.");
	exit 1;
    }
604
    if (!exists($NMAP{$node_id})) {
605
	warn "tbadb_serv::rpc_reboot: unknown/bad node_id: $node_id\n";
606 607 608 609
	send_error($data->{FID}, RPCERR_BADARGS, "Unknown/bad node_id.");
	exit 1;
    }
    my $serial = $NMAP{$node_id};
610

611 612 613 614 615 616 617 618
    # Reboot the unit as directed.
    if ($fastboot) {
	warn "tbadb_serv::rpc_reboot: rebooting node into fastboot: $node_id\n";
	if (!enter_fastboot($node_id)) {
	    warn "tbadb_serv::rpc_reboot: failed to enter fastboot: $node_id!\n";
	    send_error($data->{FID}, RPCERR_NODE_ERR, "Reboot (fastboot) failed.");
	    exit 1;
	}
619
    }
620 621 622 623 624
    else {
	warn "tbadb_serv::rpc_reboot: rebooting node $node_id\n";
	if (!reboot_android($node_id)) {
	    warn "tbadb_serv::rpc_reboot: failed to reboot $node_id!\n";
	    send_error($data->{FID}, RPCERR_NODE_ERR, "Reboot failed.");
625 626
	    exit 1;
	}
627 628 629 630 631 632 633 634 635 636

	# Wait for device to boot up, if requested to do so.
	if ($dowait) {
	    warn "tbadb_serv::rpc_reboot: waiting for $node_id to come up.\n";
	    if (!wait_for_node($node_id)) {
		warn "tbadb_serv::rpc_reboot: failed waiting for $node_id to boot!\n";
		send_error($data->{FID}, RPCERR_NODE_ERR, "Boot failed.");
		exit 1;
	    }
	}
637 638 639
    }

    # Report success.
640
    warn "tbadb_serv::rpc_reboot: reboot of $node_id finished.\n";
641
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
642 643 644 645
    return;
}


646
sub rpc_forward($) {
647 648
    my ($data) = @_;

649 650
    my $node_id  = $data->{ARGS}->{NODE_ID};
    my $thost    = $data->{ARGS}->{TARGET_HOST};
651

652
    if (!$node_id || !$thost) {
653
	warn "tbadb_serv::rpc_forward: Missing RPC args!\n";
654 655 656
	send_error($data->{FID}, RPCERR_BADARGS, "One or more arguments missing.");
	exit 1;
    }
657
    if ($node_id !~ /^([-\w]+)$/) {
658
	warn "tbadb_serv::rpc_forward: Malformed node_id argument!\n";
659 660 661 662 663
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
	exit 1;
    }
    $node_id = $1;
    if ($thost !~ /^([-\.\w]+)$/) {
664
	warn "tbadb_serv::rpc_forward: Malformed target host argument!\n";
665 666 667 668
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed target host argument.");
	exit 1;
    }
    $thost = $1;
669
    if (!exists($NMAP{$node_id})) {
670
	warn "tbadb::rpc_forward: unknown/bad node_id: $node_id\n";
671 672 673 674 675 676 677
	send_error($data->{FID}, RPCERR_BADARGS, "Unknown/bad node_id.");
	exit 1;
    }
    my $serial = $NMAP{$node_id};

    # Setup forwarding from the ADB daemon on the unit to the
    # destionation port, and report back to caller!
678
    warn "tbadb_serv::rpc_forward: fowarding ADB on node $node_id to destination host $thost\n";
679 680
    my $port = setup_android_forward($node_id, $thost);
    if (!$port) {
681
	warn "tbadb_serv::rpc_forward: failed to setup adb port forwarding for $node_id to $thost\n";
682 683 684
	send_error($data->{FID}, RPCERR_NODE_ERR, "adb forwarding failed.");
	exit 1;
    }
685

686
    # Report success.
687
    warn "tbadb_serv::rpc_forward: forwarding setup for $node_id finished.\n";
688
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { PORT => $port }));
689 690 691 692
    return;
}


693 694 695 696 697 698
sub rpc_unforward($) {
    my ($data) = @_;

    my $node_id  = $data->{ARGS}->{NODE_ID};

    if (!$node_id) {
699
	warn "tbadb_serv::rpc_unforward: Missing node_id argument!\n";
700 701 702 703
	send_error($data->{FID}, RPCERR_BADARGS, "node_id argument missing.");
	exit 1;
    }
    if ($node_id !~ /^([-\w]+)$/) {
704
	warn "tbadb_serv::rpc_unforward: Malformed node_id argument!\n";
705 706 707 708 709 710 711
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
	exit 1;
    }
    $node_id = $1;

    # Setup forwarding from the ADB daemon on the unit to the
    # destionation port, and report back to caller!
712
    warn "tbadb_serv::rpc_unforward: removing ADB forwarding setup for node $node_id\n";
713
    if (!remove_android_forward($node_id)) {
714
	warn "tbadb_serv::rpc_unforward: failed to clear adb port forwarding for $node_id!\n";
715 716 717 718 719
	send_error($data->{FID}, RPCERR_NODE_ERR, "adb forwarding removal failed.");
	exit 1;
    }

    # Report success.
720
    warn "tbadb_serv::rpc_unforward: forwarding cleared for $node_id\n";
721
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
722 723 724 725
    return;
}


Kirk Webb's avatar
Kirk Webb committed
726 727 728 729 730 731
sub rpc_nodewait($) {
    my ($data) = @_;

    my $node_id  = $data->{ARGS}->{NODE_ID};

    if (!$node_id) {
732
	warn "tbadb_serv::rpc_nodewait: Missing node_id argument!\n";
Kirk Webb's avatar
Kirk Webb committed
733 734 735 736
	send_error($data->{FID}, RPCERR_BADARGS, "node_id argument missing.");
	exit 1;
    }
    if ($node_id !~ /^([-\w]+)$/) {
737
	warn "tbadb_serv::rpc_nodewait: Malformed node_id argument!\n";
Kirk Webb's avatar
Kirk Webb committed
738 739 740 741 742 743
	send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
	exit 1;
    }
    $node_id = $1;

    # Wait for a while for the node to appear in adb.
744
    warn "tbadb_serv::rpc_nodewait: waiting for $node_id to become available.\n";
Kirk Webb's avatar
Kirk Webb committed
745
    if (!wait_for_android($node_id)) {
746
	warn "tbadb_serv::rpc_nodewait: failed while waiting for $node_id!\n";
Kirk Webb's avatar
Kirk Webb committed
747 748 749 750 751
	send_error($data->{FID}, RPCERR_NODE_ERR, "node wait failed.");
	exit 1;
    }

    # Report success.
752
    warn "tbadb_serv::rpc_nodewait: $node_id is now ready.\n";
753
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
Kirk Webb's avatar
Kirk Webb committed
754 755 756 757
    return;
}


758 759 760
sub rpc_ping($) {
    my ($data) = @_;

761
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { PONG => 1 }));
762 763 764
    return;
}

765

766 767 768 769
sub rpc_exit($) {
    my ($data) = @_;

    # Send acknowledgement and exit.  No arguments expected.
770
    SendRPCData($RPCOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
771 772
    exit 0;
}
773 774 775 776 777 778 779 780 781 782 783

##############################################################################
#
# Helper functions follow.
#

# Simple shorthand wrapper that sends an error back to the caller.
sub send_error($$$) {
    my ($fid, $code, $message) = @_;

    # Send failure result back.
784
    return SendRPCData($RPCOUT, EncodeError($fid, $code, $message));
785 786 787 788 789 790 791
}

# Helper that does LRU on our image cache
sub do_lru_cleanup() {
    my $tot = 0;
    my @files = ();
    my $err = 0;
792
    my $lru_lock;
793 794

    # Make sure the cache dir actually exists.
795
    if (!-d $IMAGE_CACHE) {
796
	warn "tbadb_serv::do_lru_cleanup: Image dir is not valid: $IMAGE_CACHE\n";
797 798 799
	return 0;
    }

800 801
    # Grab the global LRU lock.
    # XXX: We should also skip this process if we did it recently.
802 803
    my $lock_res = TBScriptLock("lrulock", TBSCRIPTLOCK_GLOBALWAIT,
				$LRULOCK_TMO, \$lru_lock);
804
    LOCKSW: for ($lock_res) {
805
	$_ == TBSCRIPTLOCK_IGNORE && do {
806 807 808
	    # Another process just did the LRU, so let's not do it again.
	    return 1;
	};
809
	$_ == TBSCRIPTLOCK_OKAY && do {
810 811 812 813
	    # Got the lock, just bail out of here and proceed.
	    last LOCKSW;
	};
	# Default case (error condition)
814
	warn "tbadb_serv::do_lru_cleanup: Failed to get the LRU lock!\n";
815 816 817
	return 0;
    }

818 819
    # Read entries in cache dir and tally size
    my $dh;
820
    if (!opendir($dh, $IMAGE_CACHE)) {
821
	warn "tbadb_serv::do_lru_cleanup: Failed to clean up img dir!\n";
822
	goto BADLRU;
823 824
    }
    while (my $ent = readdir($dh)) {
825 826
	next if ($ent !~ /^(\w[-\w]+)$/);
	$ent = $1;
827
	my $dname = "$IMAGE_CACHE/$ent";
828
	next if (!-d $dname || $ent eq "PNSYSTEM");
829 830
	my $subdh;
	if (!opendir($subdh, $dname)) {
831
	    warn "tbadb_serv::do_lru_cleanup: Could not descend into $dname\n";
832 833 834 835
	    $err = 1;
	    last;
	}
	while (my $subent = readdir($subdh)) {
836 837
	    next if ($subent !~ /^(\w[-\.\w]+)$/);
	    $subent = $1;
838
	    my $imfile  = "$dname/$subent";
839
	    next if (!-f $imfile);
840 841
	    my $lrufile = "$dname/.$subent.lru";
	    if (!-e $lrufile) {
842
		warn "tbadb_serv::do_lru_cleanup: creating missing LRU file for: $imfile";
843
		if (system($TOUCH, "$lrufile") != 0) {
844
		    warn "tbadb_serv::do_lru_cleanup: could not create LRU file: $lrufile\n";
845 846 847 848 849 850 851
		    $err = 1;
		    last;
		}
	    }
	    my $lutime  = (stat($lrufile))[9];
	    my $imsize  = (stat($imfile))[7];
	    if (!defined($lutime) || $lutime < 0) {
852
		warn "tbadb_serv::do_lru_cleanup: Something weird happened while trying to stat $lrufile!\n";
853 854 855 856
		$err = 1;
		last;
	    }
	    if (!defined($imsize) || $imsize < 0) {
857
		warn "tbadb_serv::do_lru_cleanup: Something weird happened while trying to stat $imfile!\n";
858 859 860
		$err = 1;
		last;
	    }
861
	    warn "tbadb_serv::do_lru_cleanup: image file found: $ent/$subent:$lutime:$imsize\n"
862 863 864 865 866 867 868 869
		if $debug;
	    push @files, [$imfile, $lutime, $imsize];
	    $tot += $imsize;
	}
	close $subdh;
	last if $err;
    }
    close $dh;
870
    goto BADLRU if $err;
871 872 873

    # Prune?
    if ($tot > $WM_HIGH) {
874
	warn "tbadb_serv::do_lru_cleanup: Invoking LRU cleanup ($tot)\n";
875 876 877 878 879 880
	my @sortedfiles = sort { $a->[1] <=> $b->[1] } @files;
	while ($tot > $WM_LOW && scalar(@sortedfiles)) {
	    my ($imfile, undef, $imsize) = @{shift(@sortedfiles)};
	    $imfile =~ /^(.+)\/(.+)$/;
	    my $lrufile = "$1/.$2.lru";
	    if (!unlink($imfile, $lrufile)) {
881
		warn "tbadb_serv::do_lru_cleanup: Could not remove $imfile: $!\n";
882
		goto BADLRU;
883
	    }
884 885 886 887 888
	    if (system("$RM -rf ${imfile}.work") != 0) {
		warn "tbadb_serv::do_lru_cleanup: Could not remove unpacked bundle dir for $imfile\n";
		goto BADLRU;
	    }
	    warn "tbadb_serv::do_lru_cleanup: removed image $imfile\n";
889 890 891 892
	    $tot -= $imsize;
	}
	if ($tot > $WM_LOW) {
	    warn "Unable to prune enough images to get under low watermark ($WM_LOW)!\n";
893
	    goto BADLRU;
894 895 896
	}
    }

897
    TBScriptUnlock($lru_lock);
898
    return 1;
899 900 901 902 903 904 905 906

  BADLRU:
    TBScriptUnlock($lru_lock);
    return 0;
}

# Unpack a zip bundle containting one or more android image files. Do NOT
# call this function without a lock!  It should not run in parallel across
907
# tbadb_serv processes for the same image bundle.
908 909 910 911 912
sub unpack_bundle($$) {
    my ($fname, $destdir) = @_;
    my $unpack_lock;

    # Grab the unpack lock.
913 914
    if (TBScriptLock("unpacklock", undef, 
		     $UNPACKLOCK_TMO, \$unpack_lock) != TBSCRIPTLOCK_OKAY) {
915
	warn "tbadb_serv::unpack_bundle: Failed to get the unpack lock!\n";
916 917 918 919 920 921
	return 0;
    }

    # Make sure image is a zip bundle
    my $ftype = `$FILE -b -i $fname 2>&1`;
    if ($ftype !~ /application\/zip/) {
922
	warn "tbadb_serv::unpack_bundle: file is not a zip bundle: $fname\n";
923 924 925 926 927 928
	goto BADUNPACK;
    }
    # Delete the old destdir if it exists and is older than the image.
    if (-d $destdir &&
	(stat($destdir))[9] < (stat($fname))[9]) {
	if (system("$RM -rf $destdir >/dev/null 2>&1") != 0) {
929
	    warn "tbadb_serv::unpack_bundle: could not remove existing workdir for image bundle: $fname\n";
930 931 932 933 934 935
	    goto BADUNPACK;
	}
    }
    # If the image directory doesn't exist, we must unpack!
    if (!-d $destdir) {
	if (!mkdir($destdir, 0750)) {
936
	    warn "tbadb_serv::unpack_bundle: could not create workdir for image bundle: $fname: $!\n";
937 938 939
	    goto BADUNPACK;
	}
	if (system("$UNZIP -d $destdir $fname >/dev/null 2>&1") != 0) {
940
	    warn "tbadb_serv::unpack_bundle: failed to unpack image bundle: $fname\n";
941 942 943 944 945 946 947 948 949 950
	    goto BADUNPACK;
	}
    }

    TBScriptUnlock($unpack_lock);
    return 1;

  BADUNPACK:
    TBScriptUnlock($unpack_lock);
    return 0;
951 952 953 954 955 956 957 958
}

# Put a given android device into fastboot mode.
sub enter_fastboot($) {
    my ($node_id) = @_;

    my $serial = $NMAP{$node_id};
    if (!$serial) {
959
	warn "tbadb_serv::enter_fastboot: No serial number for $node_id!\n";
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
	return 0;
    }

    # First see if it is already sitting in fastboot.
    my $state = `$FASTBOOT devices 2>&1`;
    return 1 if $state =~ /$serial\s+fastboot/;

    # Next try to reboot via adb if adbd is running on the device.
    $state = `$ADB -s $serial get-state 2>&1`;
    chomp $state;
    if ($state ne "unknown") {
	if (system("$ADB -s $serial reboot-bootloader > /dev/null 2>&1") != 0) {
	    warn "Failed to reboot $node_id via adb!\n";
	    return 0;
	}
	# Now wait for fastboot to see it
	my $stime = time();
	while (1) {
	    sleep 5;
	    $state = `$FASTBOOT devices 2>&1`;
	    return 1 if $state =~ /$serial\s+fastboot/;
981
	    if (time() - $stime > $FASTBOOT_TMO) {
982
		warn "tbadb_serv::enter_fastboot: timed out waiting for $node_id\n";
983 984 985 986 987
		return 0;
	    }
	}
    }

988
    warn "tbadb_serv::enter_fastboot: Could not find device!\n";
989 990 991 992 993 994 995 996 997
    return 0;
}

# Flash an image to a device on the given partition.
sub load_android_image($$$) {
    my ($node_id, $partition, $impath) = @_;

    my $serial = $NMAP{$node_id};
    if (!$serial) {
998
	warn "tbadb_serv::load_android_image: No serial number for $node_id!\n";
999 1000 1001 1002 1003
	return 0;
    }

    my $state = `$FASTBOOT devices 2>&1`;
    if ($state !~ /$serial\s+fastboot/) {
1004
	warn "tbadb_serv::load_android_image: device $node_id is not in fastboot. Can't flash image!";
1005 1006 1007 1008 1009
	return 0;
    }

    my $imgout = `$FASTBOOT -u -s $serial flash $partition $impath 2>&1`;
    if ($?) {
1010
	warn "tbadb_serv::load_android_image: fastboot failed to flash $partition with $impath on $node_id!\n";
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
	warn "Output:\n$imgout\n";
	return 0;
    }

    return 1;
}

# Reboot a device, but don't wait for it to come up.
sub reboot_android($) {
    my ($node_id) = @_;

    my $serial = $NMAP{$node_id};
    if (!$serial) {
1024
	warn "tbadb_serv::reboot_android: No serial number for $node_id!\n";
1025 1026 1027 1028 1029 1030 1031
	return 0;
    }

    # Figure out if node is setting in fastboot or regular boot, and reboot.
    my $state = `$FASTBOOT devices 2>&1`;
    if ($state =~ /$serial\s+fastboot/) {
	if (system("$FASTBOOT -s $serial reboot >/dev/null 2>&1") != 0) {
1032
	    warn "tbadb_serv::reboot_android: fastboot failed to reboot $node_id!\n";
1033 1034 1035 1036
	    return 0;
	}
    } else {
	$state = `$ADB -s $serial get-state 2>&1`;
Kirk Webb's avatar
Kirk Webb committed
1037
	chomp $state;
1038 1039
	if ($state ne "unknown") {
	    if (system("$ADB -s $serial reboot") != 0) {
1040
		warn "tbadb_serv::reboot_android: adb failed to reboot $node_id!\n";
1041 1042 1043
		return 0;
	    }
	} else {
1044
	    warn "tbadb_serv::reboot_android: could not find device $node_id!\n";
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	    return 0;
	}
    }

    # We don't wait for the node to come up...
    return 1;
}

# Helper that waits (bounded by a timeout) for a device to become available
# via adb.
sub wait_for_android($) {
    my ($node_id) = @_;

    my $serial = $NMAP{$node_id};
    if (!$serial) {
1060
	warn "tbadb_serv::wait_for_android: No serial number for $node_id!\n";
1061 1062 1063 1064 1065 1066
	return 0;
    }

    my $stime = time();
    while (1) {
	my $state = `$ADB -s $serial get-state 2>&1`;
Kirk Webb's avatar
Kirk Webb committed
1067
	chomp $state;
1068 1069 1070 1071
	if ($state eq "device") {
	    last;
	}
	if (time() - $stime > $ANDROID_BOOT_TMO) {
1072
	    warn "tbadb_serv::wait_for_android: timed out waiting for $node_id to boot!\n";
1073 1074 1075 1076 1077 1078 1079 1080
	    return 0;
	}
	sleep 5;
    }

    return 1;
}

1081
# Helper that sets up adb listener port for a device
1082 1083
sub setup_android_forward($$;$$) {
    my ($node_id, $thost, $port, $locknload) = @_;
1084
    my $fwd_lock;
1085

1086 1087
    $locknload = defined($locknload) ? $locknload : 1;

1088 1089
    my $serial = $NMAP{$node_id};
    if (!$serial) {
1090
	warn "tbadb_serv::setup_android_forward: No serial number for $node_id!\n";
1091 1092 1093 1094 1095 1096 1097 1098 1099
	return 0;
    }

    # Resolve the IP address for the target host if necessary
    if ($thost !~ /^\d+\.\d+\.\d+\.\d+$/) {
	my $tres = `$HOST -t A $thost 2>&1`;
	if ($tres =~ /has address ([\.\d]+)/) {
	    $thost = $1;
	} else {
1100
	    warn "tbadb_serv::setup_android_forward: could not lookup ip for host $thost\n";
1101 1102 1103 1104
	    return 0;
	}
    }

1105 1106 1107 1108
    # Lock-n-load bro!
    if ($locknload) {
	if (TBScriptLock("fwdports", undef, 
			 $FWDLOCK_TMO, \$fwd_lock) != TBSCRIPTLOCK_OKAY) {
1109
	    warn "tbadb_serv::setup_android_forward: Failed to get the forwarding lock!\n";
1110 1111 1112 1113 1114 1115 1116 1117
	    return 0;
	}
	# Tie to the forwarding ports DB file.
	if (!tie(%FWDPORTS, 'DB_File', $FWDDBFILE, 
		 O_RDWR | O_CREAT, 0755, $DB_HASH)) {
	    warn "tbadb::setup_android_forward: unable to load forwarding database!\n";
	    goto BADFWD;
	}
1118 1119
    }

1120
    # Scour any existing forwarding setup for the device
1121
    if (!remove_android_forward($node_id, 0)) {
1122
	warn "tbadb_serv::setup_android_forward: could not remove old forwarding setup for $node_id!\n";
1123
	goto BADFWD;
1124 1125
    }

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
    # Pick an available port (if not specified, which is an internal option).
    if (!$port) {
	my $pnum;
	for ($pnum = $MINPORT; $pnum < $MAXPORT; $pnum++) {