Commit b17acd1a authored by Kirk Webb's avatar Kirk Webb

tbadb: rework client side to use an IO::Socket transport

Instead of using SSH as the transport.  Soooooo much simpler on
the client side now!
parent cc9716aa
......@@ -28,9 +28,11 @@ use English;
use POSIX ":sys_wait_h";
use Getopt::Std;
use Data::Dumper;
use IO::Socket::INET;
use lib "@prefix@/lib";
use libjsonrpc;
use tbadb_rpc;
use EmulabConstants;
use User;
use Node;
......@@ -42,10 +44,8 @@ sub cmd_setup($@);
sub cmd_loadimage($@);
sub cmd_forward($@);
sub cmd_reboot($;@);
sub GetSSHPipe();
sub chldhandler(;$);
sub genhandler($);
sub cleanexit($);
sub GetRPCPipeHandles($);
sub ConnectRPCHost($);
# Global variables
my $TB = "@prefix@";
......@@ -53,7 +53,7 @@ my $MINHLEN = 2;
my $MAXHLEN = 32;
my $MINCMDLEN = 2;
my $MAXCMDLEN = 32;
my %SSHPIPES = ();
my %RPCPIPES = ();
my $TBADB_PROXYCMD = "/usr/testbed/sbin/tbadb_proxy";
my $TBADB_HELLO_TMO = 10;
my $TBADB_CHECKIMAGE_TMO = 30;
......@@ -62,9 +62,6 @@ my $TBADB_FORWARD_TMO = 15;
my $TBADB_REBOOT_TMO = 60;
my $CHILD_WAIT_TMO = 10;
my $SCP = "/usr/bin/scp";
my $SSHTB = "$TB/bin/sshtb";
my @SSH_ARGS = ('-o','BatchMode=yes','-o','StrictHostKeyChecking=no',
'-o','ConnectTimeout=10');
my %DISPATCH = (
'setup' => \&cmd_setup,
......@@ -384,184 +381,50 @@ sub GetRPCPipeHandles($) {
my ($node) = @_;
my ($rpcin, $rpcout);
# Lookup the node's control server and create SSH pipe object if we
# don't have one yet.
# Look up the node's control (console) server and connect to it if
# we haven't done so yet. Otherwise grab and return the open pipe.
my $conserver;
$node->TipServer(\$conserver);
die "tbadb::GetRPCPipeHandles: Could not lookup control server for $node!\n"
if (!$conserver);
if (!exists($SSHPIPES{$conserver})) {
$SSHPIPES{$conserver} =
TBADB::SSHPipe->New($conserver, $TBADB_PROXYCMD);
if (!exists($RPCPIPES{$conserver})) {
$RPCPIPES{$conserver} = ConnectRPCHost($conserver);
}
my $sshpipe = $SSHPIPES{$conserver};
my $rpcpipe = $RPCPIPES{$conserver};
die "tbadb::GetRPCPipeHandles: RPC pipe for $conserver closed unexpectedly!"
if (!$rpcpipe->connected());
# Open the SSH Pipe if necessary.
if (!$sshpipe->isopen()) {
my $SAVEUID = $UID;
$EUID = $UID = 0; # Flip to root to run!
($rpcin, $rpcout) = $sshpipe->GetPipe();
$EUID = $UID = $SAVEUID; # Flip back.
my $pdu;
my $res = RecvRPCData($rpcin, \$pdu, $TBADB_HELLO_TMO);
if ($res == -1) {
die "tbadb: Timeout while opening SSH Pipe!\n";
}
elsif ($res == 0) {
die "tbadb: Error encountered while opening SSH Pipe!\n";
}
# Look for the hello.
my $hello = DecodeRPCData($pdu);
die "tbadb: Unexpected data received when opening SSH Pipe!\n"
if (!$hello);
die "tbadb: Did not receive valid 'hello' from remote end!\n"
if (!exists($hello->{RESULT}->{HELLO}));
} else {
($rpcin, $rpcout) = $sshpipe->GetPipe();
}
return ($rpcin, $rpcout);
}
# This subroutine is called via any exit path to ensure the RPC
# Pipe (and associated child process) are terminated/closed.
sub cleanexit($) {
my $exval = shift;
my @tokill = ();
while (my ($cs_node_id, $sshpipe) = each %SSHPIPES) {
if ($sshpipe->child()) {
push @tokill, $sshpipe->child();
} else {
delete $SSHPIPES{$cs_node_id};
}
}
if (@tokill) {
warn "tbadb: Killing SSH pipe processes: @tokill\n";
my $SAVEUID = $UID;
$EUID = $UID = 0; # Flip to root to kill!
kill("TERM", @tokill);
$EUID = $UID = $SAVEUID; # Flip back.
my $stime = time();
while (keys %SSHPIPES) {
chldhandler();
if (time() - $stime > $CHILD_WAIT_TMO) {
warn "tbadb: Timed out waiting for SSH RPC pipes to exit!\n".
"Remaining: ". join(", ", keys %SSHPIPES) ."\n";
last;
}
sleep 1;
}
}
exit $exval;
}
sub chldhandler(;$) {
local ($!, $?); # Don't let issues in the handler affect these globals.
while ((my $pid = waitpid(-1, WNOHANG)) > 0) {
my $found = 0;
while (my ($cs_node_id, $sshpipe) = each %SSHPIPES) {
if ($sshpipe->child() == $pid) {
print "tbadb: SSH Pipe exited for $cs_node_id\n"
if $debug;
delete $SSHPIPES{$cs_node_id};
$found = 1;
last;
}
}
warn "tbadb: Non-pipe child process exited: $pid\n"
if (!$found && $debug);
}
}
sub genhandler($) {
cleanexit(1);
return ($rpcpipe, $rpcpipe);
}
sub mysystem(@) {
local $SIG{CHLD} = 'DEFAULT';
my $retcode = system(@_);
chldhandler();
return $retcode;
}
END {
warn "This is the end...\n" if $debug;
cleanexit($?) if (keys %SSHPIPES);
}
#
# SSHPipe session wrapper class
#
package TBADB::SSHPipe;
use IPC::Open2;
# We don't immediately open the pipe. Wait until the pipe file handles
# are requested.
sub New($$$) {
my ($class, $rhost, $rcmd) = @_;
my $self = {};
$self->{RHOST} = $rhost;
$self->{RCMD} = $rcmd;
$self->{CHIN} = undef;
$self->{CHOUT} = undef;
$self->{CHPID} = 0;
bless($self, $class);
return $self;
}
sub DESTROY($) {
my $self = shift;
$self->{CHIN} = undef;
$self->{CHOUT} = undef;
$self->{CHPID} = 0;
}
sub child($) { return $_[0]->{CHPID}; }
# Test to see if the SSH pipe is open, including testing to see if the
# SSH pipe process is still running (if it has been started).
sub isopen($) {
my ($self) = @_;
my $isopen = 0;
if ($self->{CHPID}) {
if (kill("ZERO", $self->{CHPID})) {
$isopen = 1;
} else {
# Zap stale SSH child info.
$self->{CHPID} = undef;
$self->{CHIN} = undef;
$self->{CHOUT} = undef;
}
# Helper that connects to a remote TBADB RPC proxy service.
sub ConnectRPCHost($) {
my ($host) = @_;
# Connect and read in expected "hello" message.
my $socket =
IO::Socket::INET->new(
PeerAddr => $host,
PeerPort => TBADB_PORT,
Proto => 'tcp'
);
die "tbadb::ConnectRPCHost: Could not connect to tbadb proxy on host $host: $!\n"
if (!$socket);
$socket->autoflush(1);
my $pdu;
my $res = RecvRPCData($socket, \$pdu, $TBADB_HELLO_TMO);
if ($res == -1) {
die "tbadb::ConnectRPCHost: Timeout while opening RPC Pipe!\n";
}
return $isopen;
}
# Return the SSH Pipe file handles, first opening the connection to
# thre remote host if necessary.
sub GetPipe($) {
my ($self) = @_;
my ($chin, $chout);
if (!$self->{CHPID}) {
my $chpid = open2($chin, $chout, $SSHTB, @SSH_ARGS,
"-host", $self->{RHOST}, $self->{RCMD});
if ($chpid < 1) {
warn "TBADB: Failed to open SSH pipe process!\n";
}
$self->{CHPID} = $chpid;
$self->{CHIN} = $chin;
$self->{CHOUT} = $chout;
elsif ($res == 0) {
die "tbadb::ConnectRPCHost: Error encountered while opening RPC Pipe!\n";
}
return ($self->{CHIN}, $self->{CHOUT});
# Look for the hello.
my $hello = DecodeRPCData($pdu);
die "tbadb::ConnectRPCHost: Unexpected data received when opening RPC Pipe!\n"
if (!$hello);
die "tbadb::ConnectRPCHost: Did not receive valid 'hello' from remote end!\n"
if (!exists($hello->{RESULT}->{HELLO}));
return $socket;
}
......@@ -66,7 +66,6 @@ my %DISPATCH = (
);
# Constants
my $LISTENPORT = 8888;
my $DEF_LOGLEVEL = 2;
my $PIDFILE = "/var/run/tbadb_proxy.pid";
my $LOGFILE = "$LOGDIR/tbadb_proxy.log";
......@@ -95,7 +94,7 @@ my $debug = 0;
# Invoke the parent Net::Server class' run routine.
TBADBProxy::Server->run({
port => $LISTENPORT,
port => TBADB_PORT,
log_file => $LOGFILE,
log_level => $DEF_LOGLEVEL,
ipv => 4,
......@@ -128,7 +127,7 @@ sub options($$) {
#
# Do some overriding of config variables when debugging is requested
#
sub configure_hook($) {
sub post_configure {
my $self = shift;
my $prop = $self->{'server'};
......@@ -140,11 +139,16 @@ sub configure_hook($) {
$prop->{'background'} = 0;
$prop->{'setsid'} = 0;
$prop->{'log_level'} = 4;
$prop->{'log_file'} = undef;
}
$self->SUPER::post_configure(@_);
}
#
# Do a bit of post-processing/checking after internal Net::Server arg parsing
# Do a bit of post-processing/checking after internal Net::Server
# configuration stage (which includes going into the background,
# setting up logging, etc.)
#
sub post_configure_hook($) {
my $self = shift;
......@@ -189,7 +193,7 @@ sub process_request($) {
$| = 1;
# Send "hello" To let remote end know we are ready.
die "tbadb_proxy: Could not send hello to caller!\n"
die "tbadb_proxy: Could not send hello to caller. Terminating connection.\n"
if !SendRPCData(*STDOUT, EncodeResult(-1, { HELLO => 1 }));
while (1) {
......@@ -197,18 +201,18 @@ sub process_request($) {
my $pdu;
my $rcode = RecvRPCData(*STDIN, \$pdu);
if ($rcode == -1) {
warn "tbadb_proxy: Timed out waiting for RPC data. Exiting!\n";
warn "tbadb_proxy: timed out waiting for RPC data. Terminating connection\n";
exit 1;
}
elsif ($rcode == 0) {
warn "tbadb_proxy: Error receiving RPC data. Exiting!\n";
warn "tbadb_proxy: EOF from RPC pipe. Terminating connection\n";
exit 1;
}
# Decode PDU
my $data = DecodeRPCData($pdu);
if (!$data) {
warn "tbadb_proxy: Unable to decode RPC data. Exiting!\n";
warn "tbadb_proxy: unable to decode RPC data. Terminating connection.\n";
exit 1;
}
......@@ -216,13 +220,13 @@ sub process_request($) {
my $func = $data->{FUNCTION};
if (defined($func)) {
if (!exists($DISPATCH{$func})) {
warn "Unkown RPC called: $func. Exiting!\n";
warn "tbadb_proxy: Unkown RPC: $func. Sending error and terminating connection.\n";
send_error($data->{FID}, RPCERR_BADFUNC, "Unknown function: $func");
exit 1;
}
$DISPATCH{$func}->($data);
} else {
warn "tbadb_proxy: Received RPC data that was not a function call. Exiting!\n";
warn "tbadb_proxy: Received RPC data that was not a function call. Terminating connection.\n";
exit 1;
}
}
......@@ -240,11 +244,24 @@ sub rpc_checkimage($) {
my $srctime = $data->{ARGS}->{IMG_TIME};
my $srcsize = $data->{ARGS}->{IMG_SIZE};
# Arg checking and untainting.
if (!$proj || !$srcname || !$srctime || !defined($srcsize)) {
warn "tbadb_proxy::rpc_checkimage: Argument(s) missing from RPC!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Argument(s) missing.");
exit 1;
}
if ($proj !~ /^([-\w]+)$/) {
warn "tbadb_proxy::rpc_checkimage: Malformed project argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
exit 1;
}
$proj = $1;
if ($srcname !~ /^([-\.\w]+)$/) {
warn "tbadb_proxy::rpc_checkimage: Malformed image name argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed image name argument.");
exit 1;
}
$srcname = $1;
warn "tbadb_proxy::rpc_checkimage: Image check requested for $proj/$srcname\n";
......@@ -347,7 +364,7 @@ sub rpc_checkimage($) {
sub rpc_loadimage($) {
my ($data) = @_;
# Check arguments
# Check and untaint arguments
my $node_id = $data->{ARGS}->{NODE_ID};
my $imgname = $data->{ARGS}->{IMG_NAME};
my $proj = $data->{ARGS}->{IMG_PROJ};
......@@ -356,6 +373,24 @@ sub rpc_loadimage($) {
send_error($data->{FID}, RPCERR_BADARGS, "Missing arguments.");
exit 1;
}
if ($proj !~ /^([-\w]+)$/) {
warn "tbadb_proxy::rpc_loadimage: Malformed project argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed project argument.");
exit 1;
}
$proj = $1;
if ($imgname !~ /^([-\.\w]+)$/) {
warn "tbadb_proxy::rpc_loadimage: Malformed image name argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed image name argument.");
exit 1;
}
$imgname = $1;
if ($node_id !~ /^([-\w]+)$/) {
warn "tbadb_proxy::rpc_loadimage: Malformed node_id argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
exit 1;
}
$node_id = $1;
if (!exists($NMAP{$node_id})) {
warn "tbadb::rpc_loadimage: unknown/bad node_id: $node_id\n";
send_error($data->{FID}, RPCERR_BADARGS, "Unknown/bad node_id.");
......@@ -434,6 +469,7 @@ sub rpc_loadimage($) {
}
# Send success result back to caller.
warn "tbadb::rpc_loadimage: finished loading $proj/$imgname on $node_id\n";
SendRPCData(*STDOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
return;
}
......@@ -488,6 +524,7 @@ sub rpc_reboot($) {
}
# Report success.
warn "tbadb_proxy::rpc_reboot: reboot of $node_id finished.\n";
SendRPCData(*STDOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
return;
}
......@@ -505,13 +542,26 @@ sub rpc_forward($) {
send_error($data->{FID}, RPCERR_BADARGS, "One or more arguments missing.");
exit 1;
}
if ($tport < 1 || $tport > 65535) {
warn "tbadb_proxy::rpc_forwardadb: requested port is out of range: $tport!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Requested port is out of range.");
if ($node_id !~ /^([-\w]+)$/) {
warn "tbadb_proxy::rpc_forward: Malformed node_id argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed node_id argument.");
exit 1;
}
$node_id = $1;
if ($thost !~ /^([-\.\w]+)$/) {
warn "tbadb_proxy::rpc_forward: Malformed target host argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed target host argument.");
exit 1;
}
$thost = $1;
if ($tport !~ /^(\d+)$/ || $tport < 1 || $tport > 65534) {
warn "tbadb_proxy::rpc_forward: Malformed target port argument!\n";
send_error($data->{FID}, RPCERR_BADARGS, "Malformed target port argument.");
exit 1;
}
$tport = $1;
if (!exists($NMAP{$node_id})) {
warn "tbadb::rpc_loadimage: unknown/bad node_id: $node_id\n";
warn "tbadb::rpc_forward: unknown/bad node_id: $node_id\n";
send_error($data->{FID}, RPCERR_BADARGS, "Unknown/bad node_id.");
exit 1;
}
......@@ -527,6 +577,7 @@ sub rpc_forward($) {
}
# Report success.
warn "tbadb_proxy::rpc_forward: forwarding setup for $node_id finished.\n";
SendRPCData(*STDOUT, EncodeResult($data->{FID}, { SUCCESS => 1 }));
return;
}
......
......@@ -28,7 +28,9 @@ use Exporter;
@ISA = "Exporter";
@EXPORT =
qw ( RPCERR_BADARGS RPCERR_BADFUNC RPCERR_NOTIMPL
qw ( TBADB_PORT
RPCERR_BADARGS RPCERR_BADFUNC RPCERR_NOTIMPL
RPCERR_NODE_ERR RPCERR_INTERNAL
IMG_RECOVERY IMG_BOOT IMG_USERDATA IMG_CACHE
......@@ -37,6 +39,8 @@ use Exporter;
use strict;
use English;
sub TBADB_PORT { return 8888; }
sub RPCERR_BADARGS { return 2; }
sub RPCERR_BADFUNC { return 3; }
sub RPCERR_NOTIMPL { return 4; }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment