Commit 6f546d14 authored by David Johnson's avatar David Johnson

Add Docker serial console support.

We do this similarly to Xen.  There's a new script (container2pty.py)
that attaches to the Docker container, via the docker daemon, and
exports its stdio as a pty.  Then we run capture on a symlink to that
pty.  New options to capture tell it to keep retrying to open the pty
maxretries times (we invoke with infinitely many retries); and to not
prepend /dev to the device string.
parent d7b5d3b5
...@@ -174,6 +174,8 @@ int remotemode; ...@@ -174,6 +174,8 @@ int remotemode;
int programmode; int programmode;
char *xendomain; char *xendomain;
int retryinterval = 5000; int retryinterval = 5000;
int maxretries = 0;
int nomodpath = 0;
int maxfailures = 10; int maxfailures = 10;
int failures; int failures;
int upportnum = -1, upfd = -1, upfilefd = -1; int upportnum = -1, upfd = -1, upfilefd = -1;
...@@ -381,7 +383,7 @@ main(int argc, char **argv) ...@@ -381,7 +383,7 @@ main(int argc, char **argv)
else else
Progname = *argv; Progname = *argv;
while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:")) != EOF) while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:y:A")) != EOF)
switch (op) { switch (op) {
#ifdef USESOCKETS #ifdef USESOCKETS
#ifdef WITHSSL #ifdef WITHSSL
...@@ -428,6 +430,12 @@ main(int argc, char **argv) ...@@ -428,6 +430,12 @@ main(int argc, char **argv)
usage(); usage();
} }
break; break;
case 'y':
maxretries = atoi(optarg);
break;
case 'A':
nomodpath = 1;
break;
#endif /* USESOCKETS */ #endif /* USESOCKETS */
case 'H': case 'H':
++hwflow; ++hwflow;
...@@ -510,6 +518,8 @@ main(int argc, char **argv) ...@@ -510,6 +518,8 @@ main(int argc, char **argv)
} }
else if (xendomain) else if (xendomain)
strcpy(strbuf, xendomain); strcpy(strbuf, xendomain);
else if (nomodpath)
strcpy(strbuf, argv[1]);
else else
(void) snprintf(strbuf, sizeof(strbuf), (void) snprintf(strbuf, sizeof(strbuf),
DEVNAME, DEVPATH, argv[1]); DEVNAME, DEVPATH, argv[1]);
...@@ -997,6 +1007,8 @@ capture(void) ...@@ -997,6 +1007,8 @@ capture(void)
sigset_t omask; sigset_t omask;
char buf[BUFSIZE]; char buf[BUFSIZE];
struct timeval timeout; struct timeval timeout;
int nretries;
/* /*
* XXX for now we make both directions non-blocking. This is a * XXX for now we make both directions non-blocking. This is a
...@@ -1128,7 +1140,8 @@ capture(void) ...@@ -1128,7 +1140,8 @@ capture(void)
Machine, cc); Machine, cc);
if (cc <= 0) { if (cc <= 0) {
#ifdef USESOCKETS #ifdef USESOCKETS
if (remotemode || programmode || xendomain) { if (remotemode || programmode || xendomain
|| maxretries) {
FD_CLR(devfd, &sfds); FD_CLR(devfd, &sfds);
close(devfd); close(devfd);
devfd = -1; devfd = -1;
...@@ -1154,7 +1167,7 @@ capture(void) ...@@ -1154,7 +1167,7 @@ capture(void)
} }
} }
else { else if (xendomain) {
warning("xen console %s closed;" warning("xen console %s closed;"
" attempting to reopen", " attempting to reopen",
Devname); Devname);
...@@ -1169,6 +1182,22 @@ capture(void) ...@@ -1169,6 +1182,22 @@ capture(void)
fdcount = xsfd + 1; fdcount = xsfd + 1;
} }
} }
else {
warning("devfd %s closed;"
" attempting to reopen",
Devname);
nretries = 0;
while (rawmode(Devname,speed) != 0) {
if (maxretries > 0
&& nretries > maxretries) {
die("%s: failed to reopen (%d tries)",
Devname,nretries);
}
++nretries;
usleep(retryinterval
* 1000);
}
}
if (devfd >= 0) { if (devfd >= 0) {
FD_SET(devfd, &sfds); FD_SET(devfd, &sfds);
if (devfd >= fdcount) if (devfd >= fdcount)
......
#!/usr/bin/env python
#
# This is a simple python script that attaches to a Docker container,
# and exports its stdout/err and stdin as a pty.
#
# We have this little python gem because it isn't possible to easily
# override the perl LWP::Protocol::SocketUnixAlt thing to steal its
# socket. The standard python docker API wrapper, OTOH, gives it to us
# straightaway.
#
import os
import fcntl
import sys
import traceback
import pty
import docker
import select
if docker.version_info[0] < 2:
import docker.client
client = docker.client.AutoVersionClient()
socket = client.attach_socket(
resource_id=sys.argv[1],
params=dict(stdout=True,stderr=True,stdin=True,stream=True,logs=True))
else:
client = docker.from_env()
container = client.containers.get(sys.argv[1])
socket = container.attach_socket(
params=dict(stdout=True,stderr=True,stdin=True,stream=True,logs=True))
sockfd = socket.fileno()
flags = fcntl.fcntl(sockfd,fcntl.F_GETFL)
fcntl.fcntl(sockfd,fcntl.F_SETFL,flags | os.O_NONBLOCK)
(mfd,sfd) = pty.openpty()
print os.ttyname(sfd)
if len(sys.argv) > 2:
os.symlink(os.ttyname(sfd),sys.argv[2])
flags = fcntl.fcntl(mfd,fcntl.F_GETFL)
fcntl.fcntl(mfd,fcntl.F_SETFL,flags | os.O_NONBLOCK)
retval = 0
try:
(sockbuf,mbuf) = ('','')
while True:
(rlist,wlist,xlist) = select.select([sockfd,mfd],[sockfd,mfd],[],None)
if sockfd in rlist:
sockbuf += socket.recv(4096)
if mfd in rlist:
mbuf += os.read(mfd,4096)
if len(mbuf) and sockfd in wlist:
n = socket.send(mbuf)
if n < len(mbuf):
mbuf = mbuf[n:]
else:
mbuf = ''
if len(sockbuf) and mfd in wlist:
n = os.write(mfd,sockbuf)
if n < len(sockbuf):
sockbuf = sockbuf[n:]
else:
sockbuf = ''
except:
traceback.print_exc()
retval = -1
try:
os.close(mfd)
os.close(sfd)
socket.close()
except:
pass
if len(sys.argv) > 2:
os.unlink(sys.argv[2])
sys.exit(retval)
...@@ -230,6 +230,29 @@ my $NEW_LVM = 0; ...@@ -230,6 +230,29 @@ my $NEW_LVM = 0;
# could take to pull a large Docker image. This is a wild guess, obviously. # could take to pull a large Docker image. This is a wild guess, obviously.
# #
my $MAXIMAGEWAIT = 1800; my $MAXIMAGEWAIT = 1800;
#
# Serial console handling. We fire up a capture per active vnode.
# We use a fine assortment of capture options:
#
# -i: standalone mode, don't try to contact capserver directly
# -l: (added later) set directory where log, ACL, and pid files are kept.
# -C: use a circular buffer to capture activity while no user
# is connected. This gets dumped to the user when they connect.
# -T: Put out a timestamp if there has been no previous output
# for at least 10 seconds.
# -L: In conjunction with -T, the timestamp message includes how
# long it has been since the last output.
# -R: Retry interval of 1 second. When capture is disconnected
# from the pty (due to container reboot/shutdowns), this is how
# long we wait between attempts to reconnect.
# -y: When capture disconnects from the pty, we retry forever to reopen.
# -A: tell capture not to prepend '/dev' to the device path we supply.
#
my $CAPTURE = "/usr/local/sbin/capture-nossl";
my $CAPTUREOPTS = "-i -C -L -T 10 -R 1000 -y -1 -A";
my $C2P = "/usr/local/etc/emulab/container2pty.py";
# #
# Create a thin pool with the name $POOL_NAME using not more # Create a thin pool with the name $POOL_NAME using not more
# than $POOL_FRAC of any disk. # than $POOL_FRAC of any disk.
...@@ -373,6 +396,8 @@ sub RunProxies($$); ...@@ -373,6 +396,8 @@ sub RunProxies($$);
sub KillProxies($$); sub KillProxies($$);
sub InsertPostBootIptablesRules($$$$); sub InsertPostBootIptablesRules($$$$);
sub RemovePostBootIptablesRules($$$$); sub RemovePostBootIptablesRules($$$$);
sub captureRunning($);
sub captureStart($$);
# #
# A single client object per load of this file is safe. # A single client object per load of this file is safe.
...@@ -384,7 +409,7 @@ sub getClient() ...@@ -384,7 +409,7 @@ sub getClient()
return $_CLIENT return $_CLIENT
if (defined($_CLIENT)); if (defined($_CLIENT));
# Load late, because this requires a bunch of deps we might have # Load late, because this requires a bunch of deps we might have
# installed in ensurePerlDeps(). # installed in ensureDeps().
require dockerclient; require dockerclient;
$_CLIENT = dockerclient->new(); $_CLIENT = dockerclient->new();
$_CLIENT->debug($apidebug); $_CLIENT->debug($apidebug);
...@@ -571,7 +596,7 @@ sub refreshNetworkDeviceMaps() ...@@ -571,7 +596,7 @@ sub refreshNetworkDeviceMaps()
} }
} }
sub ensurePerlDeps() sub ensureDeps()
{ {
if (aptNotInstalled("libwww-perl")) { if (aptNotInstalled("libwww-perl")) {
aptGetInstall("libwww-perl"); aptGetInstall("libwww-perl");
...@@ -591,6 +616,9 @@ sub ensurePerlDeps() ...@@ -591,6 +616,9 @@ sub ensurePerlDeps()
if ($@) { if ($@) {
mysystem("cpan -i LWP::Protocol::http::SocketUnixAlt"); mysystem("cpan -i LWP::Protocol::http::SocketUnixAlt");
} }
if (aptNotInstalled("python-docker")) {
aptGetInstall("python-docker");
}
} }
# (Must be called only after refreshNetworkDeviceMaps() is called for # (Must be called only after refreshNetworkDeviceMaps() is called for
...@@ -1109,7 +1137,7 @@ sub rootPreConfig($) ...@@ -1109,7 +1137,7 @@ sub rootPreConfig($)
# #
# Make sure we have all our Perl deps. # Make sure we have all our Perl deps.
# #
ensurePerlDeps(); ensureDeps();
# #
# Make sure we have a bunch of other common tools. # Make sure we have a bunch of other common tools.
...@@ -2712,6 +2740,19 @@ sub vnodeCreate($$$$) ...@@ -2712,6 +2740,19 @@ sub vnodeCreate($$$$)
if ($debug) { if ($debug) {
print STDERR "container_create($vnode_id) args:\n".Dumper(%args)."\n"; print STDERR "container_create($vnode_id) args:\n".Dumper(%args)."\n";
} }
#
# Kill off a capture that might be running for this container.
#
if (-x "$CAPTURE") {
my $rpid = captureRunning($vnode_id);
if ($rpid) {
print STDERR "WARNING: capture already running ($rpid)!?".
" Killing...\n";
kill("TERM", $rpid);
sleep(1);
}
}
# #
# Go ahead and create. # Go ahead and create.
...@@ -3078,6 +3119,33 @@ sub vnodeBootHook($$$$) ...@@ -3078,6 +3119,33 @@ sub vnodeBootHook($$$$)
my ($vnode_id, $vmid, $vnconfig, $private) = @_; my ($vnode_id, $vmid, $vnconfig, $private) = @_;
my $vninfo = $private; my $vninfo = $private;
#
# Start up our Docker-to-pty script for this container; the capture
# will attach to it. We always fire this off here; it cannot
# survive when the container reboots or shuts down.
#
my $PTYLINKFILE = "$VMDIR/$vnode_id/vnode.pty";
TBDebugTimeStamp("vnodeBootHook: starting container2pty;".
" symlink $PTYLINKFILE");
mysystem("$C2P $vnode_id $PTYLINKFILE &");
# Wait 5 seconds to ensure $PTYLINKFILE appears...
my $tries = 10;
while (! -e $PTYLINKFILE && $tries > 0) {
sleep(1);
$tries -= 1;
TBDebugTimeStamp("vnodeBootHook: waiting for $PTYLINKFILE...");
}
#
# Start a capture if there isn't one running.
#
if (-x "$CAPTURE") {
my $rpid = captureRunning($vnode_id);
if ($rpid == 0) {
captureStart($vnode_id,$PTYLINKFILE);
}
}
# #
# This function is not yet part of the libvnode API, but our # This function is not yet part of the libvnode API, but our
# vnodeBoot and vnodeReboot functions call it. # vnodeBoot and vnodeReboot functions call it.
...@@ -3501,6 +3569,52 @@ sub vnodeDestroy($$$$) ...@@ -3501,6 +3569,52 @@ sub vnodeDestroy($$$$)
delete($private->{'preboot_iptables_rules'}); delete($private->{'preboot_iptables_rules'});
} }
#
# Shutdown the capture now that it is gone. We leave the log around
# til next time this vnode comes back.
#
if (-x "$CAPTURE") {
my $LOGPATH = "$VMDIR/$vnode_id";
my $pidfile = "$LOGPATH/$vnode_id.pid";
my $pid = 0;
if (-r "$pidfile" && open(PID, "<$pidfile")) {
my $pid = <PID>;
close(PID);
chomp($pid);
if ($pid =~ /^(\d+)$/ && $1 > 1) {
$pid = $1;
} else {
print STDERR "WARNING: bogus pid in capture pidfile ($pid)\n";
$pid = 0;
}
}
# XXX sanity: make sure pidfile matches reality
my $rpid = captureRunning($vnode_id);
if ($rpid == 0) {
print STDERR "WARNING: capture not running";
if ($pid > 0) {
print STDERR ", should have been pid $pid";
$pid = 0;
}
print STDERR "\n";
} elsif ($pid != $rpid) {
if ($pid == 0) {
print STDERR "WARNING: no recorded capture pid, ".
"but found process ($rpid)\n";
} else {
print STDERR "WARNING: recorded capture pid ($pid) ".
"does not match actual pid ($rpid)\n";
}
$pid = $rpid;
}
if ($pid > 0) {
kill("TERM", $pid);
}
}
# Kill the chains. # Kill the chains.
# Ick, iptables has a 28 character limit on chain names. But we have to # Ick, iptables has a 28 character limit on chain names. But we have to
# be backwards compatible with existing chain names. See corresponding # be backwards compatible with existing chain names. See corresponding
...@@ -5827,6 +5941,82 @@ sub hostControlNet() ...@@ -5827,6 +5941,82 @@ sub hostControlNet()
die("hostControlNet: could not create control net virtual IP"); die("hostControlNet: could not create control net virtual IP");
} }
#
# If there is a capture running for the indicated vnode, return the pid.
# Otherwise return 0.
#
# Note: we do not use the pidfile here! This is all about sanity checking.
#
sub captureRunning($)
{
my ($vnode_id) = @_;
my $LOGPATH = "$VMDIR/$vnode_id";
my $rpid = `pgrep -f '^$CAPTURE .*-l $LOGPATH $vnode_id'`;
if ($? == 0) {
chomp($rpid);
if ($rpid =~ /^(\d+)$/) {
return $1;
}
}
return 0;
}
sub captureStart($$)
{
my ($vnode_id,$ptyfile) = @_;
my $LOGPATH = "$VMDIR/$vnode_id";
my $acl = "$LOGPATH/$vnode_id.acl";
my $logfile = "$LOGPATH/$vnode_id.log";
my $pidfile = "$LOGPATH/$vnode_id.pid";
# unlink ACL file so that we know when capture has started
unlink($acl)
if (-e $acl);
# remove old log file before start
unlink($logfile)
if (-e $logfile);
# and old pid file
unlink($pidfile)
if (-e $pidfile);
TBDebugTimeStamp("captureStart: starting capture on pty symlink $ptyfile");
# XXX see start of file for meaning of the options
mysystem2("$CAPTURE $CAPTUREOPTS -l $LOGPATH $vnode_id $ptyfile");
#
# We need to report the ACL info to capserver via tmcc. But do not
# hang, use timeout. Also need to wait for the acl file, since
# capture is running in the background.
#
if (! $?) {
for (my $i = 0; $i < 10; $i++) {
last
if (-e $acl && -s $acl);
print "waiting 1 sec for capture ACL file...\n" if ($sleepdebug);
sleep(1);
}
if (! (-e $acl && -s $acl)) {
print STDERR "WARNING: $acl does not exist after 10 seconds; ".
"capture may not have started correctly.\n";
}
else {
if (mysystem2("$BINDIR/tmcc.bin -n $vnode_id -t 5 ".
" -f $acl tiplineinfo")) {
print STDERR "WARNING: could not report tiplineinfo; ".
"remote console connections may not work.\n";
}
}
} else {
print STDERR "WARNING: capture not started!\n";
}
}
# convert 123456 into 12:34:56 # convert 123456 into 12:34:56
sub fixupMac($) sub fixupMac($)
{ {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment