diff --git a/clientside/os/capture/capture.c b/clientside/os/capture/capture.c index b6474c3a7d5ade0c13ccc5ee35393d08651b0382..3dd506b491c4e33c451bd28d55e5acb721c025ed 100644 --- a/clientside/os/capture/capture.c +++ b/clientside/os/capture/capture.c @@ -174,6 +174,8 @@ int remotemode; int programmode; char *xendomain; int retryinterval = 5000; +int maxretries = 0; +int nomodpath = 0; int maxfailures = 10; int failures; int upportnum = -1, upfd = -1, upfilefd = -1; @@ -381,7 +383,7 @@ main(int argc, char **argv) else Progname = *argv; - while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:")) != EOF) + while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:y:A")) != EOF) switch (op) { #ifdef USESOCKETS #ifdef WITHSSL @@ -428,6 +430,12 @@ main(int argc, char **argv) usage(); } break; + case 'y': + maxretries = atoi(optarg); + break; + case 'A': + nomodpath = 1; + break; #endif /* USESOCKETS */ case 'H': ++hwflow; @@ -510,6 +518,8 @@ main(int argc, char **argv) } else if (xendomain) strcpy(strbuf, xendomain); + else if (nomodpath) + strcpy(strbuf, argv[1]); else (void) snprintf(strbuf, sizeof(strbuf), DEVNAME, DEVPATH, argv[1]); @@ -997,6 +1007,8 @@ capture(void) sigset_t omask; char buf[BUFSIZE]; struct timeval timeout; + int nretries; + /* * XXX for now we make both directions non-blocking. This is a @@ -1128,7 +1140,8 @@ capture(void) Machine, cc); if (cc <= 0) { #ifdef USESOCKETS - if (remotemode || programmode || xendomain) { + if (remotemode || programmode || xendomain + || maxretries) { FD_CLR(devfd, &sfds); close(devfd); devfd = -1; @@ -1154,7 +1167,7 @@ capture(void) } } - else { + else if (xendomain) { warning("xen console %s closed;" " attempting to reopen", Devname); @@ -1169,6 +1182,22 @@ capture(void) fdcount = xsfd + 1; } } + else { + warning("devfd %s closed;" + " attempting to reopen", + Devname); + nretries = 0; + while (rawmode(Devname,speed) != 0) { + if (maxretries > 0 + && nretries > maxretries) { + die("%s: failed to reopen (%d tries)", + Devname,nretries); + } + ++nretries; + usleep(retryinterval + * 1000); + } + } if (devfd >= 0) { FD_SET(devfd, &sfds); if (devfd >= fdcount) diff --git a/clientside/tmcc/linux/docker/container2pty.py b/clientside/tmcc/linux/docker/container2pty.py new file mode 100644 index 0000000000000000000000000000000000000000..106c94fdb5527c3b03b5289cc5c18bbb3b72a7dc --- /dev/null +++ b/clientside/tmcc/linux/docker/container2pty.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +# +# This is a simple python script that attaches to a Docker container, +# and exports its stdout/err and stdin as a pty. +# +# We have this little python gem because it isn't possible to easily +# override the perl LWP::Protocol::SocketUnixAlt thing to steal its +# socket. The standard python docker API wrapper, OTOH, gives it to us +# straightaway. +# + +import os +import fcntl +import sys +import traceback +import pty +import docker +import select + +if docker.version_info[0] < 2: + import docker.client + client = docker.client.AutoVersionClient() + socket = client.attach_socket( + resource_id=sys.argv[1], + params=dict(stdout=True,stderr=True,stdin=True,stream=True,logs=True)) +else: + client = docker.from_env() + container = client.containers.get(sys.argv[1]) + socket = container.attach_socket( + params=dict(stdout=True,stderr=True,stdin=True,stream=True,logs=True)) +sockfd = socket.fileno() +flags = fcntl.fcntl(sockfd,fcntl.F_GETFL) +fcntl.fcntl(sockfd,fcntl.F_SETFL,flags | os.O_NONBLOCK) +(mfd,sfd) = pty.openpty() +print os.ttyname(sfd) +if len(sys.argv) > 2: + os.symlink(os.ttyname(sfd),sys.argv[2]) +flags = fcntl.fcntl(mfd,fcntl.F_GETFL) +fcntl.fcntl(mfd,fcntl.F_SETFL,flags | os.O_NONBLOCK) + +retval = 0 +try: + (sockbuf,mbuf) = ('','') + while True: + (rlist,wlist,xlist) = select.select([sockfd,mfd],[sockfd,mfd],[],None) + if sockfd in rlist: + sockbuf += socket.recv(4096) + if mfd in rlist: + mbuf += os.read(mfd,4096) + if len(mbuf) and sockfd in wlist: + n = socket.send(mbuf) + if n < len(mbuf): + mbuf = mbuf[n:] + else: + mbuf = '' + if len(sockbuf) and mfd in wlist: + n = os.write(mfd,sockbuf) + if n < len(sockbuf): + sockbuf = sockbuf[n:] + else: + sockbuf = '' +except: + traceback.print_exc() + retval = -1 + +try: + os.close(mfd) + os.close(sfd) + socket.close() +except: + pass +if len(sys.argv) > 2: + os.unlink(sys.argv[2]) +sys.exit(retval) diff --git a/clientside/tmcc/linux/docker/libvnode_docker.pm b/clientside/tmcc/linux/docker/libvnode_docker.pm index a2cd14a89dfed235564bf388bae127c4ec6d1704..e88970aae3b11c7828645d029f797a0dde92d711 100644 --- a/clientside/tmcc/linux/docker/libvnode_docker.pm +++ b/clientside/tmcc/linux/docker/libvnode_docker.pm @@ -230,6 +230,29 @@ my $NEW_LVM = 0; # could take to pull a large Docker image. This is a wild guess, obviously. # my $MAXIMAGEWAIT = 1800; + +# +# Serial console handling. We fire up a capture per active vnode. +# We use a fine assortment of capture options: +# +# -i: standalone mode, don't try to contact capserver directly +# -l: (added later) set directory where log, ACL, and pid files are kept. +# -C: use a circular buffer to capture activity while no user +# is connected. This gets dumped to the user when they connect. +# -T: Put out a timestamp if there has been no previous output +# for at least 10 seconds. +# -L: In conjunction with -T, the timestamp message includes how +# long it has been since the last output. +# -R: Retry interval of 1 second. When capture is disconnected +# from the pty (due to container reboot/shutdowns), this is how +# long we wait between attempts to reconnect. +# -y: When capture disconnects from the pty, we retry forever to reopen. +# -A: tell capture not to prepend '/dev' to the device path we supply. +# +my $CAPTURE = "/usr/local/sbin/capture-nossl"; +my $CAPTUREOPTS = "-i -C -L -T 10 -R 1000 -y -1 -A"; +my $C2P = "/usr/local/etc/emulab/container2pty.py"; + # # Create a thin pool with the name $POOL_NAME using not more # than $POOL_FRAC of any disk. @@ -373,6 +396,8 @@ sub RunProxies($$); sub KillProxies($$); sub InsertPostBootIptablesRules($$$$); sub RemovePostBootIptablesRules($$$$); +sub captureRunning($); +sub captureStart($$); # # A single client object per load of this file is safe. @@ -384,7 +409,7 @@ sub getClient() return $_CLIENT if (defined($_CLIENT)); # Load late, because this requires a bunch of deps we might have - # installed in ensurePerlDeps(). + # installed in ensureDeps(). require dockerclient; $_CLIENT = dockerclient->new(); $_CLIENT->debug($apidebug); @@ -571,7 +596,7 @@ sub refreshNetworkDeviceMaps() } } -sub ensurePerlDeps() +sub ensureDeps() { if (aptNotInstalled("libwww-perl")) { aptGetInstall("libwww-perl"); @@ -591,6 +616,9 @@ sub ensurePerlDeps() if ($@) { mysystem("cpan -i LWP::Protocol::http::SocketUnixAlt"); } + if (aptNotInstalled("python-docker")) { + aptGetInstall("python-docker"); + } } # (Must be called only after refreshNetworkDeviceMaps() is called for @@ -1109,7 +1137,7 @@ sub rootPreConfig($) # # Make sure we have all our Perl deps. # - ensurePerlDeps(); + ensureDeps(); # # Make sure we have a bunch of other common tools. @@ -2712,6 +2740,19 @@ sub vnodeCreate($$$$) if ($debug) { print STDERR "container_create($vnode_id) args:\n".Dumper(%args)."\n"; } + + # + # Kill off a capture that might be running for this container. + # + if (-x "$CAPTURE") { + my $rpid = captureRunning($vnode_id); + if ($rpid) { + print STDERR "WARNING: capture already running ($rpid)!?". + " Killing...\n"; + kill("TERM", $rpid); + sleep(1); + } + } # # Go ahead and create. @@ -3078,6 +3119,33 @@ sub vnodeBootHook($$$$) my ($vnode_id, $vmid, $vnconfig, $private) = @_; my $vninfo = $private; + # + # Start up our Docker-to-pty script for this container; the capture + # will attach to it. We always fire this off here; it cannot + # survive when the container reboots or shuts down. + # + my $PTYLINKFILE = "$VMDIR/$vnode_id/vnode.pty"; + TBDebugTimeStamp("vnodeBootHook: starting container2pty;". + " symlink $PTYLINKFILE"); + mysystem("$C2P $vnode_id $PTYLINKFILE &"); + # Wait 5 seconds to ensure $PTYLINKFILE appears... + my $tries = 10; + while (! -e $PTYLINKFILE && $tries > 0) { + sleep(1); + $tries -= 1; + TBDebugTimeStamp("vnodeBootHook: waiting for $PTYLINKFILE..."); + } + + # + # Start a capture if there isn't one running. + # + if (-x "$CAPTURE") { + my $rpid = captureRunning($vnode_id); + if ($rpid == 0) { + captureStart($vnode_id,$PTYLINKFILE); + } + } + # # This function is not yet part of the libvnode API, but our # vnodeBoot and vnodeReboot functions call it. @@ -3501,6 +3569,52 @@ sub vnodeDestroy($$$$) delete($private->{'preboot_iptables_rules'}); } + # + # Shutdown the capture now that it is gone. We leave the log around + # til next time this vnode comes back. + # + if (-x "$CAPTURE") { + my $LOGPATH = "$VMDIR/$vnode_id"; + my $pidfile = "$LOGPATH/$vnode_id.pid"; + my $pid = 0; + + if (-r "$pidfile" && open(PID, "<$pidfile")) { + my $pid = ; + close(PID); + chomp($pid); + if ($pid =~ /^(\d+)$/ && $1 > 1) { + $pid = $1; + } else { + print STDERR "WARNING: bogus pid in capture pidfile ($pid)\n"; + $pid = 0; + } + } + + # XXX sanity: make sure pidfile matches reality + my $rpid = captureRunning($vnode_id); + if ($rpid == 0) { + print STDERR "WARNING: capture not running"; + if ($pid > 0) { + print STDERR ", should have been pid $pid"; + $pid = 0; + } + print STDERR "\n"; + } elsif ($pid != $rpid) { + if ($pid == 0) { + print STDERR "WARNING: no recorded capture pid, ". + "but found process ($rpid)\n"; + } else { + print STDERR "WARNING: recorded capture pid ($pid) ". + "does not match actual pid ($rpid)\n"; + } + $pid = $rpid; + } + + if ($pid > 0) { + kill("TERM", $pid); + } + } + # Kill the chains. # Ick, iptables has a 28 character limit on chain names. But we have to # be backwards compatible with existing chain names. See corresponding @@ -5827,6 +5941,82 @@ sub hostControlNet() die("hostControlNet: could not create control net virtual IP"); } + +# +# If there is a capture running for the indicated vnode, return the pid. +# Otherwise return 0. +# +# Note: we do not use the pidfile here! This is all about sanity checking. +# +sub captureRunning($) +{ + my ($vnode_id) = @_; + my $LOGPATH = "$VMDIR/$vnode_id"; + + my $rpid = `pgrep -f '^$CAPTURE .*-l $LOGPATH $vnode_id'`; + if ($? == 0) { + chomp($rpid); + if ($rpid =~ /^(\d+)$/) { + return $1; + } + } + + return 0; +} + +sub captureStart($$) +{ + my ($vnode_id,$ptyfile) = @_; + my $LOGPATH = "$VMDIR/$vnode_id"; + my $acl = "$LOGPATH/$vnode_id.acl"; + my $logfile = "$LOGPATH/$vnode_id.log"; + my $pidfile = "$LOGPATH/$vnode_id.pid"; + + # unlink ACL file so that we know when capture has started + unlink($acl) + if (-e $acl); + + # remove old log file before start + unlink($logfile) + if (-e $logfile); + + # and old pid file + unlink($pidfile) + if (-e $pidfile); + + TBDebugTimeStamp("captureStart: starting capture on pty symlink $ptyfile"); + + # XXX see start of file for meaning of the options + mysystem2("$CAPTURE $CAPTUREOPTS -l $LOGPATH $vnode_id $ptyfile"); + + # + # We need to report the ACL info to capserver via tmcc. But do not + # hang, use timeout. Also need to wait for the acl file, since + # capture is running in the background. + # + if (! $?) { + for (my $i = 0; $i < 10; $i++) { + last + if (-e $acl && -s $acl); + print "waiting 1 sec for capture ACL file...\n" if ($sleepdebug); + sleep(1); + } + if (! (-e $acl && -s $acl)) { + print STDERR "WARNING: $acl does not exist after 10 seconds; ". + "capture may not have started correctly.\n"; + } + else { + if (mysystem2("$BINDIR/tmcc.bin -n $vnode_id -t 5 ". + " -f $acl tiplineinfo")) { + print STDERR "WARNING: could not report tiplineinfo; ". + "remote console connections may not work.\n"; + } + } + } else { + print STDERR "WARNING: capture not started!\n"; + } +} + # convert 123456 into 12:34:56 sub fixupMac($) {