More changes to get our root pubkey ducks in a row.

See emulab/emulab-devel issue #303. Ensure we have a controlled set of
pubkeys in root's .ssh/authorized_keys file when we create and load new
images. But allow for a user added key to survive node reboots if they
customize it within an experiment.

clientside/tmcc/common/config/rc.localize

 #!/usr/bin/perl -w
 #
-# Copyright (c) 2004-2012 University of Utah and the Flux Group.
+# Copyright (c) 2004-2017 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -134,13 +134,13 @@ sub doboot()
     }
 
     #
-    # Write new pubkeys to root authkeys file. As a safety mechanism, back
-    # up old authkeys to authkeys2 file, which is also used by openssh sshd. 
+    # Ensure that the given keys are in root's authorized_keys file,
+    # put them there if not.
     #
     if (@pubkeys) {
 	my $authdir   = (WINDOWS() ? "/sshkeys/root" : "/root/.ssh");
 	my $authkeys  = $authdir . "/authorized_keys";
-	my $authkeys2 = $authkeys . "2";
+	my $authkeysold = $authkeys . ".old";
 	my $authkeysnew = $authkeys . ".new";
 	my $oldumask  = umask(022);
 
@@ -159,29 +159,75 @@ sub doboot()
 		    or fatal("Failed to chmod $authdir");
 	    }
 
-	    if (-e "$authkeys") {
-		system("cp -pf $authkeys $authkeys2") == 0
+	    if (-e $authkeys && ! -e $authkeysold) {
+		system("cp -pf $authkeys $authkeysold") == 0
 		    or fatal("Could not backup root ssh authorized_keys file");
 	    }
 	}
 
-	if (!open(AUTHKEYS, "> $authkeysnew")) {
+	#
+	# Hash for tracking our keys
+	#
+	my %keyhash = ();
+	foreach my $key (@pubkeys) {
+	    $keyhash{$key} = 0;
+	}
+
+	#
+	# Read the old keys, taking note of our keys that are already there.
+	# It is okay for there to be no existing file, we will just make a
+	# new one with Emulab keys.
+	#
+	my @lines = ();
+	if (open(OKEYS, "<$authkeys")) {
+	    while (my $key = <OKEYS>) {
+		chomp $key;
+		if ($key && $key !~ /^\s*#/ && exists($keyhash{$key})) {
+		    $keyhash{$key} = 1;
+		}
+		push @lines, $key;
+	    }
+	    close(OKEYS);
+	}
+
+	#
+	# Add any of our keys that are not already there
+	#
+	my $added = 0;
+	foreach my $key (@pubkeys) {
+	    if (!$keyhash{$key}) {
+		push @lines, $key;
+		$added++;
+	    }
+	}
+
+	#
+	# All the keys are there, nothing to do
+	#
+	if (!$added) {
+	    umask($oldumask);
+	    return 0;
+	}
+
+	#
+	# Otherwise write a new file with missing keys added
+	#
+	if (!open(NKEYS, ">$authkeysnew")) {
 	    warning("Could not open $authkeysnew: $!");
 	    umask($oldumask);
 	    return -1;
 	}
 	umask($oldumask);
-	print AUTHKEYS "#\n";
-	print AUTHKEYS "# DO NOT EDIT! This file auto generated at bootup.\n";
-	print AUTHKEYS "#\n";
     
-	foreach my $key (@pubkeys) {
-	    print AUTHKEYS "$key\n";
+	print NKEYS "# Updated by Emulab on " . scalar(localtime()) . "\n";
+	foreach my $line (@lines) {
+	    next
+		if ($line =~ /^# Updated by Emulab on/);
+	    print NKEYS "$line\n";
 	}
-	close(AUTHKEYS);
+	close(NKEYS);
 
-	if (system("cmp -s $authkeysnew $authkeys") &&
-	    system("mv -f $authkeysnew $authkeys")) {
+	if (system("mv -f $authkeysnew $authkeys")) {
 	    warning("Could not mv $authkeysnew to $authkeys");
 	}
 	return 0;

clientside/tmcc/common/localize_image

@@ -99,26 +99,17 @@ localize_image() {
 
     # Check the host keys.
     changehostkeys=0
-    if [ -e /etc/ssh/ssh_host_key ]; then
-	cmp -s /etc/ssh/ssh_host_key $MNT/etc/ssh/ssh_host_key
-	if [ $? -ne 0 ]; then
-	    changehostkeys=1
-	fi
-    fi
-    if [ -e /etc/ssh/ssh_host_rsa_key ]; then
-	cmp -s /etc/ssh/ssh_host_rsa_key $MNT/etc/ssh/ssh_host_rsa_key
-	if [ $? -ne 0 ]; then
-	    changehostkeys=1
-	fi
-    fi
-    if [ -e /etc/ssh/ssh_host_dsa_key ]; then
-	cmp -s /etc/ssh/ssh_host_dsa_key $MNT/etc/ssh/ssh_host_dsa_key
-	if [ $? -ne 0 ]; then
-	    changehostkeys=1
+    for k in "" dsa_ ecdsa_ ed25519_ rsa_; do
+	if [ -e /etc/ssh/ssh_host_${k}key ]; then
+	    cmp -s /etc/ssh/ssh_host_${k}key $MNT/etc/ssh/ssh_host_${k}key
+	    if [ $? -ne 0 ]; then
+		changehostkeys=1
+	    fi
 	fi
-    fi
+    done
+
     if [ $changehostkeys -eq 1 ]; then
-	echo "  updating /etc/ssh/hostkeys"
+	echo "  updating /etc/ssh host keys"
 	
 	if [ ! -d $MNT/etc/ssh ]; then
 	    mkdir -m 755 $MNT/etc/ssh || {
@@ -127,7 +118,7 @@ localize_image() {
 	    }
 	fi
 	cp -pf /etc/ssh/ssh_host_* $MNT/etc/ssh/ || {
-	    echo "Failed to create /etc/ssh/hostkeys"
+	    echo "Failed to update /etc/ssh host keys"
 	    return 1
 	}
     fi

clientside/tmcc/freebsd/prepare

@@ -49,6 +49,7 @@ my $ENTROPY	= "/var/db/entropy/*";
 my $LOADERCONF	= "/boot/loader.conf";
 my $PUBSUBCONF  = "/usr/local/etc/pubsubd.conf";
 my $PUBSUBEXPR  = "/usr/local/etc/pubsubd.expr";
+my $ROOTSSHDIR	= "/root/.ssh";
 
 #
 #
@@ -245,6 +246,21 @@ if (-f $HISTORY) {
     system("rm -f /root/.saves-*");
 }
 
+#
+# Remove /root/.ssh and then regenerate a clean version with only an
+# approved authorized_keys file.
+#
+print "Cleaning root's .ssh directory ...\n";
+if (system("rm -rf $ROOTSSHDIR.bak") ||
+    system("mv $ROOTSSHDIR $ROOTSSHDIR.bak")) {
+    die("Could not move $ROOTSSHDIR to $ROOTSSHDIR.bak");
+}
+if (! -x "$BINDIR/rc/rc.localize" || system("$BINDIR/rc/rc.localize boot")) {
+    system("rm -rf $ROOTSSHDIR");
+    system("mv $ROOTSSHDIR.bak $ROOTSSHDIR");
+    die("Could not clean root .ssh directory");
+}
+
 print "Cleaning mail spool files ...\n";
 system("rm -rf $MAILDIR/*");
 

clientside/tmcc/freebsd/slicefix

@@ -254,25 +254,166 @@ getloadervar() {
     echo $_val
 }
 
+#
+# Make sure /root/.ssh contains only an authorized_keys file with the boot
+# root pubkey.
+#
+# Called with arg=1 if you just want to see if anything is wrong (returns
+# non-zero if so), 0 to fix.
+#
+dofixauthkeys() {
+    _test=$1
+
+    if [ $_test -ne 0 ]; then
+	if [ ! -d /mnt/root/.ssh ]; then
+	    return 1
+	fi
+	if [ -x /usr/bin/stat ]; then
+	    _stat=`/usr/bin/stat -f '%u,%g,%p' /mnt/root/.ssh`
+	    if [ "$_stat" != "0,0,40700" ]; then
+		return 1
+	    fi
+	fi
+	if [ ! -e /mnt/root/.ssh/authorized_keys ]; then
+	    return 1
+	fi
+	if [ -e /mnt/root/.ssh/authorized_keys2 ]; then
+	    return 1
+	fi
+    fi
+
+    #
+    # If we are a localized MFS, we just need to use the authorized_keys2
+    # file from the MFS. Otherwise we get the key(s) from tmcd and put
+    # them into the MFS authorized_keys2 file.
+    #
+    if ! islocalized; then
+	rm -f /root/.ssh/authorized_keys2
+	_key=`$BINDIR/tmcc localization | grep 'ROOTPUBKEY=' | head -1 | \
+	      sed -e "s/^ROOTPUBKEY='//" | sed -e "s/'$//"`
+	if [ $? -ne 0 -o -z "$_key" ]; then
+	    echo "WARNING: no boss pubkey returned!"
+	else
+	    echo "$_key" > /root/.ssh/authorized_keys2
+	fi
+    fi
+
+    if [ $_test -ne 0 ]; then
+	cmp -s /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys
+	if [ $? -ne 0 ]; then
+	    return 1
+	fi
+    else
+	echo "  updating /root/.ssh"
+
+	# make sure /root/.ssh exists and has proper permissions
+	mkdir -p /mnt/root/.ssh
+	chown root:0 /mnt/root/.ssh
+	chmod 700 /mnt/root/.ssh
+	rm -f /mnt/root/.ssh/authorized_keys2
+
+	#
+	# XXX no proper pubkey, just leave the current file intact.
+	# XXX maybe we should just nuke it instead?
+	#
+	if [ ! -r /root/.ssh/authorized_keys2 ]; then
+	    return 0
+	fi
+
+	# create authkeys file with just root key(s)
+	rm -f /mnt/root/.ssh/authorized_keys 
+	cp /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys
+	chmod 644 /mnt/root/.ssh/authorized_keys
+    fi
+
+    return 0
+}
+
 #
 # Make sshd more secure by default: no password based login.
-# XXX argh! First use wins, so we have to comment out before adding ours!
+# We will fix if there are multiple settings of the same variable,
+# if it is set incorrectly, or it is not set at all.
+#
+# Called with arg=1 if you just want to see if anything is wrong (returns
+# non-zero if so), 0 to fix.
 #
 dofixsshd() {
-    echo "  updating /etc/ssh/sshd_config"
-    sed -i .preemulab \
-	-e 's;^Protocol;#Protocol;' \
-	-e 's;^PasswordAuth;#PasswordAuth;' \
-	-e 's;^ChallengeResp;#ChallengeResp;' \
-	-e 's;^PermitRootLogin;#PermitRootLogin;' /mnt/etc/ssh/sshd_config
-    cat <<EOF8 >>/mnt/etc/ssh/sshd_config
+    _test=$1
+
+    if [ $_test -ne 0 ]; then
+	# sshd_config doesn't exist, call it okay
+	if [ ! -f /mnt/etc/ssh/sshd_config ]; then
+	    echo "WARNING: no sshd_config found!"
+	    return 0
+	fi
 
+	# find all uncommented instances of variables we care about
+	OIFS="$IFS"
+	IFS='
+'
+	_fix=0
+	_valP=
+	_valPA=
+	_valCRA=
+	_valPRL=
+	for _opt in `grep -E '^(Protocol|PasswordAuthentication|ChallengeResponseAuthentication|PermitRootLogin) ' /mnt/etc/ssh/sshd_config`; do
+	    _k=${_opt%% *}
+	    _v=${_opt#* }
+	    case $_k in
+	    Protocol)
+		if [ -n "$_valP" -o "$_v" != "2" ]; then
+		    _fix=1
+		fi
+		_valP=$_v
+		;;
+	    PasswordAuthentication)
+		if [ -n "$_valPA" -o "$_v" != "no" ]; then
+		    _fix=1
+		fi
+		_valPA=$_v
+		;;
+	    ChallengeResponseAuthentication)
+		if [ -n "$_valCRA" -o "$_v" != "no" ]; then
+		    _fix=1
+		fi
+		_valCRA=$_v
+		;;
+	    PermitRootLogin)
+		if [ -n "$_valPRL" -o "$_v" != "without-password" ]; then
+		    _fix=1
+		fi
+		_valPRL=$_v
+		;;
+	    esac
+	done
+	IFS=$OIFS
+	# a var had wrong value or more than one setting, fix
+	if [ $_fix -ne 0 ]; then
+	    return 1
+	fi
+	# a var was not explicitly set, fix
+	if [ -z "$_valP" -o -z "$_valPA" -o -z "$_valCRA" -o -z "$_valPRL" ]; then
+	    return 1
+	fi
+    else
+	echo "  updating /etc/ssh/sshd_config"
+
+	sed -i .preemulab \
+	    -e '/^Protocol /d' \
+	    -e '/^PasswordAuthentication /d' \
+	    -e '/^ChallengeResponseAuthentication /d' \
+	    -e '/^PermitRootLogin /d' \
+	    -e '/^# Emulab/d' /mnt/etc/ssh/sshd_config
+	cat <<EOF8 >>/mnt/etc/ssh/sshd_config
 # Emulab config
 Protocol 2
 PasswordAuthentication no
 ChallengeResponseAuthentication no
 PermitRootLogin without-password
 EOF8
+    fi
+
+    return 0
 }
 
 dofreebsd() {
@@ -348,6 +489,7 @@ dofreebsd() {
 	changezone=0
 	changentp=0
 	changesshd=0
+	changeauth=0
 
 	fixit=0
 
@@ -511,37 +653,17 @@ dofreebsd() {
 		fi
 	    fi
 
-	    # Check the root keys.
-	    if [ -e /root/.ssh/authorized_keys2 ]; then
-		cmp -s /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys
-		if [ $? -ne 0 ]; then
-		    changerootkeys=1
-		    fixit=1
-		fi
-	    fi
+	    # Root keys are handled by dofixauthkeys() below
 	
 	    # Check the host keys.
-	    if [ -e /etc/ssh/ssh_host_key ]; then
-		cmp -s /etc/ssh/ssh_host_key /mnt/etc/ssh/ssh_host_key
-		if [ $? -ne 0 ]; then
-		    changehostkeys=1
-		    fixit=1
-		fi
-	    fi
-	    if [ -e /etc/ssh/ssh_host_rsa_key ]; then
-		cmp -s /etc/ssh/ssh_host_rsa_key /mnt/etc/ssh/ssh_host_rsa_key
-		if [ $? -ne 0 ]; then
-		    changehostkeys=1
-		    fixit=1
-		fi
-	    fi
-	    if [ -e /etc/ssh/ssh_host_dsa_key ]; then
-		cmp -s /etc/ssh/ssh_host_dsa_key /mnt/etc/ssh/ssh_host_dsa_key
-		if [ $? -ne 0 ]; then
-		    changehostkeys=1
-		    fixit=1
+	    for k in "" dsa_ ecdsa_ ed25519_ rsa_; do
+		if [ -e /etc/ssh/ssh_host_${k}key ]; then
+		    cmp -s /etc/ssh/ssh_host_${k}key /mnt/etc/ssh/ssh_host_${k}key
+		    if [ $? -ne 0 ]; then
+			changehostkeys=1
+		    fi
 		fi
-	    fi
+	    done
 
 	    # Check the time zone.
 	    if [ -e /etc/localtime ]; then
@@ -563,12 +685,18 @@ dofreebsd() {
 
 	fi
 
-	if [ -r /mnt/etc/ssh/sshd_config ] && \
-	       ! grep -q '^# Emulab config' /mnt/etc/ssh/sshd_config; then
+	# See if we need to update sshd_config
+	if ! dofixsshd 1; then
 	    changesshd=1
 	    fixit=1
 	fi
 
+	# See if we need to update root's .ssh directory
+	if ! dofixauthkeys 1; then
+	    changeauth=1
+	    fixit=1
+	fi
+
 	if [ $fixit -eq 0 ]; then
 		echo "  no changes necessary"
 		umount $rootdev
@@ -796,29 +924,7 @@ EOF1
 	    }
 	fi
 
-	# Copy in new root keys
-	if [ $changerootkeys -eq 1 ]; then
-	    echo "  updating /root/.ssh/authorized_keys"
-
-	    if [ ! -d /mnt/root/.ssh ]; then
-		mkdir -m 700 /mnt/root/.ssh || {
-	            echo "Failed to mkdir /root/.ssh"
-		    umount $rootdev
-		    return 1
-		}
-	    fi
-	    # copy to both authorized_keys and _keys2
-	    cp -p /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys || {
-	        echo "Failed to create /root/.ssh/authorized_keys"
-		umount $rootdev
-		return 1
-	    }
-	    cp -p /root/.ssh/authorized_keys2 /mnt/root/.ssh/ || {
-	        echo "Failed to create /root/.ssh/authorized_keys2"
-		umount $rootdev
-		return 1
-	    }
-	fi
+	# Copy in new root keys handled by dofixauthkeys() below
 
 	# Copy in new host keys
 	if [ $changehostkeys -eq 1 ]; then
@@ -861,7 +967,11 @@ EOF1
 	fi
 
 	if [ $changesshd -eq 1 ]; then
-	    dofixsshd
+	    dofixsshd 0
+	fi
+
+	if [ $changeauth -eq 1 ]; then
+	    dofixauthkeys 0
 	fi
 
 	# actually run any postconfig scripts if we're supposed to:
@@ -1425,9 +1535,15 @@ EOF7
 	#
 	# Fixup sshd config
 	#
-	if [ -r /mnt/etc/ssh/sshd_config ] && \
-	       ! grep -q '^# Emulab config' /mnt/etc/ssh/sshd_config; then
-	    dofixsshd
+	if ! dofixsshd 1; then
+	    dofixsshd 0
+	fi
+
+	#
+	# Fixup root authorized keys
+	#
+	if ! dofixauthkeys 1; then
+	    dofixauthkeys 0
 	fi
 
 	#

clientside/tmcc/linux/prepare

@@ -53,6 +53,7 @@ my $ANACRON     = "/usr/sbin/anacron";
 my $GRUBDEF	= "/etc/default/grub";
 my $PUBSUBCONF  = "/usr/local/etc/pubsubd.conf";
 my $PUBSUBEXPR  = "/usr/local/etc/pubsubd.expr";
+my $ROOTSSHDIR	= "/root/.ssh";
 
 #
 # Dead wood in $BINDIR
@@ -351,6 +352,21 @@ if (-f "/root/$HISTORY") {
         die("Could not unlink /root/$HISTORY: $!");
 }
 
+#
+# Remove /root/.ssh and then regenerate a clean version with only an
+# approved authorized_keys file.
+#
+print "Cleaning root's .ssh directory ...\n";
+if (system("rm -rf $ROOTSSHDIR.bak") ||
+    system("mv $ROOTSSHDIR $ROOTSSHDIR.bak")) {
+    die("Could not move $ROOTSSHDIR to $ROOTSSHDIR.bak");
+}
+if (! -x "$BINDIR/rc/rc.localize" || system("$BINDIR/rc/rc.localize boot")) {
+    system("rm -rf $ROOTSSHDIR");
+    system("mv $ROOTSSHDIR.bak $ROOTSSHDIR");
+    die("Could not clean root .ssh directory");
+}
+
 print "Cleaning mail spool files ...\n";
 system("rm -rf $MAILDIR/*");
 

clientside/tmcc/linux/slicefix

@@ -269,19 +269,166 @@ getloadervar() {
     echo $_val
 }
 
+#
+# Make sure /root/.ssh contains only an authorized_keys file with the boot
+# root pubkey.
+#
+# Called with arg=1 if you just want to see if anything is wrong (returns
+# non-zero if so), 0 to fix.
+#
+dofixauthkeys() {
+    _test=$1
+
+    if [ $_test -ne 0 ]; then
+	if [ ! -d /mnt/root/.ssh ]; then
+	    return 1
+	fi
+	if [ -x /usr/bin/stat ]; then
+	    _stat=`/usr/bin/stat -f '%u,%g,%p' /mnt/root/.ssh`
+	    if [ "$_stat" != "0,0,40700" ]; then
+		return 1
+	    fi
+	fi
+	if [ ! -e /mnt/root/.ssh/authorized_keys ]; then
+	    return 1
+	fi
+	if [ -e /mnt/root/.ssh/authorized_keys2 ]; then
+	    return 1
+	fi
+    fi
+
+    #
+    # If we are a localized MFS, we just need to use the authorized_keys2
+    # file from the MFS. Otherwise we get the key(s) from tmcd and put
+    # them into the MFS authorized_keys2 file.
+    #
+    if ! islocalized; then
+	rm -f /root/.ssh/authorized_keys2
+	_key=`$BINDIR/tmcc localization | grep 'ROOTPUBKEY=' | head -1 | \
+	      sed -e "s/^ROOTPUBKEY='//" | sed -e "s/'$//"`
+	if [ $? -ne 0 -o -z "$_key" ]; then
+	    echo "WARNING: no boss pubkey returned!"
+	else
+	    echo "$_key" > /root/.ssh/authorized_keys2
+	fi
+    fi
+
+    if [ $_test -ne 0 ]; then
+	cmp -s /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys
+	if [ $? -ne 0 ]; then
+	    return 1
+	fi
+    else
+	echo "  updating /root/.ssh"
+
+	# make sure /root/.ssh exists and has proper permissions
+	mkdir -p /mnt/root/.ssh
+	chown root:0 /mnt/root/.ssh
+	chmod 700 /mnt/root/.ssh
+	rm -f /mnt/root/.ssh/authorized_keys2
+
+	#
+	# XXX no proper pubkey, just leave the current file intact.
+	# XXX maybe we should just nuke it instead?
+	#
+	if [ ! -r /root/.ssh/authorized_keys2 ]; then
+	    return 0
+	fi
+
+	# create authkeys file with just root key(s)
+	rm -f /mnt/root/.ssh/authorized_keys 
+	cp /root/.ssh/authorized_keys2 /mnt/root/.ssh/authorized_keys
+	chmod 644 /mnt/root/.ssh/authorized_keys
+    fi
+
+    return 0
+}
+
 #
 # Make sshd more secure by default: no password based login.
+# We will fix if there are multiple settings of the same variable,
+# if it is set incorrectly, or it is not set at all.
+#
+# Called with arg=1 if you just want to see if anything is wrong (returns
+# non-zero if so), 0 to fix.
 #
 dofixsshd() {
-    echo "  updating /etc/ssh/sshd_config"
-    cat <<EOF8 >>/mnt/etc/ssh/sshd_config
+    _test=$1
+
+    if [ $_test -ne 0 ]; then
+	# sshd_config doesn't exist, call it okay
+	if [ ! -f /mnt/etc/ssh/sshd_config ]; then
+	    echo "WARNING: no sshd_config found!"
+	    return 0
+	fi