Commit 2489c09b authored by Leigh B Stoller's avatar Leigh B Stoller

Changes to support XEN shared nodes and guest snapshots.

Snapshots are done a little differently then openvz of course, since
there are potentially multiple disk partitions and a kernel. The basic
operation is:

1. Fire off reboot_prepare from boss. Changes to reboot_prepare result
   in the guest "halting" insted of rebooting.

2. Fire off the create-image client script, which will take imagezips
   of all of the disks (except the swap partition), and grab a copy of
   the kernel. A new xm.conf file is written, and then the directory
   is first tar'ed and then we imagezip that bundle for upload.

3. When booting a guest, we now look for guest images that are
   packaged in this way, although we still support the older method
   for backwards compatability. All of the disks are restored, and a
   new xm.conf created that points to the new kernel.
parent 86d5e5ef
......@@ -206,6 +206,16 @@ if ($BOSSIP !~ /^\d+\.\d+\.\d+\.\d+$/) {
die "Bad bossip '$BOSSIP' from bossinfo!";
}
#
# This holds the container state set up by the library. There is state
# added here, and state added in the library ("private"). We locally
# redefine this below, so cannot be a lexical.
#
# NOTE: There should be NO state in here that needs to survive reboot.
# We just remove them all when rebooting. See above.
#
$vnstate = { "private" => {} };
#
# Quickie way to show the state.
#
......@@ -216,11 +226,26 @@ if ($showstate) {
if (! -e "$VNDIR/vnode.state") {
fatal("no vnode.state file for $vnodeid");
}
my $str = `cat $VNDIR/vnode.info`;
($vmid, $vmtype, undef) = ($str =~ /^(\d*) (\w*) ([-\w]*)$/);
my $tmp = eval { Storable::retrieve("$VNDIR/vnode.state"); };
if ($@) {
fatal("$@");
}
print Dumper($tmp);
# So the lib op works.
$vnstate = $tmp;
($ret,$err) = safeLibOp('vnodeState', 1, 0);
if ($err) {
fatal("Failed to get status for existing container: $err");
}
if ($ret eq VNODE_STATUS_UNKNOWN()) {
print "Cannot determine status container $vmid.\n";
}
print "Domain is $ret\n";
exit(0);
}
......@@ -267,16 +292,6 @@ if ($cleanup) {
exit(TearDownStaleVM());
}
#
# This holds the container state set up by the library. There is state
# added here, and state added in the library ("private"). We locally
# redefine this below, so cannot be a lexical.
#
# NOTE: There should be NO state in here that needs to survive reboot.
# We just remove them all when rebooting. See above.
#
$vnstate = { "private" => {} };
#
# Now we can start doing something useful.
#
......@@ -439,6 +454,19 @@ if (-e "$VNDIR/vnode.info") {
$teardown = 1;
}
else {
# We (might) need this to discover the state.
local $vnstate = { "private" => {} };
if (-e "$VNDIR/vnode.state") {
my $tmp = eval { Storable::retrieve("$VNDIR/vnode.state"); };
if ($@) {
print STDERR "$@";
$teardown = 1;
}
else {
$vnstate->{'private'} = $tmp->{'private'};
}
}
($ret,$err) = safeLibOp('vnodeState', 1, 0);
if ($err) {
fatal("Failed to get status for existing container: $err");
......@@ -460,40 +488,6 @@ if (-e "$VNDIR/vnode.info") {
}
}
#
# Another wrinkle; tagged vlans might not be setup yet when we get
# here, and we have to know those tags before we can proceed. We
# need to spin, but with signals enabled since we do not want to
# wait forever. Okay to get a signal and die at this point.
#
if (0 && @{ $vnconfig{'ifconfig'} }) {
again:
foreach my $ifc (@{ $vnconfig{'ifconfig'} }) {
my $lan = $ifc->{LAN};
next
if ($ifc->{ITYPE} ne "vlan");
# got the tag.
next
if ($ifc->{VTAG});
# no tag, wait and ask again.
print STDERR
"$lan does not have a tag yet. Waiting, then asking again ...\n";
sleep(5);
my @tmp = ();
fatal("getifconfig($vnodeid): $!")
if (getifconfig(\@tmp));
$vnconfig{"ifconfig"} = [ @tmp ];
# Just look through everything again; simple.
goto again;
}
}
#
# Install handlers *after* down stale container teardown, since we set
# them to IGNORE during the teardown.
......
......@@ -30,12 +30,13 @@ use File::Basename;
sub usage()
{
print "Usage: capturevm.pl [-d] vnodeid [role]\n" .
print "Usage: capturevm.pl [-d] [-r role] vnodeid\n" .
" -d Debug mode.\n".
" -r role (like boss or ops) to use instead of vnodeid.\n".
" -i Info mode only\n";
exit(-1);
}
my $optlist = "dix:";
my $optlist = "dix:r:";
my $debug = 1;
my $infomode = 0;
my $VMPATH = "/var/emulab/vms/vminfo";
......@@ -70,11 +71,14 @@ if (defined($options{"d"})) {
if (defined($options{"i"})) {
$infomode = 1;
}
if (defined($options{"r"})) {
$role = 1;
}
usage()
if (@ARGV < 1 || @ARGV > 2);
if (@ARGV != 1);
my $vnodeid = $ARGV[0];
$role = $ARGV[1] if (@ARGV == 2);
$role = $vnodeid if (!defined($role));
if (defined($options{"x"})) {
$XMINFO = $options{"x"};
......@@ -85,7 +89,7 @@ else {
CreateExtraFS();
system("mkdir $EXTRAFS/$role")
if (defined($role) && ! -e "$EXTRAFS/$role");
if (! -e "$EXTRAFS/$role");
#
# We need this file to figure out the disk info.
......@@ -123,9 +127,7 @@ $xminfo{"disksizes"} = "";
if (! -e $xminfo{"kernel"}) {
Fatal($xminfo{"kernel"} . " does not exist");
}
my $kernel = $EXTRAFS;
$kernel .= "/$role" if (defined($role));
$kernel .= "/" . basename($xminfo{"kernel"});
my $kernel = "$EXTRAFS/$role/" . basename($xminfo{"kernel"});
system("cp " . $xminfo{"kernel"} . " $kernel") == 0
or Fatal("Could not copy kernel to $kernel");
$xminfo{"kernel"} = basename($xminfo{"kernel"});
......@@ -160,9 +162,7 @@ foreach my $device (keys(%diskinfo)) {
else {
fatal("Could not parse $spec");
}
$filename = $dev;
$filename = "$role/$filename"
if (defined($role));
$filename = "$role/$dev";
#
# Figure out the size of the LVM.
......@@ -203,7 +203,7 @@ foreach my $device (keys(%diskinfo)) {
}
else {
if (! ($device =~ /disk/)) {
$opts = "-b -f";
$opts = "-b";
}
}
if ($infomode) {
......@@ -230,7 +230,7 @@ else {
#
$xminfo{"vif"} =~ s/,\s*script=[^\']*//g;
$XMINFO = (defined($role) ? "$EXTRAFS/$role/xm.conf" : "$EXTRAFS/xm.conf");
$XMINFO = "$EXTRAFS/$role/xm.conf";
open(XM, ">$XMINFO")
or fatal("Could not open $XMINFO: $!");
foreach my $key (keys(%xminfo)) {
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use English;
use Getopt::Std;
use strict;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
my $VNODESETUP = "$BINDIR/vnodesetup";
my $CAPTURE = "$BINDIR/capturevm.pl";
my $EXTRAFS = "/scratch";
my $TAR = "/bin/tar";
#
# Client-side to create a disk image. Caller must have sudo permission!
# This is the OpenVZ specific version.
#
sub usage()
{
print STDOUT "Usage: create-image [-S image-server] [-F imageid] ".
"<vnodeid> <filename>\n";
exit(-1);
}
my $optlist = "F:S:";
#
# Turn off line buffering on output
#
$| = 1;
# Need this for predicates.
use libsetup;
use libvnode;
use libvnode_xen;
#
# No configure vars.
#
my $sudo;
my $zipper = "/usr/local/bin/imagezip";
my $uploader = "/usr/local/etc/emulab/frisupload";
my $vnodeid;
my $filename;
my $error = 0;
for my $path (qw#/usr/local/bin /usr/bin#) {
if (-e "$path/sudo") {
$sudo = "$path/sudo";
last;
}
}
# Frisbee master server params
my $iserver = "boss"; # XXX
my $imageid;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (@ARGV != 2) {
usage();
}
if (defined($options{"S"})) {
$iserver = $options{"S"};
if ($iserver =~ /^([-\w\.]+)$/) {
$iserver = $1;
} else {
die("Bad -S hostname: '$iserver'");
}
}
if (defined($options{"F"})) {
$imageid = $options{"F"};
if ($imageid =~ /^(\S+)$/) {
$imageid = $1;
} else {
die("Bad -F imageid: '$imageid'");
}
}
$vnodeid = $ARGV[0];
if (defined($imageid)) {
$filename = "-";
} else {
$filename = $ARGV[1];
}
#
# Untaint the arguments.
#
# Note different taint check (allow /).
if ($filename =~ /^([-\w.\/\+]+)$/) {
$filename = $1;
}
else {
die("Tainted output filename: $filename");
}
sub domainStatus($)
{
my ($id) = @_;
my $status = `$sudo xm list --long $id 2>/dev/null`;
if ($status =~ /\(state ([\w-]+)\)/) {
return $1;
}
return "";
}
#
# Check contaner status. If it is running, we need to stop it,
# but first set it up to run "prepare" on the way down.
#
# XEN: Any status means it is running and needs to halted.
#
my $status = domainStatus($vnodeid);
#
# Boss has already arranged for prepare to run; we cannot exec a
# command inside the container from here, which is pretty dumb.
# The VM should halt on its own, so we wait for a bit and if it
# does not happen, we call it an error.
#
if ($status ne "") {
for (my $i = 20; $i >= 0; $i--) {
sleep(5);
$status = domainStatus($vnodeid);
if ($status eq "") {
$status = "stopped";
last;
}
print STDERR "Container is still running. Waiting ...\n";
}
if ($status ne "stopped") {
die("Container would not stop!\n");
}
}
#
# Use capture.pl; there are potentially multiple disks, the kernel,
# and a config file. All of this goes into a directory which we then
# tar up and upload.
#
system("$sudo $CAPTURE $vnodeid");
if ($?) {
print STDERR "Failed to capture the container!\n";
$error = 1;
goto cleanup;
}
if (! -e "$EXTRAFS/$vnodeid" || ! -e "$EXTRAFS/$vnodeid/kernel") {
print STDERR "$EXTRAFS/$vnodeid appears to be missing or incomplete\n";
$error = 1;
goto cleanup;
}
#
# If imageid is defined, we use the frisbee uploader.
#
my $cmd = "$TAR zcf - -C $EXTRAFS/$vnodeid . | $zipper -f - $filename";
if (defined($imageid)) {
$cmd .= " | $uploader -S $iserver -F $imageid";
if (SHAREDHOST()) {
$cmd .= " -P $vnodeid";
}
$cmd .= " -";
}
#
# Run the command using sudo, since by definition only testbed users
# with proper trust should be able to zip up a disk. sudo will fail
# if the user is not in the proper group.
#
if (system("$sudo $cmd")) {
print STDERR "*** Failed to create image!\n";
print STDERR " command: '$sudo $cmd'\n";
$error = 1;
}
cleanup:
# Clean up the directory.
system("$sudo /bin/rm -rf $EXTRAFS/$vnodeid");
#
# Reboot the vnode.
#
system("$sudo $VNODESETUP -jbVt $vnodeid");
if ($?) {
die("Could not restart container!\n");
}
exit($error);
This diff is collapsed.
......@@ -27,18 +27,21 @@ use English;
use Errno;
use Data::Dumper;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
sub usage()
{
print "Usage: restorevm.pl [-d] vnodeid path\n" .
print "Usage: restorevm.pl [-d] [-t targetdir] vnodeid path\n" .
" -d Debug mode.\n".
" -t Write new xm.conf and copy kernel to targetdir\n".
" -i Info mode only\n";
exit(-1);
}
my $optlist = "dix";
my $optlist = "dixt:";
my $debug = 1;
my $infomode = 0;
my $VMPATH = "/var/xen/configs";
my $VGNAME = "xen-vg";
my $targetdir;
my $IMAGEUNZIP = "imageunzip";
my $IMAGEDUMP = "imagedump";
......@@ -47,6 +50,12 @@ my $IMAGEDUMP = "imagedump";
#
$| = 1;
use libvnode_xen;
use libvnode;
# From the library
my $VGNAME = $libvnode_xen::VGNAME;
# Locals
my %xminfo = ();
......@@ -64,6 +73,9 @@ if (! getopts($optlist, \%options)) {
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"t"})) {
$targetdir = $options{"t"};
}
if (defined($options{"i"})) {
$infomode = 1;
}
......@@ -113,9 +125,18 @@ while (<XM>) {
close(XM);
#
# Localize the path to the kernel.
# Localize the path to the kernel. Copy out if there is a target dir.
#
$xminfo{"kernel"} = $path . "/" . $xminfo{"kernel"};
if (defined($targetdir)) {
if (!$infomode) {
system("/bin/cp -pf $path/" . $xminfo{"kernel"} .
" $targetdir/" . $xminfo{"kernel"});
}
$xminfo{"kernel"} = $targetdir . "/" . $xminfo{"kernel"};
}
else {
$xminfo{"kernel"} = $path . "/" . $xminfo{"kernel"};
}
#
# Fix up the network interfaces.
......@@ -156,6 +177,8 @@ foreach my $disk (@$disklist) {
Fatal("Cannot parse disk: $disk");
}
}
my %newdiskinfo = ();
my $newlvms = {};
#
# And the size info.
......@@ -186,8 +209,9 @@ foreach my $physinfo (keys(%diskinfo)) {
#
# Form a new lvmname and create the LVM using the size.
# Swap has to be treated special for now.
#
my $lvmname = "${vnodeid}.${dev}";
my $lvmname = ($spec =~ /swap/ ? "${vnodeid}.swap" : "${vnodeid}.${dev}");
my $device = "/dev/$VGNAME/$lvmname";
if (! -e $device) {
......@@ -198,7 +222,9 @@ foreach my $physinfo (keys(%diskinfo)) {
}
# Rewrite the diskinfo path for new xm.conf
delete($diskinfo{$physinfo});
$diskinfo{$device} = "phy:$device,$dev,w";
$newdiskinfo{$device} = "phy:$device,$dev,w";
# For cleanup on error.
$newlvms->{$lvmname} = $lvmname;
#
# For swap, just need to mark it as a linux swap partition.
......@@ -230,6 +256,9 @@ foreach my $physinfo (keys(%diskinfo)) {
}
else {
system("$IMAGEUNZIP -o $opts $filename $device");
if ($?) {
Fatal("Failed to unzip $filename to $device!");
}
}
}
......@@ -239,7 +268,7 @@ foreach my $physinfo (keys(%diskinfo)) {
delete($xminfo{"disksizes"});
$xminfo{"name"} = $vnodeid;
$xminfo{"memory"} = "2048";
$xminfo{"disk"} = "[" . join(",", map {"'$_'" } values(%diskinfo)) . "]";
$xminfo{"disk"} = "[" . join(",", map {"'$_'" } values(%newdiskinfo)) . "]";
if ($infomode) {
print Dumper(\%xminfo);
......@@ -249,11 +278,17 @@ else {
# Before we write it out, need to munge the vif spec since there is
# no need for the script. Use just the default.
#
$XMINFO = "/var/tmp/${vnodeid}.conf";
print "Writing new xen config to $XMINFO\n";
my $xmconf;
if (defined($targetdir)) {
$xmconf = "$targetdir/xm.conf";
}
else {
$xmconf = "/var/tmp/${vnodeid}.conf";
}
print "Writing new xen config to $xmconf\n";
open(XM, ">$XMINFO")
or fatal("Could not open $XMINFO: $!");
open(XM, ">$xmconf")
or fatal("Could not open $xmconf: $!");
foreach my $key (keys(%xminfo)) {
my $val = $xminfo{$key};
if ($val =~ /^\[/) {
......@@ -272,6 +307,14 @@ sub Fatal($)
{
my ($msg) = @_;
#
# Destroy any new lvms we created.
#
if (defined($newlvms)) {
foreach my $lvname (keys(%{ $newlvms })) {
system("lvremove -f $VGNAME/$lvname");
}
}
die("*** $0:\n".
" $msg\n");
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment