Commit df90d7a7 authored by Leigh Stoller's avatar Leigh Stoller

Reservations system changes:

1) Rework so that instead of relying on swapin__last + autoswap timeout,
   set expt_expires for classic experiments at the beginning of swapin
   time. This is cause swapin_last is not set till the end of swapin,
   and so during swapin the res system is in an inconsistent state since
   there is no way to determine when the experiment ends.

2) On the Geni path, simplify expiration handling; do not allow a slice
   modification and expiration change at the same time; the bookkeeping
   and failure rollback is a pain, especially wrt reservation system,
   and this rarely ever actually happens, so get rid of a lot of
   complication.
parent 06f46fd0
......@@ -2223,6 +2223,29 @@ sub PreSwap($$$$)
#
}
#
# Before we allocate any resources we have to make sure the reservation
# system and the autoswapper are on the same page wrt when these
# resources are going to be released. We can no longer wait till the
# end of swapin (when swapin_last is updated), we need to base it from
# the current time, or else the res system can get into an overbook
# situation. As a result, users will see somewhat shorter autoswap then
# they are used to, depending on how long the experiment takes to
# swapin. So add a little padding, not worth worrying too much since
# this is Emulab Classic. Expiration is handled on the geni path so
# skip.
#
if (!$self->geniflags() &&
($which eq $EXPT_SWAPIN || $which eq $EXPT_START)) {
if ($self->autoswap() && $self->autoswap_timeout()) {
$self->SetExpiration(time() +
(($self->autoswap_timeout() + 30) * 60));
}
else {
$self->SetExpiration(undef);
}
}
# Old swap gathering stuff.
$self->GatherSwapStats($swapper, $which, 0,
TBDB_STATS_FLAGS_START()) == 0
......@@ -2297,7 +2320,11 @@ sub SwapFail($$$$;$)
" last_error=$session ".
"where exptidx=$exptidx");
}
# Clear this on failed swapin.
if ($which eq $EXPT_START || $which eq $EXPT_SWAPIN) {
$self->SetExpiration(undef);
}
#
# Get current and last rsrc record direct from DB to avoid local cache.
#
......@@ -2578,7 +2605,8 @@ sub PostSwap($$$$)
}
}
# Lets clear this since a swaped experiment has no expiration.
if ($which eq $EXPT_SWAPOUT) {
# Expiration is handled on the geni path so skip.
if (!$self->geniflags() && $which eq $EXPT_SWAPOUT) {
$self->SetExpiration(undef);
}
......
......@@ -555,7 +555,7 @@ sub LookupAll($$;$$)
return $cache{$type} if( exists( $cache{$type} ) );
Tidy( $class );
# Tidy( $class );
# Mysql 5.7 group by nonsense. Revisit later, like when hell freezes.
DBQueryWarn("SET SESSION sql_mode=(SELECT REPLACE(\@\@sql_mode,".
......@@ -565,8 +565,8 @@ sub LookupAll($$;$$)
my $query = $PGENISUPPORT ? "SELECT COUNT(*), e.pid, e.eid, " .
"e.expt_swap_uid, " .
"UNIX_TIMESTAMP( stats.swapin_last ) + " .
"e.autoswap_timeout * 60, e.autoswap, " .
"UNIX_TIMESTAMP( e.expt_expires ), ".
"e.autoswap, " .
"nr.pid, UNIX_TIMESTAMP( s.expires ), " .
"s.lockdown as slice_lockdown, ".
"n.reserved_pid, " .
......@@ -592,8 +592,8 @@ sub LookupAll($$;$$)
"UNIX_TIMESTAMP( pr.end )" :
"SELECT COUNT(*), e.pid, e.eid, " .
"e.expt_swap_uid, " .
"UNIX_TIMESTAMP( stats.swapin_last ) + " .
"e.autoswap_timeout * 60, e.autoswap, " .
"UNIX_TIMESTAMP( e.expt_expires ), " .
"e.autoswap, " .
"nr.pid, NULL, " .
"NULL, n.reserved_pid, " .
"UNIX_TIMESTAMP( pr.end ), " .
......@@ -1236,26 +1236,63 @@ sub ExtendSlice($$$;$$$) {
}
#
# This is identical to above, but for experiment autoswap.
# This is similar to above, but for experiment autoswap.
#
sub AutoSwapTimeout($$$;$$$) {
my ($class, $expt, $minutes, $error, $impotent, $force) = @_;
if( $minutes <= $expt->autoswap_timeout()) {
if( $impotent ) {
#
# So if the timeout is smaller then current, it is okay to just
# change it: If autoswap is currently on, then this is safe. If
# autoswap is currently off (and expt not locked down), then we
# are going from no expiration set, so clearly that is safe too.
#
if ($minutes <= $expt->autoswap_timeout() || !$expt->autoswap()) {
if ($impotent) {
return 0;
} else {
my $result = $expt->SetAutoswapTimeout($minutes);
}
else {
my $expires;
if( $result < 0 && ref( $error ) ) {
$$error = "Couldn't update experiment autoswap_timeout";
if ($expt->state() eq EXPTSTATE_ACTIVATING()) {
$expires = $expt->tstamp();
}
return $result;
else {
$expires = str2time($expt->swapin_last());
}
$expires += ($minutes * 60);
if ($expt->SetExpiration($expires) ||
$expt->SetAutoswapTimeout($minutes) ||
$expt->SetAutoswap(1)) {
if (ref($error)) {
$$error = "Could not update experiment autoswap_timeout";
}
return -1;
}
return 0;
}
}
my @types = ExptTypes( $expt->idx() );
#
# We need a new expiration time, based on when the experiment actually
# started. swapin_last does not work since it is set at the end of
# swapin, which can take an arbitrary amount of time. We know that
# autoswap() was already on (see above) so we can use the current
# expiration time to compute a new expiration.
#
my $expires = $expt->expt_expires();
if (!defined($expires)) {
if (ref($error)) {
$$error = "experiment does not have expiration set";
}
return -1;
}
$expires = str2time($expires);
# Add difference in autoswap_timeout to expiration;
$expires += (($minutes - $expt->autoswap_timeout()) * 60);
while( 1 ) {
my $version = GetVersion( $class );
......@@ -1265,7 +1302,7 @@ sub AutoSwapTimeout($$$;$$$) {
if( defined( $res->pid() ) && defined( $res->eid() ) &&
$res->pid() eq $expt->pid() &&
$res->eid() eq $expt->eid() ) {
$res->{'END'} = time() + ($minutes * 60);
$res->{'END'} = $expires;
last;
}
}
......@@ -1277,15 +1314,19 @@ sub AutoSwapTimeout($$$;$$$) {
if( $impotent );
next if( !defined( BeginTransaction( $class, $version, "experiments")));
my $result = $expt->SetAutoswapTimeout($minutes);
if( $result < 0 && ref( $error ) ) {
$$error = "Couldn't update experiment autoswap_timeout";
if ($expt->SetExpiration($expires) ||
# This must be after above line.
$expt->SetAutoswapTimeout($minutes) ||
$expt->SetAutoswap(1)) {
if (ref($error)) {
$$error = "Couldn't update experiment autoswap_timeout";
}
EndTransaction($class);
return -1;
}
EndTransaction( $class );
return $result;
return 0;
}
}
......@@ -1435,12 +1476,13 @@ sub DisableAutoSwap($$;$$$)
my ($error) = @_;
my $result;
$result = $experiment->SetAutoswap(0);
if( $result < 0 && ref( $error ) ) {
$$error = "Couldn't update autoswap";
if ($experiment->SetAutoswap(0) || $experiment->SetExpiration(undef)) {
if (ref($error)) {
$$error = "Couldn't update autoswap";
}
return -1;
}
return $result;
return 0;
};
return LockdownAux($class, $experiment,
$coderef, $error, $impotent, $force);
......@@ -1483,6 +1525,14 @@ sub LockdownAux($$$$$$)
}
}
#
# Strictly for admission control in the mapper. For geni experiments,
# LookupAll uses the current slice expiration to determine when the
# experiment ends. But for mapper admission control we use experiment
# expires, since on the geni path you can both modify an expriment and
# change its expiration at the same time. We do not want to change the
# slice expiration until we confirm that the mapper allows it.
#
sub ExpectedEnd($$) {
my ($class, $experiment) = @_;
......@@ -1494,18 +1544,28 @@ sub ExpectedEnd($$) {
if (($experiment->lockdown() || !$experiment->swappable()) &&
!$experiment->geniflags()) {
return undef;
} elsif( $experiment->autoswap() ) {
if ($experiment->state() eq EXPTSTATE_SWAPPED() ||
$experiment->state() eq EXPTSTATE_ACTIVATING()) {
return time() + ($experiment->autoswap_timeout * 60);
}
elsif ($experiment->geniflags()) {
if (!defined($experiment->expt_expires())) {
print STDERR "ExpectedEnd: No expiration for $experiment\n";
return undef;
}
return str2time($experiment->swapin_last()) +
($experiment->autoswap_timeout * 60);
} elsif( defined( $experiment->expt_expires() ) ) {
return str2time( $experiment->expt_expires() );
} else {
return undef;
return str2time($experiment->expt_expires());
}
elsif (defined($experiment->expt_expires())) {
return str2time($experiment->expt_expires());
}
#
# This should not happen, execpt that we sometimes run the mapper in
# debugging mode. So if we are here and the experiment is swapped, use
# the current time.
#
print STDERR "ExpectedEnd: No expiration for $experiment\n";
if ($experiment->state() eq EXPTSTATE_SWAPPED() &&
$experiment->autoswap()) {
return time() + ($experiment->autoswap_timeout() * 60);
}
return undef;
}
#
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -311,15 +311,16 @@ EOT
}
$sql = <<EOT;
select e.pid,e.eid,e.autoswap_timeout,
select e.pid,e.eid,e.autoswap_timeout,e.expt_expires,
(unix_timestamp(now()) - unix_timestamp(s.swapin_last))/60 as activemin
from experiments as e
left join experiment_stats as s on s.exptidx=e.idx
where e.swappable>0 and e.state="active" and e.expt_locked is null
and e.autoswap>0
and now()>e.expt_expires
and e.geniflags=0
and e.paniced=0
having activemin>=e.autoswap_timeout order by e.pid,e.eid
order by e.pid,e.eid
EOT
$q = DBQueryFatal($sql);
while (%r = $q->fetchhash()) {
......@@ -327,8 +328,10 @@ EOT
$eid = $r{'eid'};
my $autotimo = $r{'autoswap_timeout'};
my $activemin = $r{'activemin'};
my $expires = $r{'expt_expires'};
Notify("$pid/$pid: active:$activemin, autotimo:$autotimo\n");
Notify("$pid/$eid: active:$activemin, ".
"autotimo:$autotimo, expires:$expires\n");
if ($i) {
Debug("Would autoswap $pid,$eid\n");
......@@ -383,13 +386,14 @@ EOT
if ($auto_warnmin>0) {
$sql = <<EOT;
select e.pid,e.eid,e.autoswap_timeout,
select e.pid,e.eid,e.autoswap_timeout,e.expt_expires,
(unix_timestamp(now()) - unix_timestamp(s.swapin_last))/60 as activemin
from experiments as e
left join experiment_stats as s on s.exptidx=e.idx
where e.swappable>0 and e.state="active" and e.autoswap>0
having activemin+$auto_warnmin>=e.autoswap_timeout and
activemin+$auto_warnmin<=e.autoswap_timeout+$window order by e.pid,e.eid
where e.swappable>0 and e.state="active" and e.autoswap>0 and
now()>=date_sub(e.expt_expires, interval $auto_warnmin minute) and
now()<=date_sub(e.expt_expires, interval ${auto_warnmin}-${window} minute)>now()
order by e.pid,e.eid
EOT
$q = DBQueryFatal($sql);
while (%r = $q->fetchhash()) {
......
......@@ -702,8 +702,9 @@ sub GetTicketAuxAux($)
}
if (!$allow_externalusers && !$isExempted && !$user->IsLocal()) {
return GeniResponse->Create(GENIRESPONSE_UNAVAILABLE, undef,
"External users temporarily denied");
$response = GeniResponse->Create(GENIRESPONSE_UNAVAILABLE, undef,
"External users temporarily denied");
goto bad;
}
#
......@@ -711,19 +712,28 @@ sub GetTicketAuxAux($)
# ticket is redeemed, it will expire according to the rspec request.
# If nothing specified in the rspec, then it will expire when the
# slice record expires, which was given by the expiration time of the
# slice credential, or the local policy max_sliver_lifetime. See
# CreateSliceFromCertificate() in this file.
# slice credential, or the local policy max_sliver_lifetime.
#
my $expires = GeniXML::GetExpires($rspec);
if (defined($expires)) {
if (GeniResponse::IsResponse($expires)) {
return $expires;
}
# Note "checkonly" flag; we do not actually change the slice
# until the ticket is redeemed.
my $tmp = SetSliceExpiration($slice, $expires, 1, 0, 0, @{ $credentials });
if (GeniResponse::IsResponse($tmp)) {
return $tmp;
if (defined($expires) && GeniResponse::IsResponse($expires)) {
$response = $expires;
goto bad;
}
#
# We no longer allow changing the expiration during an update call.
# Too much hassle cause of the reservation system and the fact that
# an abnormal exit can leave the slice with the new expiration. The
# problem with the reservation system is that we have to change the
# expiration so that admission control will check all of the resources
# based on the new expiration. Before the reservation system, we did
# not need to change the slice expiration until later in redeem ticket,
# thus ensuring that if the ticket was not redeemed, the resources
# would be released.
#
if ($isupdate) {
if (SliceExpirationChanged($slice, $expires, @{$credentials})) {
return GeniResponse->Create(GENIRESPONSE_REFUSED, undef,
"Not allowed to change expiration when updating");
}
}
......@@ -763,6 +773,10 @@ sub GetTicketAuxAux($)
"Slice has expired");
}
#
# Gotto goto bad below this point!
#
#
# For now, there can be only a single toplevel aggregate per slice.
# The existence of an aggregate means the slice is active here.
......@@ -790,29 +804,25 @@ sub GetTicketAuxAux($)
$slice_experiment = undef;
goto bad;
}
my $pid = $slice_experiment->pid();
my $eid = $slice_experiment->eid();
#
# Mark the expires slot in the experiment, for admission control
# during the mapper run.
# Set the slice expiration. First off, this does a policy check on the
# desired expiration. Since this is a new slice, there are no resources
# allocated and so the reservation system check will succeed. The
# experiment expiration will also be set, for admission control.
#
# Convert to a localtime.
# See above, we no longer allow expiration to change during update,
# so skip when doing an update.
#
if (defined($expires)) {
$expires = eval { timegm(strptime($expires)); };
if ($@ || !defined($expires)) {
$response = GeniResponse->Create(GENIRESPONSE_BADARGS, undef,
"Could not parse expiration");
goto bad;
}
}
else {
$expires = $slice->expires();
}
if ($slice_experiment->SetExpiration($expires)) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
"set experiment expiration");
print STDERR "Could not set experiment expiration\n";
goto bad;
if (!$isupdate) {
$response =
SetSliceExpiration($slice, $expires, 0, 0, 0, @{$credentials});
goto bad
if (GeniResponse::IsResponse($response));
}
my $realuser = FlipToUser($slice, $user);
if (! (defined($realuser) && $realuser)) {
$response = GeniResponse->Create(GENIRESPONSE_ERROR, undef,
......@@ -820,8 +830,6 @@ sub GetTicketAuxAux($)
print STDERR "Error flipping to real user\n";
goto bad;
}
my $pid = $slice_experiment->pid();
my $eid = $slice_experiment->eid();
#
# Mark the experiment locally as coming from the cooked interface.
......@@ -3864,7 +3872,7 @@ sub GetTicketAuxAux($)
$slice->SetAsyncError($response);
}
$slice->UnLock()
if (defined($slice) && !$nolock);
if (defined($slice) && !$nolock && $slice->LOCKED());
$ticket->UnLock()
if (defined($ticket) && $ticket->stored());
return $response;
......@@ -4177,23 +4185,6 @@ sub SliverWorkAux($)
}
}
#
# Figure out new expiration time; this is the time at which we can
# idleswap the slice out.
#
my $expires = GeniXML::GetExpires($rspec);
if (defined($expires)) {
if (GeniResponse::IsResponse($expires)) {
$message = "Illegal valid_until in rspec";
goto bad;
}
my $tmp = SetSliceExpiration($slice, $expires, 0, 0, 0, $credential);
if (GeniResponse::IsResponse($tmp)) {
$message = GeniResponse::output($tmp);
goto bad;
}
}
my %phys2nickname = ();
#
......@@ -6624,39 +6615,8 @@ sub CreateSliceFromCertificate($$)
"Cannot create slice object")
if (!defined($slice));
#
# The slice expires when the credential expires, or when the local policy
# limit says, which ever is shorter.
#
my $max_sliver_lifetime = 0;
if (!GetSiteVar('protogeni/max_sliver_lifetime', \$max_sliver_lifetime)) {
# Cannot get the value, default it to 90 days.
$max_sliver_lifetime = 90;
}
my $initial_sliver_lifetime = 0;
if (!GetSiteVar('protogeni/initial_sliver_lifetime',
\$initial_sliver_lifetime)) {
# Cannot get the value, default it to 6 hours.
$initial_sliver_lifetime = 6;
}
elsif ($initial_sliver_lifetime == 0) {
$initial_sliver_lifetime = $max_sliver_lifetime * 24;
}
my $expires = $credential->expires();
# This is already a localtime.
my $when = eval { timelocal(strptime($expires)); };
if ($@) {
$expires = time() + (3600 * $max_sliver_lifetime);
}
else {
my $diff = $when - time();
if ($diff > (3600 * 24 * $max_sliver_lifetime)) {
# Shorten to policy maximum. Okay to use a unix time.
$expires = time() + (3600 * $max_sliver_lifetime);
}
}
$slice->SetExpiration($expires);
# Initial expiration of an hour, we will set this for real very soon.
$slice->SetExpiration(time() + 3600);
$slice->SetPublicID();
return $slice;
}
......@@ -7388,6 +7348,8 @@ sub GeniExperiment($;$)
(defined($creator) ? $creator->urn() : undef),
"protogeni");
$slice->SetExperiment($experiment);
# Initialize, this will get reset in GetTicket().
$experiment->SetExpiration($slice->expires());
#
# Check for feature to back off and let something else manage the
......@@ -8018,7 +7980,7 @@ sub SetSliceExpiration($$$$$@)
}
}
else {
$when = $slice_expires;
$when = $slice_when;
}
# The slice is already set correctly, skip the rest of this. This
# avoids an issue with ticket update, whereby the extra permission
......@@ -8075,6 +8037,16 @@ sub SetSliceExpiration($$$$$@)
}
return 0
if ($checkonly && !$slice_experiment);
#
# No slice experiment, just set it since this is a new slice and no
# resources are allocated, clearly will not cause a violation.
# The mapper will determine that later when it does admission control.
#
if (!$slice_experiment) {
$slice->SetExpiration($when);
return 0;
}
#
# Now that we have the number we want, make sure the reservation system
......@@ -8114,7 +8086,7 @@ sub SetSliceExpiration($$$$$@)
}
#
# Mark the expires slot in the experiment, for admission control
# during the mapper run.
# during the mapper run.
#
if (defined($slice_experiment) &&
$slice_experiment->SetExpiration($when)) {
......@@ -8128,6 +8100,41 @@ sub SetSliceExpiration($$$$$@)
return GeniResponse->Create(GENIRESPONSE_REFUSED, undef, $message);
}
#
# Helper function to see if caller is requesting a change to expiration.
#
sub SliceExpirationChanged($$@)
{
my ($slice, $expires, @credentials) = @_;
my $current_slice_expiration = str2time($slice->expires());
my $message;
if (!defined($expires)) {
#
# Ick, assume first credential is the slice credential. Bad.
#
my $slice_credential = shift(@credentials);
#
# Maximum expiration is what the credential says.
#
$expires = $slice_credential->expires();
if (!defined($expires)) {
print STDERR "SliceExpirationChanged: ".
"No expiration time in credential\n";
return 1;
}
}
# Convert slice expiration to a time.
my $slice_when = str2time($expires);
if (!defined($slice_when)) {
print STDERR "SliceExpirationChanged: ".
"Could not parse expiration in credential\n";
return 1;
}
return $current_slice_expiration != $slice_when ? 1 : 0;
}
#
# Handle BlockStores.
#
......
use strict;
use Date::Parse;
use POSIX qw(strftime);
use emdb;
use EmulabConstants;
use Experiment;
sub TimeStamp($)
{
my ($seconds) = @_;
return POSIX::strftime("%m/%d/20%y %H:%M:%S", localtime($seconds));
}
sub DoUpdate($$$)
{
my ($dbhandle, $dbname, $version) = @_;
my $swapped = EXPTSTATE_SWAPPED();
my $query_result =
DBQueryFatal("select idx from experiments where ".
" (state='$swapped' and geniflags=0 and ".
" expt_expires is not null) or ".
" (state!='$swapped' and autoswap and ".
" expt_expires is null)");
while (my ($idx) = $query_result->fetchrow_array()) {
my $experiment = Experiment->Lookup($idx);
next
if (!defined($experiment));
if ($experiment->state() eq EXPTSTATE_SWAPPED()
&& !$experiment->geniflags() &&
defined($experiment->expt_expires())) {
$experiment->SetExpiration(undef) == 0
or return -1;
next;
}
if ($experiment->state() ne EXPTSTATE_SWAPPED() &&
$experiment->state() ne EXPTSTATE_NEW() &&
$experiment->autoswap() &&
!defined($experiment->expt_expires())) {
my $state = $experiment->state();
#
# Look to see if it has any nodes;
#
if ($experiment->NodeList(1)) {
my $expires;
print "No expiration for $experiment ($state).\n";
#
# ACTIVATING is the one annoying state, we do not have
# a valid swapin time, since it is not set until the
# swapin is done.
#
if ($experiment->state() eq EXPTSTATE_ACTIVATING()) {
$expires = $experiment->tstamp();
}
else {
# Easy.
$expires = $experiment->swapin_last();
if (!defined($expires)) {
fatal("No swapin_last for $experiment");
}
$expires = str2time($expires);
}
$expires += $experiment->autoswap_timeout() * 60;
print "-> Setting expiration to ". TimeStamp($expires) . "\n";
$experiment->SetExpiration($expires) == 0
or return -1;
}
}
next;
}
return 0;
}
# Local Variables:
# mode:perl
# End:
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -38,9 +38,10 @@ sub usage()
" manage_expsettings autoswap experiment clear\n");
exit(-1);
}
my $optlist = "ds";
my $optlist = "dsn";
my $debug = 0;
my $silent = 0;
my $impotent = 0;
my $this_user;
#
......@@ -95,6 +96,9 @@ if (! getopts($optlist, \%options)) {
if (defined($options{"d"})) {
$debug++;
}
if (defined($options{"n"})) {
$impotent = 1;
}
if (defined($options{"s"})) {
$silent = 1;
}
......@@ -187,7 +191,8 @@ sub DoLockdown()
}
}
else {
if (Reservation->Lockdown($experiment, \$errmsg, 0, $force)) {
if (Reservation->Lockdown($experiment,
\$errmsg, $impotent, $force)) {
$errcode = 1;
$errmsg = "Could not lockdown experiment"
if (!defined($errmsg));
......@@ -256,28 +261,38 @@ sub DoAutoswap()
goto bad;
}
if ($which eq "set") {
my $current = $experiment->autoswap();
my $current_timeout = $experiment->autoswap_timeout();
if ($experiment->Update({"autoswap" => 1})) {
$errmsg = "Could not update database";
$errcode = -1;
goto bad;
#
# If the experiment is not running, then this operation is
# trivial; just update the database since there is no effect
# on the reservation system.
#
if ($experiment->state() eq EXPTSTATE_SWAPPED()) {
if ($experiment->Update({"autoswap" => 1,
"autoswap_timeout" => $timeout})) {
$errmsg = "Could not update database";
$errcode = -1;
goto bad;
}
}
#
# Otherwise we ask the reservation system to do it.
#
# This is a bit problematic; this experiment might have already
# have out the system into overbook, but we cannot really tell
# have put the system into overbook, but we cannot really tell
# that (that it is this experiment). We only know when making a
# change puts the system into overbook. So go ahead and do it and
# if the change is rejected, we can report that.
#
if (Reservation->AutoSwapTimeout($experiment, $timeout,
\$errmsg, 0, $force)) {
\$errmsg, $impotent, $force)) {
$errcode = 1;
$errmsg = "Could not change autoswap timeout for experiment"
if (!defined($errmsg));