Merge remote-tracking branch 'upstream/master' into docker-alpine

Syncing the fork with the upstream origin before submitting a merge
request back to the core project.

account/addpubkey.in

 #!/usr/bin/perl -wT
 #
-# Copyright (c) 2000-2016 University of Utah and the Flux Group.
+# Copyright (c) 2000-2017 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -499,7 +499,7 @@ sub ParseKey($) {
 		 "SSH Public Key for '$user_uid' added:\n".
 		 "\n".
 		 "$chunked\n",
-		 "$TBOPS");
+		 "$TBOPS", "Bcc: $TBAUDIT");
     }
     return 1;
 }

account/tbacct.in

@@ -258,7 +258,12 @@ if (! defined($target_user)) {
 my $this_user;
 
 if (getpwuid($UID) eq "nobody") {
-    $this_user = $target_user;
+    # The web interface sets this.
+    $this_user = User->ImpliedUser();
+    # This can happen using the forget password link; user not logged in.
+    if (!defined($this_user)) {
+	$this_user = $target_user;
+    }
 }
 else {
     $this_user = User->ThisUser();

apt/APT_Instance.pm.in

@@ -2113,9 +2113,9 @@ sub Terminate($)
 #
 # Ask to extend. 
 #
-sub Extend($$$)
+sub Extend($$$$)
 {
-    my ($self, $new_expires, $this_user) = @_;
+    my ($self, $new_expires, $this_user, $force) = @_;
     my $credentials;
     my $method;
     my @params;
@@ -2209,10 +2209,14 @@ sub Extend($$$)
 	unlink($credname);
 	chomp($extcred);
 
-	@params = ({"slice_urn"   => $slice->urn(),
+	my $args = {"slice_urn"   => $slice->urn(),
 		    "expiration"  => $new_expires,
 		    "credentials" => [@$credentials, $extcred],
-		   });
+	};
+	if ($force) {
+	    $args->{"force"} = 1;
+	}
+	@params = ($args);
 	$method = "RenewSlice";
     }
     my $cmurl = $authority->url();

apt/APT_Profile.pm.in

@@ -654,6 +654,9 @@ sub UpdateMetaData($$)
 	    $mods{$key} = $argref->{$key};
 	}
     }
+    return 0
+	if (!keys(%mods));
+    
     my $query = "update apt_profiles set ".
 	join(",", map("$_=" . DBQuoteSpecial($mods{$_}), keys(%mods)));
 

apt/create_instance.in

@@ -711,6 +711,9 @@ my $slice_id    = (defined($userslice_id) ? $userslice_id :
 my $slice_auth  = ($pid eq $gid ? $pid : "${pid}:${gid}");
 my $slice_urn   = GeniHRN::Generate("${OURDOMAIN}:${slice_auth}",
 				    "slice", $slice_id);
+if (!defined($slice_urn)) {
+    fatal("Could not generate a valid slice urn!");
+}
 my $slice_hrn   = "${PGENIDOMAIN}.${pid}.${slice_id}";
 my $SERVER_NAME = (exists($ENV{"SERVER_NAME"}) ? $ENV{"SERVER_NAME"} : "");
 

apt/manage_instance.in

@@ -1364,7 +1364,7 @@ sub DoTerminate()
 #
 sub DoExtend()
 {
-    my $force = 0;
+    my $force    = 0;
     my $lockdown = 0;
     my $errcode  = 1;
     my $autoextend_maximum = GetSiteVar("aptui/autoextend_maximum");
@@ -1391,7 +1391,7 @@ sub DoExtend()
     usage()
 	if (!@ARGV);
 
-    my $optlist = "m:f:h";
+    my $optlist = "m:f:hF";
     my %options = ();
     if (! getopts($optlist, \%options)) {
 	usage();
@@ -1399,7 +1399,12 @@ sub DoExtend()
     usage()
 	if (!@ARGV);
     my $wanted = shift(@ARGV);
-
+    
+    if (defined($options{"F"})) {
+	# Be careful
+	$force = 1
+	    if (defined($this_user) && $this_user->IsAdmin());
+    }
     if (defined($options{"m"})) {
 	$reason = $options{"m"};
     }
@@ -1442,12 +1447,29 @@ sub DoExtend()
 	$errcode = 1;
 	goto bad;
     }
-    if (!TBcheck_dbslot($wanted, "default", "int",
-			 TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
-	$errmsg  = "Illegal integer for length";
-	$errcode = 1;
-	goto bad;
+    # Lets allow a date too.
+    if ($wanted !~ /^\d+$/) {
+	my $when = str2time($wanted);
+	if (!$when) {
+	    $errmsg  = "Illegal number of days or date";
+	    $errcode = 1;
+	    goto bad;
+	}
+	if ($when < time() + 3600) {
+	    $errmsg  = "Expiration is too soon";
+	    $errcode = 1;
+	    goto bad;
+	}
+	my $hours = int(($when - time()) / 3600);
+	if ($hours < 24) {
+	    $wanted  = $hours;
+	    $inhours = 1;
+	}
+	else {
+	    $wanted = int($hours / 24);
+	}
     }
+    
     # Helper function.
     my $needAdminApproval = sub {
 	my ($wanted, $granted, $reason, $message) = @_;
@@ -1639,7 +1661,7 @@ sub DoExtend()
 	my $seconds = $granted * 3600;
 	$seconds *= 24 if (!$inhours);
 	    
-	if ($errcode = ExtendInternal($slice, $seconds, 0, 0, \$errmsg)) {
+	if ($errcode = ExtendInternal($slice, $seconds, $force, 0, \$errmsg)) {
 	    goto bad;
 	}
     }
@@ -1797,31 +1819,13 @@ sub ExtendInternal($$$$$)
     }
     my $new_expires = $slice->ExpirationGMT();
 
-    #
-    # Do a reservation check first. We do not care if it fails (yet),
-    # just keep going. We want to report results back to the web interface.
-    #
-    if (defined($webtask)) {
-	$webtask->CheckReservationResult(0);
-	$webtask->CheckReservationMessage("");
-    }
-    if (CheckReservationInternal($slice, $new_expires, \$errmsg)) {
-	if ($debug) {
-	    print "Reservation denied! $errmsg\n";
-	}
-	if (defined($webtask)) {
-	    $webtask->CheckReservationMessage($errmsg);
-	    $webtask->CheckReservationResult(1);
-	}
-    }
-
     my $coderef = sub {
 	my ($sliver) = @_;
 	my $webtask  = $sliver->webtask();
 	my $domain   = $sliver->domain();
 	my $errmsg;
 
-	my $response = $sliver->Extend($new_expires, $this_user);
+	my $response = $sliver->Extend($new_expires, $this_user, $force);
 	if (!defined($response)) {
 	    $errmsg = "Internal error calling Renew at $domain";
 	    goto bad;
@@ -1942,7 +1946,6 @@ sub DoCheckReservation()
 sub CheckReservationInternal($$$)
 {
     my ($slice, $expiration, $perrmsg) = @_;
-    my $rescheck = "$TB/etc/rescheck.txt";
     my $errcode = -1;
     my $errmsg;
 
@@ -2003,16 +2006,8 @@ sub CheckReservationInternal($$$)
 	    $agg->webtask()->Refresh();
 	    $errmsg  = $agg->webtask()->output();
 	    $errcode = $agg->webtask()->exitcode();
-	    if ($errcode == GENIRESPONSE_REFUSED) {
-		system("/bin/echo 'DENIED: $slice, $expiration, $agg, $errmsg'".
-		       "  >> $rescheck");
-	    }
 	    goto bad;
 	}
-	else {
-	    system("/bin/echo 'ACCEPT: $slice, $expiration, $agg' ".
-		   " >> $rescheck");
-	}
     }
     return 0;
 
@@ -2050,8 +2045,7 @@ sub DoMaxExtension()
 	    goto bad;
 	}
 	if ($response->code() != GENIRESPONSE_SUCCESS) {
-	    $errmsg = "Failed to get max extension at $domain: ".
-		$response->output();
+	    $errmsg = $response->output();
 	    
 	    # This is something the user should see.
 	    if ($response->code() == GENIRESPONSE_REFUSED ||
@@ -2062,6 +2056,10 @@ sub DoMaxExtension()
 		$webtask->Exited($response->code());
 		return 1;
 	    }
+	    # Forbidden means cluster does not allow it, ignore.
+	    if ($response->code() == GENIRESPONSE_FORBIDDEN) {
+		return 0;
+	    }
 	    goto bad;
 	}
 	$webtask->MaxExtension($response->value());
@@ -2096,6 +2094,9 @@ sub DoMaxExtension()
 	    $errcode = $agg->webtask()->exitcode();
 	    goto bad;
 	}
+	next
+	    if (!defined($agg->webtask()->MaxExtension()));
+	
 	my $max = str2time($agg->webtask()->MaxExtension());
 	$newmax = $max
 	    if (!defined($newmax));
@@ -2292,7 +2293,7 @@ sub DoExtendOld()
 	my $domain   = $sliver->domain();
 	my $errmsg;
 
-	my $response = $sliver->Extend($new_expires, $this_user);
+	my $response = $sliver->Extend($new_expires, $this_user, $force);
 	if (!defined($response)) {
 	    $errmsg = "Internal error calling Renew at $domain";
 	    goto bad;
@@ -3665,7 +3666,18 @@ sub DoUtilization()
 {
     my $utilization;
     my $errmsg;
+    my $errcode;
     my @aggregates = ();
+
+    my $slice = $instance->GetGeniSlice();
+    if (!defined($slice)) {
+	fatal("No slice for instance!");
+    }
+    if ($slice->Lock()) {
+	$errcode = GENIRESPONSE_BUSY;
+	$errmsg  ="Experiment is busy, cannot lock it. Try again later.";
+	goto bad;
+    }
     
     #
     # Get the nodeid to client id mapping
@@ -3694,11 +3706,17 @@ sub DoUtilization()
 	foreach my $node (@nodes) {
 	    my $client_id = GeniXML::GetVirtualId($node);
 	    my $node_id   = GeniXML::GetVnodeId($node);
+	    my $manager_urn = GeniXML::GetManagerId($node);
+
+	    # Combined rspec.
+	    next
+		if (!defined($manager_urn) ||
+		    $manager_urn ne $aggregate->aggregate_urn());
 
 	    $client_ids{$aggregate->aggregate_urn()}->{$node_id} = $client_id;
 	}
     }
-    my $errcode = CallAggregateMethod("Utilization",\$utilization,@aggregates);
+    $errcode = CallAggregateMethod("Utilization",\$utilization,@aggregates);
     if ($errcode) {
 	$errmsg = $utilization;
 	goto bad;
@@ -3717,6 +3735,7 @@ sub DoUtilization()
 	}
 	$aggregate->webtask()->results($blob);
     }
+    $slice->UnLock();
     exit(0);
   bad:
     print STDERR $errmsg . "\n";
@@ -3724,6 +3743,7 @@ sub DoUtilization()
 	$webtask->output($errmsg);
 	$webtask->Exited($errcode);
     }
+    $slice->UnLock();
     exit($errcode);
 }
 
@@ -3817,8 +3837,18 @@ sub DoIdleData()
 {
     my $idledata;
     my $errmsg;
+    my $errcode;
     my @aggregates = ();
     
+    my $slice = $instance->GetGeniSlice();
+    if (!defined($slice)) {
+	fatal("No slice for instance!");
+    }
+    if ($slice->Lock()) {
+	$errcode = GENIRESPONSE_BUSY;
+	$errmsg  ="Experiment is busy, cannot lock it. Try again later.";
+	goto bad;
+    }
     #
     # Cull out any aggregates with no nodes.
     #
@@ -3826,7 +3856,7 @@ sub DoIdleData()
 	push(@aggregates, $aggregate)
 	    if ($aggregate->physnode_count() || $aggregate->virtnode_count());
     }
-    my $errcode = CallAggregateMethod("IdleData", \$idledata, @aggregates);
+    $errcode = CallAggregateMethod("IdleData", \$idledata, @aggregates);
     if ($errcode) {
 	$errmsg = $idledata;
 	goto bad;
@@ -3839,6 +3869,7 @@ sub DoIdleData()
 	}
 	$aggregate->webtask()->idledata($json);
     }
+    $slice->UnLock();
     exit(0);
   bad:
     print STDERR $errmsg . "\n";
@@ -3846,6 +3877,7 @@ sub DoIdleData()
 	$webtask->output($errmsg);
 	$webtask->Exited($errcode);
     }
+    $slice->UnLock();
     exit($errcode);
 }
 
@@ -4431,7 +4463,7 @@ sub CallAggregateMethod($$@)
 	    $$prval = $errmsg;
 	    return $code;
 	}
-	push(@return_values, $webtask->results());
+	push(@return_values, $agg->webtask()->results());
     }
     $$prval = \@return_values;
     return 0;

apt/manage_profile.in

@@ -978,7 +978,7 @@ sub UpdateProfileFromRepo()
 	"as the result of a push webhook.",
 	$project->Brand()->OpsEmailAddress(),
 	# Temporary.
-	"BCC: " . $project->Brand()->OpsEmailAddress());
+	"BCC: " . $project->Brand()->LogsEmailAddress());
     
     return 0;
 }

apt/manage_reservations.in

@@ -86,6 +86,7 @@ use libtestbed;
 use WebTask;
 use APT_Geni;
 use APT_Aggregate;
+use APT_Instance;
 use GeniResponse;
 use GeniUser;
 
@@ -451,6 +452,9 @@ sub DoList()
 	fatal($response->output());
     }
     my $list = $response->value()->{'reservations'};
+    my $history = $response->value()->{'history'}
+        if (exists($response->value()->{'history'}));
+    
     #
     # Map remote URNs to local projects and users. Not all of them can
     # be mapped of course, we leave those as is. 
@@ -477,6 +481,34 @@ sub DoList()
 	else {
 	    $details->{'pid'} = $projhrn->id();
 	}
+
+	#
+	# If we have the history, then go through and map the
+	# experiments to local experiments so we can link to them in
+	# the web interface.
+	#
+	if (exists($history->{$projhrn})) {
+	    foreach my $ref (@{ $history->{$projhrn} }) {
+		#print Dumper($ref);
+		    
+		# Local experiment on the remote cluster.
+		next
+		    if (!(exists($ref->{'urn'}) && exists($ref->{'slice_uuid'})));
+		# Already processed.
+		next
+		    if (exists($ref->{'instance_uuid'}));
+
+		my $hrn = GeniHRN->new($ref->{'urn'});
+		if (defined($hrn) &&
+		    $hrn->domain() eq $OURDOMAIN && defined($hrn->project())) {
+		    my $instance = APT_Instance->LookupBySlice($hrn);
+		    if (defined($instance) &&
+			$instance->slice_uuid() eq $ref->{'slice_uuid'}) {
+			$ref->{'instance_uuid'} = $instance->uuid();
+		    }
+		}
+	    }
+	}
     }
     #
     # Strip out unwanted results if we asked as an admin for a specific
@@ -487,27 +519,36 @@ sub DoList()
 	
 	foreach my $key (keys(%$list)) {
 	    my $details = $list->{$key};
+	    my $projhrn = $details->{'project'};
 
 	    if (defined($project)) {
-		next
-		    if (!defined($details->{'pid_idx'}) ||
-			$details->{'pid_idx'} != $project->pid_idx());
+		if (!defined($details->{'pid_idx'}) ||
+		    $details->{'pid_idx'} != $project->pid_idx()) {
+		    # Kill the history entry when skipping.
+		    delete($history->{$projhrn})
+			if (defined($history) && exists($history->{$projhrn}));
+		    next
+		}
 	    }
 	    else {
-		next
-		    if (!defined($details->{'uid_idx'}) ||
-			$details->{'uid_idx'} != $user->uid_idx());
+		if (!defined($details->{'uid_idx'}) ||
+		    $details->{'uid_idx'} != $user->uid_idx()) {
+		    # Kill the history entry when skipping.
+		    delete($history->{$projhrn}) 
+			if (defined($history) && exists($history->{$projhrn}));
+		    next
+		}
 	    }
 	    $tmp->{$key} = $details;
 	}
-	$list = $tmp;
+	$response->value()->{'reservations'} = $tmp;
     }
     if (defined($webtask)) {
-	$webtask->value($list);
+	$webtask->value($response->value());
 	$webtask->Exited(0);
     }
     else {
-	print Dumper($list);
+	print Dumper($response->value());
     }
     exit(0);
 }
@@ -801,10 +842,11 @@ sub DoPrediction()
 	if ($this_user->ProjectMembershipList(\@plist)) {
 	    fatal("Could not get project membership list");
 	}
+	# Do not need CREATE experiment permission, just membership.
+	# This solves the problem of users having "user" privs in the
+	# project, but local_root in a subgroup.
 	foreach my $project (@plist) {
-	    if ($project->AccessCheck($this_user, TB_PROJECT_CREATEEXPT())) {
-		push(@projlist, $project->urn());
-	    }
+	    push(@projlist, $project->urn());
 	}
 	if (!@projlist) {
 	    fatal("No projects to create experiments in");

assign/assign.cc

 /*
- * Copyright (c) 2000-2010 University of Utah and the Flux Group.
+ * Copyright (c) 2000-2017 University of Utah and the Flux Group.
  *
  * {{{EMULAB-LICENSE
  *
@@ -588,7 +588,7 @@ int type_precheck(int round) {
     }
 
     if (ok) {
-      cout << "Type precheck passed." << endl;
+      cout << "Type precheck succeeded" << endl;
       return 1;
     } else {
       cout << "*** Type precheck failed!" << endl;
@@ -1319,7 +1319,12 @@ int main(int argc,char **argv) {
 #endif
 
   if (violated != 0) {
-      exit(EXIT_RETRYABLE);
+      if (vinfo.max_types) {
+	  exit(EXIT_UNRETRYABLE);
+      }
+      else {
+	  exit(EXIT_RETRYABLE);
+      }
   } else {
       exit(EXIT_SUCCESS);
   }

clientside/os/capture/capture.c

@@ -164,6 +164,7 @@ char   **programargv;
 #define programmode 0
 #define xendomain 0
 #define retryinterval 0
+#define nomodpath 0
 #else
 char		  *Bossnode = BOSSNODE;
 struct sockaddr_in Bossaddr;
@@ -174,6 +175,8 @@ int		   remotemode;
 int		   programmode;
 char		  *xendomain;
 int		   retryinterval = 5000;
+int		   maxretries = 0;
+int		   nomodpath = 0;
 int		   maxfailures = 10;
 int		   failures;
 int		   upportnum = -1, upfd = -1, upfilefd = -1;
@@ -381,7 +384,7 @@ main(int argc, char **argv)
 	else
 		Progname = *argv;
 
-	while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:")) != EOF)
+	while ((op = getopt(argc, argv, "rds:Hb:ip:c:T:aonu:v:PmMLCl:X:R:y:A")) != EOF)
 		switch (op) {
 #ifdef	USESOCKETS
 #ifdef  WITHSSL
@@ -428,6 +431,12 @@ main(int argc, char **argv)
 				usage();
 			}
 			break;
+		case 'y':
+			maxretries = atoi(optarg);
+			break;
+		case 'A':
+			nomodpath = 1;
+			break;
 #endif /* USESOCKETS */
 		case 'H':
 			++hwflow;
@@ -510,6 +519,8 @@ main(int argc, char **argv)
 	}
 	else if (xendomain)
 		strcpy(strbuf, xendomain);
+	else if (nomodpath)
+		strcpy(strbuf, argv[1]);
 	else
 		(void) snprintf(strbuf, sizeof(strbuf),
 				DEVNAME, DEVPATH, argv[1]);
@@ -997,6 +1008,8 @@ capture(void)
 	sigset_t omask;
 	char buf[BUFSIZE];
 	struct timeval timeout;
+	int nretries;
+
 
 	/*
 	 * XXX for now we make both directions non-blocking.  This is a
@@ -1128,7 +1141,8 @@ capture(void)
 					Machine, cc);
 			if (cc <= 0) {
 #ifdef  USESOCKETS
-				if (remotemode || programmode || xendomain) {
+				if (remotemode || programmode || xendomain
+				    || maxretries) {
 					FD_CLR(devfd, &sfds);
 					close(devfd);
 					devfd = -1;
@@ -1154,7 +1168,7 @@ capture(void)
 						}
 
 					}
-					else {
+					else if (xendomain) {
 						warning("xen console %s closed;"
 							" attempting to reopen",
 							Devname);
@@ -1169,6 +1183,22 @@ capture(void)
 								fdcount = xsfd + 1;
 						}
 					}
+					else {
+						warning("devfd %s closed;"
+							" attempting to reopen",