Backend support for repository-based profiles. clone and update and

remove are run on ops via the proxy script. Other smaller operations
like getting the source, logs, commit info are currently run on boss via
NFS to /usr/testbed/opsdir/repositories, which are one per profile.
Lots of work still to do on optimizing repositories.

apt/GNUmakefile.in

@@ -32,20 +32,20 @@ SUBDIRS		=
 
 BIN_SCRIPTS	= manage_profile manage_instance manage_dataset \
                   create_instance rungenilib ns2rspec nsgenilib.py \
-                  rspec2genilib ns2genilib manage_reservations
+                  rspec2genilib ns2genilib manage_reservations manage_gitrepo
 SBIN_SCRIPTS	= apt_daemon aptevent_daemon portal_xmlrpc apt_checkup \
 		  portal_monitor
 LIB_SCRIPTS     = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm \
 		  APT_Aggregate.pm APT_Utility.pm
 WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
                   webcreate_instance webrungenilib webns2rspec webns2genilib \
-		  webrspec2genilib webmanage_reservations
-WEB_SBIN_SCRIPTS= webportal_xmlrpc
+		  webrspec2genilib webmanage_reservations webmanage_gitrepo
+WEB_SBIN_SCRIPTS= webportal_xmlrpc 
 LIBEXEC_SCRIPTS	= $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS)
-USERLIBEXEC     = rungenilib.proxy genilib-jail genilib-iocage
+USERLIBEXEC     = rungenilib.proxy genilib-jail genilib-iocage gitrepo.proxy
 
 # These scripts installed setuid, with sudo. 
-SETUID_BIN_SCRIPTS   = rungenilib
+SETUID_BIN_SCRIPTS   = rungenilib manage_gitrepo
 SETUID_SBIN_SCRIPTS  = 
 SETUID_SUEXEC_SCRIPTS= 
 

apt/create_instance.in

@@ -262,7 +262,7 @@ foreach my $key ("username", "email", "profile", "portal") {
 #
 my ($value, $user_urn, $user_uid, $user_hrn, $user_email, $project, $pid,
     $gid, $group, $sshkey, $profile, $profileid, $version, $rspecstr, $errmsg,
-    $userslice_id, $portal);
+    $userslice_id, $portal, $script, $reporef, $repohash);
 
 # This is used internally to determine which portal was used.
 $portal = $xmlparse->{'attribute'}->{"portal"}->{'value'};
@@ -312,7 +312,7 @@ $profileid = $profile->profileid();
 $version   = $profile->version();
 
 #
-# Optional rspec, as for a Parameterized Profile.
+# Optional rspec, as for a Parameterized Profile or a repo-based profile.
 #
 if (exists($xmlparse->{'attribute'}->{"rspec"})) {
     $rspecstr  = $xmlparse->{'attribute'}->{"rspec"}->{'value'};
@@ -320,6 +320,43 @@ if (exists($xmlparse->{'attribute'}->{"rspec"})) {
 else {
     $rspecstr  = $profile->CheckFirewall(!$localuser);
 }
+#
+# Optional rspec and/or script, as for a repo-based profile.
+#
+if ($profile->repourl()) {
+    if (exists($xmlparse->{'attribute'}->{"script"})) {
+	$script = $xmlparse->{'attribute'}->{"script"}->{'value'};	
+
+	if (! TBcheck_dbslot($script, "apt_profiles", "script",
+			     TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
+	    fatal("Illegal script for repo-based profile");
+	}
+	if (!exists($xmlparse->{'attribute'}->{"reporef"})) {
+	    fatal("Missing refspec for repository");
+	}
+    }
+    if (exists($xmlparse->{'attribute'}->{"reporef"})) {
+	if (!exists($xmlparse->{'attribute'}->{"repohash"})) {
+	    fatal("Missing hash for repository");
+	}
+	$reporef  = $xmlparse->{'attribute'}->{"reporef"}->{'value'};
+	$repohash = $xmlparse->{'attribute'}->{"repohash"}->{'value'};
+
+	if (! TBcheck_dbslot($repohash, "apt_profiles", "repohash",
+			     TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
+	    fatal("Illegal repository hash");
+	}
+	if (! TBcheck_dbslot($reporef, "default", "tinytext",
+			     TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
+	    fatal("Illegal repository refspec");
+	}
+    }
+    my $tmp = APT_Profile::SetRepo(\$rspecstr, $profile->repourl(),
+				   $reporef, $repohash, \$errmsg);
+    if ($tmp) {
+	($tmp < 0 ? fatal($errmsg) : UserError($errmsg));
+    }
+}
 
 #
 # Update rspec with site aggregate urns.
@@ -763,6 +800,21 @@ my $blob = {'uuid'         => $quickvm_uuid,
 	    'cert'         => $alt_certificate->cert(),
 	    'privkey'      => $alt_certificate->PrivKeyDelimited(),
 };
+if ($profile->repourl()) {
+    if (defined($script)) {
+	$blob->{"script"}  = $script;
+    }
+    $blob->{"repourl"}   = $profile->repourl();
+    # These come in from the instantiate page.
+    if (defined($reporef)) {
+	$blob->{"reporef"}   = $reporef;
+	$blob->{"repohash"}  = $repohash;
+    }
+    else {
+	# Otherwise we are instantiating whatever the profile itself references.
+	$blob->{"repohash"}  = $profile->repohash();
+    }
+}
 if (defined($project)) {
     $blob->{"pid"}     = $project->pid();
     $blob->{"pid_idx"} = $project->pid_idx();

apt/gitrepo.proxy.in

+#!/usr/bin/perl -w
+
+#
+# Copyright (c) 2000-2016 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+use strict;
+use English;
+use Getopt::Std;
+use BSD::Resource;
+use POSIX qw(:signal_h);
+    
+#
+# Simply a wrapper for the geni-lib python environment
+#
+sub usage()
+{
+    print STDOUT
+	"Usage: gitrepo.proxy -n reponame clone url\n".
+	"Usage: gitrepo.proxy -n reponame update\n".
+	"Usage: gitrepo.proxy -n reponame delete\n";
+ 
+    exit(-1);
+}
+my $optlist   = "dn:";
+my $debug    = 0;
+my $reponame;
+
+#
+# Configure variables
+#
+my $TB         = "@prefix@";
+my $TBOPS      = "@TBOPSEMAIL@";
+my $REPODIR    = "$TB/repositories";
+my $GIT        = "/usr/local/bin/git";
+
+# Locals
+
+# Protos
+sub fatal($);
+sub RunCommand($);
+sub Clone();
+sub Delete();
+sub Update();
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libtestbed;
+
+#
+# Parse command arguments. Once we return from getopts, all that should
+# left are the required arguments.
+#
+my %options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (defined($options{"n"})) {
+    $reponame = $options{"n"};
+}
+usage()
+    if (@ARGV < 1 || !(defined($reponame)));
+
+my $action   = shift(@ARGV);
+
+if (! -e $REPODIR) {
+    fatal("$REPODIR directory does not exist");
+}
+
+# Do not want to be asked for password, we want straight out failure.
+$ENV{"GIT_ASKPASS"} = "/bin/true";
+
+#
+# Dispatch.
+#
+if ($action eq "clone") {
+    exit(Clone());
+}
+elsif ($action eq "update") {
+    exit(Update());
+}
+elsif ($action eq "delete") {
+    exit(Delete());
+}
+else {
+    fatal("Unknown command");
+}
+exit(-1);
+
+#
+# Clone a repo. 
+#
+sub Clone()
+{
+    usage()
+	if (!@ARGV);
+    my $repourl = shift(@ARGV);
+    
+    if (-e "$REPODIR/$reponame") {
+	fatal("Repository has already been cloned");
+    }
+    chdir($REPODIR) or
+	fatal("Could not chdir to $REPODIR");
+
+    my $status = RunCommand("$GIT clone --bare ".
+			    "   $repourl $reponame");
+    if ($status) {
+	fatal("Not able to clone repo from $repourl");
+    }
+    if (! -d "$REPODIR/$reponame") {
+	fatal("Cannot find the git repo after cloning");
+    }
+    #
+    # Make sure a source file exists, delete if not. The caller will
+    # pick up the file via NFS if it needs it.
+    #
+    chdir("$REPODIR/$reponame") or
+	fatal("Could not chdir to $REPODIR/$reponame");
+    
+    if (system("$GIT cat-file -e refs/heads/master:profile.py") &&
+	system("$GIT cat-file -e refs/heads/master:profile.rspec")) {
+	chdir("$REPODIR");
+	system("/bin/rm -rf $REPODIR/$reponame");
+	fatal("No geni-lib script or rspec in this repository");
+    }
+    return 0;
+}
+
+#
+# Delete a repository.
+#
+sub Delete()
+{
+    if (! -e "$REPODIR/$reponame") {
+	return 0;
+    }
+    system("/bin/rm -rf $REPODIR/$reponame");
+    if ($?) {
+	fatal("Not able to delete repository");
+    }
+    return 0;
+}
+
+#
+# Update a repo. 
+#
+sub Update()
+{
+    if (! -e "$REPODIR/$reponame") {
+	fatal("Repository does not exist.");
+    }
+    chdir("$REPODIR/$reponame") or
+	fatal("Could not chdir to $REPODIR/$reponame");
+    
+    my $status = RunCommand("$GIT fetch -t origin '+refs/*:refs/*'");
+    if ($status) {
+	fatal("Not able to update repo");
+    }
+    if (system("$GIT cat-file -e refs/heads/master:profile.py") &&
+	system("$GIT cat-file -e refs/heads/master:profile.rspec")) {
+	fatal("No geni-lib script or rspec in this repository");
+    }
+    return 0;
+}
+
+#
+# Run git protected by CPU limit.
+#
+sub RunCommand($)
+{
+    my ($cmd) = @_;
+    
+    #
+    # Fork a child process to run git in.
+    #
+    my $pid = fork();
+    if (!defined($pid)) {
+	fatal("Could not fork a new process!");
+    }
+
+    #
+    # Child runs git, niced down, and then exits.
+    #
+    if (! $pid) {
+	# Set the CPU limit for us.
+	setrlimit(RLIMIT_CPU, 600, 600);
+	
+	# Give parent a chance to react.
+	sleep(1);
+
+	exec("nice -5 $cmd");
+	die("Could not exec: '$cmd'!\n");
+    }
+
+    # Parent waits.
+    waitpid($pid, 0);
+    my $exit_status = $?;
+
+    #
+    # If the child was KILLed, then it overran its time limit.
+    # Otherwise, exit with result of child.
+    #
+    if (($exit_status & 0xff) == SIGKILL) {
+	print STDERR "git exceeded CPU limit\n";
+	$exit_status = 15;
+    }
+    elsif ($exit_status & 0xff) {
+	# Get the mapping from signal num. to name
+	use Config;
+	my (%sig_num, @sig_name);
+	my @names = split ' ', $Config{sig_name};
+	@sig_num{@names} = split ' ', $Config{sig_num};
+	foreach (@names) {$sig_name[$sig_num{$_}] ||= $_}
+    
+	my $signal = $exit_status & 0x7f;
+	my $signame = $sig_name[$signal];
+	
+	print STDERR "git died with SIG$signame.\n";
+	$exit_status = 128 + $signal;
+    }
+    elsif ($exit_status) {
+	$exit_status = $exit_status >> 8;
+    }
+    return $exit_status;
+}
+
+sub fatal($) {
+    my ($mesg) = $_[0];
+
+    print STDERR "*** $0:\n".
+	         "    $mesg\n";
+    exit(-1);
+}

apt/manage_profile.in

@@ -42,11 +42,13 @@ sub usage()
     print("Usage: manage_profile delete <profile> [all]\n");
     exit(-1);
 }
-my $optlist     = "ds:t:c:";
+my $optlist     = "ds:t:c:m";
 my $debug       = 0;
 my $update      = 0;
 my $snap        = 0;
 my $copy        = 0;
+my $fromrepo    = 0;
+my $updatemaster= 0;
 my $copyuuid;
 my $uuid;
 my $rspec;
@@ -66,6 +68,7 @@ my $TB		    = "@prefix@";
 my $TBOPS           = "@TBOPSEMAIL@";
 my $TBLOGS	    = "@TBLOGSEMAIL@";
 my $MANAGEINSTANCE  = "$TB/bin/manage_instance";
+my $MANAGEGITREPO   = "$TB/bin/manage_gitrepo";
 my $RUNGENILIB      = "$TB/bin/rungenilib";
 
 #
@@ -87,6 +90,7 @@ use EmulabConstants;
 use emdb;
 use emutil;
 use libEmulab;
+use libtestbed;
 use User;
 use Project;
 use APT_Profile;
@@ -210,6 +214,7 @@ my %xmlfields =
 			                          $SLOT_UPDATE|$SLOT_ADMINONLY],
      "rspec"		   => ["rspec",		$SLOT_REQUIRED|$SLOT_UPDATE],
      "script"		   => ["script",	$SLOT_OPTIONAL|$SLOT_UPDATE],
+     "repourl"		   => ["repourl",	$SLOT_OPTIONAL],
 );
 
 #
@@ -332,6 +337,60 @@ if (defined($rspec)) {
     }
 }
 
+if ($update) {
+    $profile = APT_Profile->Lookup($uuid);
+    if (!defined($profile)) {
+	fatal("Could not lookup profile for update $uuid");
+    }
+}
+
+#
+# Look for repourl. We already cloned it once to get the source, but
+# need to clone to for real now that we can assign a name to it. Only
+# for new profiles of course, the user cannot change the url later.
+#
+if (exists($new_args{'repourl'})) {
+    my $reponame;
+    my $repohash;
+    
+    if ($update) {
+	$reponame = $profile->reponame();
+	# Ignore.
+	delete($new_args{'repourl'});
+    }
+    else {
+	my $repourl  = $new_args{'repourl'};
+	$reponame = NewUUID();
+
+	my $output =
+	    emutil::ExecQuiet("$MANAGEGITREPO clone -n $reponame '$repourl'");
+	if ($?) {
+	    print STDERR $output;
+	    $errors{"error"} = $output;
+	    UserError();
+	}
+	$new_args{'reponame'} = $reponame;
+    }
+    #
+    # Get the commit hash for the HEAD commit.
+    #
+    my $output = emutil::ExecQuiet("$MANAGEGITREPO hash -n $reponame");
+    if ($?) {
+	print STDERR $output;
+	$errors{"error"} = $output;
+	UserError();
+    }
+    $repohash = $output;
+    chomp($repohash);
+    if ($update) {
+	$update_args{'repohash'} = $repohash;
+    }
+    else {
+	$new_args{'repohash'} = $repohash;
+    }
+    $fromrepo = 1;
+}
+
 #
 # See if this is a Parameterized Profile. Generate and store the form
 # data if it is. Only python scripts of course.
@@ -400,10 +459,6 @@ elsif ($copy) {
 }
 
 if ($update) {
-    $profile = APT_Profile->Lookup($uuid);
-    if (!defined($profile)) {
-	fatal("Could not lookup profile for update $uuid");
-    }
     # Kill the description.. No longer used.
     delete($update_args{"description"});
 
@@ -422,8 +477,11 @@ if ($update) {
 	if ((exists($update_args{"rspec"}) &&
 	     $update_args{"rspec"} ne $profile->rspec()) ||
 	    (exists($update_args{"script"}) &&
-	     $update_args{"script"} ne $profile->script())) {
-	    if ($doversions) {
+	     $update_args{"script"} ne $profile->script()) ||
+	    (defined($profile->repourl()) &&
+	     exists($update_args{"repohash"}) &&
+	     $update_args{"repohash"} ne $profile->repohash())) {
+	    if ($doversions && !$fromrepo) {
 		$profile = $profile->NewVersion($this_user);
 		if (!defined($profile)) {
 		    fatal("Could not create new version of the profile");
@@ -432,19 +490,15 @@ if ($update) {
 		$webtask->newProfile($profile->uuid())
 		    if (defined($webtask));
 	    }
-	    $profile->UpdateVersion({"rspec" => $update_args{"rspec"}})
-		if (exists($update_args{"rspec"}));
-	    $profile->UpdateVersion({"script" => $update_args{"script"}})
-		if (exists($update_args{"script"}));
-	    $profile->UpdateVersion({"paramdefs" => $update_args{"paramdefs"}})
-		if (exists($update_args{"paramdefs"}));
+	    foreach my $key ("rspec", "script", "paramdefs", "repohash") {
+		$profile->UpdateVersion({$key => $update_args{$key}})
+		    if (exists($update_args{$key}));
+	    }
+	}
+	foreach my $key ("rspec", "script", "paramdefs", "repohash") {
+	    delete($update_args{$key})
+		if (exists($update_args{$key}));
 	}
-	delete($update_args{"rspec"})
-	    if (exists($update_args{"rspec"}));
-	delete($update_args{"script"})
-	    if (exists($update_args{"script"}));
-	delete($update_args{"paramdefs"})
-	    if (exists($update_args{"paramdefs"}));
     }
     $profile->UpdateMetaData(\%update_args) == 0 or
 	fatal("Could not update profile record");