Commit 7c7815d8 authored by Leigh Stoller's avatar Leigh Stoller

Checkpoint some changes to repo based profiles:

1) Add support for repos that specify submodules. The main issue here is
   that a "bare" repo has no notion of sub modules, so we actually have
   to do a checkout of the repo to know there are submodules and so we
   can get the contents of them, in case the profile.py is a symlink
   into a sub module. The value of submodule supports is so that a repo
   based profile can be composed from several sub repos. At the moment,
   only admins can do this since that turns on the flag to do a checkout
   of the repo instead of a bare repo (needs more thought). But once the
   profile is created (repo cloned locally with a checkout), it can
   still be instantiated in green-dot mode.

2) Add support for running the geni-lib code with the repository mapped
   into the geni-lib jail, so the profile can reference library modules
   contained in the repo. The repo is actually a checkout since most
   repos are bare, so we need to clone it into the jail. The directory
   where the clone is created is added to PYTHONPATH for import to work
   as expected.
parent 1450f7a4
......@@ -83,7 +83,7 @@ sub usage()
exit(-1);
}
my $optlist = "du:p:o:n:CRB:N";
my $optlist = "du:p:o:n:CRB:Nr:h:";
my $basename = "py-cage";
my $jailname = "py-cage-$$";
my $user = "nobody";
......@@ -92,6 +92,8 @@ my $gid;
my $pfile;
my $ofile;
my $ifile;
my $repo;
my $reporef;
my $limits = 1;
my $haverctl = 0;
......@@ -109,6 +111,8 @@ sub mysystem($);
#
my $TBROOT = "@prefix@";
my $GENILIB = "$TBROOT/lib/geni-lib/";
my $REPODIR = "/repos";
my $GIT = "/usr/local/bin/git";
my $debug = 0;
# Watch for this being defined in the calling environment and use that.
......@@ -229,6 +233,31 @@ if (defined($options{"B"})) {
}
$basename = $base;
}
if (defined($options{"r"})) {
$repo = $options{"r"};
# Must taint check
if ($repo =~ /^([-\w]+)$/) {
$repo = $1;
}
else {
print STDERR "Bad data in argument: $repo\n";
usage();
}
if (! -e "$REPODIR/$repo") {
print STDERR "No such repo '$repo'\n";
usage();
}
if (defined($options{"h"})) {
$reporef = $options{"h"};
# Must taint check
if ($reporef =~ /^([-\w\/]+)$/) {
$reporef = $1;
}
else {
die("Bad data in argument: $reporef");
}
}
}
#
# Extract params from the environment (if invoked via rungenilib.proxy).
......@@ -387,6 +416,32 @@ if ($action != 1) {
print STDERR "Could not populate jail\n";
exit(-1);
}
if (defined($repo)) {
if (mysystem("$GIT clone -q $REPODIR/$repo ".
" ${jailrootdir}${tempdir}/repository")) {
print STDERR "Could not clone repo in the jail\n";
exit(-1);
}
# The root of the repository for module imports.
$ENV{"PYTHONPATH"} = "/${tempdir}/repository:" . $ENV{"PYTHONPATH"};
if (defined($reporef)) {
if ($reporef =~ m"^(refs/|)heads/(.+)") {
my $branchname = $2;
mysystem("cd ${jailrootdir}${tempdir}/repository; ".
"$GIT checkout -q $branchname");
}
else {
mysystem("cd ${jailrootdir}${tempdir}/repository; ".
"$GIT checkout -q $reporef");
}
if ($?) {
print STDERR "Could not clone repo in the jail\n";
exit(-1);
}
}
}
#
# XXX adjust the environment for the portal module to reflect the jail.
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
# Copyright (c) 2000-2019 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -34,7 +34,7 @@ use POSIX qw(:signal_h);
sub usage()
{
print STDOUT
"Usage: gitrepo.proxy -n reponame clone url\n".
"Usage: gitrepo.proxy -n reponame clone [-c] url\n".
"Usage: gitrepo.proxy -n reponame update\n".
"Usage: gitrepo.proxy -n reponame delete\n";
......@@ -127,8 +127,17 @@ exit(-1);
#
sub Clone()
{
my $checkout = 0;
usage()
if (!@ARGV);
if ($ARGV[0] eq "-c") {
$checkout = 1;
shift(@ARGV);
usage()
if (!@ARGV);
}
my $repourl = shift(@ARGV);
if (-e "$REPODIR/$reponame") {
......@@ -137,7 +146,7 @@ sub Clone()
chdir($REPODIR) or
fatal("Could not chdir to $REPODIR");
my $status = RunCommand("$GIT clone --bare ".
my $status = RunCommand("$GIT clone " . ($checkout ? "" : "--bare ") .
" $repourl $reponame");
if ($status) {
fatal("Not able to clone repo from $repourl");
......@@ -152,12 +161,39 @@ sub Clone()
chdir("$REPODIR/$reponame") or
fatal("Could not chdir to $REPODIR/$reponame");
#
# If we did a checkout, look for submodules that need to be initialized
#
if ($checkout) {
if (-e ".gitmodules") {
if (system("$GIT submodule init") ||
system("$GIT submodule update")) {
fatal("Could not initialize submodules");
}
}
# Need to force getting all the remote branches.
# -p prunes deleted branches. But tags are not pruned.
if (system("$GIT fetch -u -f -p -t origin '+refs/*:refs/*'")) {
fatal("Could not fetch remote branches");
}
if (-e "profile.py") {
system("/bin/cat profile.py");
}
elsif (-e "profile.rspec") {
system("/bin/cat profile.rspec");
}
else {
print STDERR "No geni-lib script or rspec in this repository\n";
}
}
else {
my $refspec = GetDefaultBranch($reponame);
if (system("$GIT cat-file -e ${refspec}:profile.py") &&
system("$GIT cat-file -e ${refspec}:profile.rspec")) {
print STDERR "No geni-lib script or rspec in this repository\n";
}
}
return 0;
}
......@@ -187,12 +223,31 @@ sub Update()
chdir("$REPODIR/$reponame") or
fatal("Could not chdir to $REPODIR/$reponame");
#
# When not using a bare repo, we have to force the fetch to update
# the current banch.
#
my $fopt = (-e ".git" ? "-u -f" : "");
# -p prunes deleted branches. But tags are not pruned.
my $status = RunCommand("$GIT fetch -p -t origin '+refs/*:refs/*'");
my $status = RunCommand("$GIT fetch $fopt -p -t origin '+refs/*:refs/*'");
if ($status) {
fatal("Not able to update repo");
}
#
# Local checkout, must merge. This is done when we have submodules.
#
if (-e ".git") {
my $status = RunCommand("$GIT merge");
if ($status) {
fatal("Not able to fetch repo");
}
$status = RunCommand("$GIT submodule update");
if ($status) {
fatal("Not able to update submodules");
}
}
my $current_refspec = GetDefaultBranch($reponame);
my $remote_refspec = GetRemoteDefaultBranch($reponame);
......@@ -206,10 +261,23 @@ sub Update()
}
$current_refspec = $remote_refspec;
}
if (-e ".git") {
if (-e "profile.py") {
system("/bin/cat profile.py");
}
elsif (-e "profile.rspec") {
system("/bin/cat profile.rspec");
}
else {
print STDERR "No geni-lib script or rspec in this repository\n";
}
}
else {
if (system("$GIT cat-file -e ${current_refspec}:profile.py") &&
system("$GIT cat-file -e ${current_refspec}:profile.rspec")) {
print STDERR "No geni-lib script or rspec in this repository\n";
}
}
return 0;
}
......
......@@ -108,6 +108,8 @@ sub DoCommitList();
sub DoCommitInfo();
sub DoGetSource();
sub DoGetRepoSize();
sub DoRemoveRepo();
sub DoPruneStaleRepos();
sub GetRepoSource($;$$);
sub GetRepoSize($);
sub GetBranchList($);
......@@ -195,6 +197,12 @@ elsif ($action eq "commitinfo") {
elsif ($action eq "reposize") {
DoGetRepoSize();
}
elsif ($action eq "remove") {
DoRemoveRepo();
}
elsif ($action eq "prune") {
DoPruneStaleRepos();
}
else {
usage();
}
......@@ -264,15 +272,17 @@ sub DoCheckRemote()
# Use -o to write the file to stdout or a file.
# Use -r to remove repo after getting the script/rspec.
# Add -u to update if repo is already cloned.
# Add -c for a full checkout. Only admins for now.
#
sub DoClone()
{
my $optlist = "o:rn:uS:";
my $optlist = "o:rn:uS:c";
my $ofile;
my $remove;
my $reponame;
my $update;
my $sourcename;
my $checkout = 0;
my %options = ();
if (! getopts($optlist, \%options)) {
......@@ -286,6 +296,9 @@ sub DoClone()
if ($repourl =~ /^(.*)$/) {
$repourl = $1;
}
if (defined($options{"c"})) {
$checkout = 1;
}
if (defined($options{"o"})) {
$ofile = $options{"o"};
}
......@@ -320,7 +333,11 @@ sub DoClone()
$cmd .= "-n $reponame update";
}
else {
$cmd .= "-n $reponame clone '$repourl'";
$cmd .= "-n $reponame clone ";
if ($checkout) {
$cmd .= " -c ";
}
$cmd .= "'$repourl'";
}
if ($debug) {
print "'$cmd'\n";
......@@ -345,7 +362,7 @@ sub DoClone()
RemoveRepo($reponame);
fatal("Could not estimate repository size");
}
if ($size > 500) {
if ($size > 150) {
RemoveRepo($reponame);
UserError("Repository is too big: greate then 500MiB");
}
......@@ -368,6 +385,7 @@ sub DoClone()
$webtask->log($log);
$webtask->hash($hash);
$webtask->size("$size MiB");
$webtask->name($reponame);
}
if (defined($ofile)) {
if ($ofile eq "-") {
......@@ -622,40 +640,43 @@ sub GetRepoSource($;$$)
fatal("Could not chdir to $repodir: $!");
foreach my $maybe (@locations) {
my $file;
if (system("$GIT cat-file -e ".
"$refspec:${maybe}.py >/dev/null 2>&1") == 0) {
$source = "${maybe}.py";
$file = "${maybe}.py";
}
elsif (system("$GIT cat-file -e ".
" $refspec:${maybe}.rspec >/dev/null 2>&1") == 0) {
$source = "${maybe}.rspec";
}
last
if ($source);
}
if (!$source) {
print STDERR "$repodir, $refspec\n";
print STDERR `/usr/bin/id`;
print STDERR `/bin/ls -la`;
print STDERR "Could not find source code in repository: $reponame\n";
return undef;
$file = "${maybe}.rspec";
}
if ($file) {
#
# Do this seemingly odd cat-file, simply cause its the only way
# --follow-symlinks works. It adds the commit hash as the first
# line of output, so see below where that first line is killed.
#
my $stuff =
emutil::ExecQuiet("echo '$refspec:$source' | ".
" $GIT cat-file --batch --follow-symlinks");
$source =
emutil::ExecQuiet("echo '$refspec:$file' | ".
" $GIT cat-file --batch ".
" --follow-symlinks");
if ($?) {
print STDERR $stuff;
print STDERR $source;
return undef;
}
# Kill first line.
$stuff =~ s/^(?:.*\n){1}//;
return $stuff;
$source =~ s/^(?:.*\n){1}//;
last;
}
}
if (!$source) {
print STDERR "$repodir, $refspec\n";
print STDERR `/usr/bin/id`;
print STDERR `/bin/ls -la`;
print STDERR "Could not find source code in repository: $reponame\n";
return undef;
}
return $source;
}
#
......@@ -756,6 +777,36 @@ sub RemoveRepo($)
return 0;
}
#
# Remove a repo
#
sub DoRemoveRepo()
{
my $optlist = "n:p:";
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
my $reponame = GetRepoName(\%options);
if (! -e "$REPODIR/$reponame") {
fatal("Repository does not exist.");
}
if (RemoveRepo($reponame)) {
if (defined($webtask)) {
$webtask->Exited(-1);
}
exit(-1);
}
else {
if (defined($webtask)) {
$webtask->Exited(0);
}
exit(0);
}
}
#
# Return a branch list.
#
......@@ -1222,6 +1273,48 @@ sub GetRepoSize($)
return $mebi;
}
#
# Prune stale repos (repos we left behind).
#
sub DoPruneStaleRepos()
{
my $optlist = "n";
my $impotent = 1;
my @stale = ();
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"n"})) {
$impotent = 1;
}
chdir("$REPODIR") or
fatal("Could not chdir to $REPODIR");
opendir(DIR, $REPODIR) or
fatal("Unable to open directory $REPODIR");
while (my $dirent = readdir(DIR)) {
next
if ($dirent eq "." || $dirent eq "..");
next
if (!ValidUUID($dirent));
my $query_result =
DBQueryFatal("select uuid,deleted from apt_profile_versions ".
"where reponame='$dirent'");
if (!$query_result->numrows) {
if ($impotent) {
print "Would delete stale repo $dirent\n";
}
push(@stale, $dirent);
next;
}
}
exit(0);
}
#
# Get estimated repository size.
#
......
......@@ -121,7 +121,7 @@ sub CanDelete($$);
sub PublishProfile($);
sub InsertImageRecords($);
sub ListImages();
sub GetScriptParameters($$);
sub GetScriptParameters($$$);
sub VerifyXML($$);
sub ModifyProfileInternal($$$);
sub UseNewGenilib($);
......@@ -254,6 +254,7 @@ sub CreateProfile()
my $parent_profile;
my $node_id;
my $usererror;
my $reponame;
my %errors = ();
my %options = ();
......@@ -308,12 +309,13 @@ sub CreateProfile()
# Need to do initial clone.
#
if (exists($new_args{'repourl'})) {
$reponame = NewUUID();
my $repourl = $new_args{'repourl'};
my $reponame = NewUUID();
my $repohash;
my $checkout = ($this_user->IsAdmin() ? "-c" : "");
my $output =
emutil::ExecQuiet("$MANAGEGITREPO clone -n $reponame ".
emutil::ExecQuiet("$MANAGEGITREPO clone $checkout -n $reponame ".
" -S " . $new_args{"name"} . " '$repourl'");
if ($?) {
UserError($output);
......@@ -344,7 +346,7 @@ sub CreateProfile()
# Script parameters
if (defined($script) && $script ne "" && $script =~ /^import/m) {
my $paramdefs;
my $retval = GetScriptParameters($script, \$paramdefs);
my $retval = GetScriptParameters($script, $reponame, \$paramdefs);
if ($retval) {
if ($retval < 0) {
fatal("Could not get paramdefs: $paramdefs!");
......@@ -738,7 +740,8 @@ sub ModifyProfileInternal($$$)
# data. Only python scripts of course.
#
my $output;
my $retval = GetScriptParameters($script, \$output);
my $retval = GetScriptParameters($script,
$profile->reponame(), \$output);
if ($retval) {
if ($retval < 0) {
$$pmsg = $output;
......@@ -1008,6 +1011,8 @@ sub UpdateProfileFromRepo()
fatal("Could not open temporary file for script");
}
my $opts = ($usenewgenilib ? "-N" : "");
# Import repo into jail.
$opts .= " -r " . $profile->reponame();
print $fh $script;
$output = emutil::ExecQuiet("$RUNGENILIB $opts $filename");
if ($?) {
......@@ -1064,9 +1069,9 @@ sub UpdateProfileFromRepo()
# For a Parameterized Profile, need to generate and store the form
# data. Only python scripts of course. Does not return on error.
#
sub GetScriptParameters($$)
sub GetScriptParameters($$$)
{
my ($script, $pref) = @_;
my ($script, $reponame, $pref) = @_;
my ($fh, $filename) = tempfile(UNLINK => 1);
if (!defined($fh)) {
......@@ -1074,6 +1079,8 @@ sub GetScriptParameters($$)
return -1;
}
my $opts = ($usenewgenilib ? "-N" : "");
# Import repo into jail.
$opts .= " -r $reponame" if (defined($reponame));
print $fh $script;
my $output = emutil::ExecQuiet("$RUNGENILIB $opts -p $filename");
......
......@@ -45,14 +45,18 @@ sub usage()
print STDERR " -P file - Generate and write parameter block to file\n";
print STDERR " -b file - Run script using the parameter defs in file\n";
print STDERR " -W - Python warnings are fatal.\n";
print STDERR " -r repo - Map repo into jail.\n";
print STDERR " -h hash - With -r, set the checkout hash.\n";
exit(-1);
}
my $optlist = "do:pP:b:WN";
my $optlist = "do:pP:b:WNr:h:";
my $debug = 0;
my $getparams = 0;
my $paramfile;
my $ofile;
my $repo;
my $reporef;
my $newgenilib = 0;
my $warningsfatal = 0;
......@@ -64,6 +68,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $CONTROL = "@USERNODE@";
my $MAINSITE = @TBMAINSITE@;
my $TAR = "/usr/bin/tar";
my $REPODIR = "/repos";
# Locals
my $SAVEUID = $UID;
......@@ -147,6 +152,29 @@ if (defined($options{"b"})) {
if (defined($options{"o"})) {
$ofile = $options{"o"};
}
if (defined($options{"r"})) {
$repo = $options{"r"};
# Must taint check
if ($repo =~ /^([-\w]+)$/) {
$repo = $1;
}
else {
die("Bad data in argument: $repo");
}
if (! -e "$REPODIR/$repo") {
die("No such repo $repo\n");
}
if (defined($options{"h"})) {
$reporef = $options{"h"};
# Must taint check
if ($reporef =~ /^([-\w\/]+)$/) {
$reporef = $1;
}
else {
die("Bad data in argument: $reporef");
}
}
}
if (@ARGV != 1) {
usage();
}
......@@ -219,6 +247,8 @@ $cmdargs .= " -u " . $this_user->uid();
$cmdargs .= ($getparams ? " -p " : "");
$cmdargs .= ($warningsfatal ? " -W " : "");
$cmdargs .= ($newgenilib ? " -N " : "");
$cmdargs .= (defined($repo) ? " -r $repo " : "");
$cmdargs .= (defined($reporef) ? " -h $reporef " : "");
#
# We want to send over both files via STDIN, so combine them, and pass
......@@ -260,6 +290,9 @@ while (<ERR>) {
$errs .= $_;
}
close(ERR);
if ($debug) {
print STDERR $errs;
}
my $exit_status = $?;
if ($exit_status) {
......
......@@ -40,13 +40,15 @@ sub usage()
exit(-1);
}
my $optlist = "u:vpb:WJB:N";
my $optlist = "u:vpb:WJB:Nr:h:";
my $user;
my $getparams= 0;
my $paramsize;
my $warningsfatal = 0;
my $usejail = 0;
my $iocagepath;
my $repo;
my $reporef;
#
# Configure variables
......@@ -56,6 +58,7 @@ my $TBOPS = "@TBOPSEMAIL@";
my $TESTMODE = 0;
my $GENILIB = "$TB/lib/geni-lib";
my $JAILPROG = "$TB/libexec/genilib-jail";
my $REPODIR = "/repos";
my $TAR = "/usr/bin/tar";
my $debug = 0;
......@@ -119,6 +122,29 @@ if (defined($options{"J"})) {
$iocagepath = $options{"B"};
}
}
if (defined($options{"r"})) {
$repo = $options{"r"};
# Must taint check
if ($repo =~ /^([-\w]+)$/) {
$repo = $1;
}
else {
die("Bad data in argument: $repo");
}
if (! -e "$REPODIR/$repo") {
die("No such repo $repo\n");
}
if (defined($options{"h"})) {
$reporef = $options{"h"};
# Must taint check
if ($reporef =~ /^([-\w\/]+)$/) {
$reporef = $1;
}
else {
die("Bad data in argument: $reporef");
}
}
}
#
# First option has to be the -u option, the user to run this script as.
......@@ -232,6 +258,14 @@ if ($warningsfatal) {
my $exit_status;
if ($usejail) {
my $bopt = (defined($iocagepath) ? "-B $iocagepath" : "");
my $ropt = "";
if (defined($repo)) {
$ropt = "-r $repo";
if (defined($reporef)) {
$ropt .= " -h $reporef";
}
}
#
# We are executing the command in a jail, fire off the jail script.
......@@ -245,7 +279,7 @@ if ($usejail) {
# name space. Those copies will be owned by the user so they can be
# read/written.
#
$exit_status = system("$JAILPROG $bopt -u $user $ifile");
$exit_status = system("$JAILPROG $bopt $ropt -u $user $ifile");
#
# Now that we are done with the files, chown them to the user and
......
......@@ -1045,6 +1045,17 @@ function Do_RunScript()
if ($newgenilib) {
$command .= " -N ";
}
if ($profile && $profile->repourl()) {
$command .= " -r " . $profile->reponame();
if (isset($ajax_args["refspec"])) {
if (!preg_match('/^[-\w\/]+$/', $ajax_args["refspec"])) {
SPITAJAX_ERROR(1, "Invalid refspec");
return;
}
$command .= " -h " . escapeshellarg($ajax_args["refspec"]);
}
}
}
elseif (preg_match("/^source tb_compat/m", $script)) {
$command = "webns2rspec -a";
......
......@@ -905,10 +905,14 @@ $(function ()
return;
}
};
var args = {"uuid" : uuid};
// Another repo based profile thing.
if (window.REFSPEC !== undefined) {
args["refspec"] = window.REFSPEC;
}
$("#waitwait-modal").modal('show');
var xmlthing = sup.CallServerMethod(null, "instantiate",
"RunScript",
{"uuid" : uuid});
"RunScript", args);
xmlthing.done(callback);
};
......@@ -1717,11 +1721,11 @@ $(function ()
alert("Could not get profile: " + json.value);
return;
}
//console.info(json);
console.info("GetProfile:", json);
var xmlDoc = $.parseXML(json.value.rspec);
var xml = $(xmlDoc);
console.log(json);
/*
* We now use the desciption from inside the rspec, unless there
* is none, in which case look to see if the we got one in the
......@@ -1762,7 +1766,8 @@ $(function ()
json.value.repohash = hash;
if (pythonRe.test(source)) {
ConvertScript(source, profile, function(rspec, paramdefs){
ConvertScript(source, profile, which,
function(rspec, paramdefs) {
// Need to pass these along at submit.
$('#rspec_textarea').val(rspec);
$('#script_textarea').val(source);
......@@ -1801,7 +1806,7 @@ $(function ()
// We use this on repo-based profiles, where we have to get the
// source code from the repo, and convert to an rspec.