All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit 17a09142 authored by Elijah Grubb's avatar Elijah Grubb

Merge remote-tracking branch 'origin' into docker-runit-home

parents c5e31fe3 66366489
......@@ -811,7 +811,8 @@ sub ComputeNodeCounts($)
if (defined($virtualization_type) &&
($virtualization_type eq "emulab-xen" ||
$virtualization_type eq "emulab-blockstore")) {
$virtualization_type eq "emulab-blockstore" ||
$virtualization_type eq "emulab-docker")) {
$vcount++;
next;
}
......
......@@ -860,8 +860,9 @@ sub CheckFirewall($$)
# and closed.
#
my $style = "closed";
if (defined($virtualization_type) &&
$virtualization_type eq "emulab-xen" && !@routable_control_ip) {
if (defined($virtualization_type) && !@routable_control_ip &&
($virtualization_type eq "emulab-xen"
|| $virtualization_type eq "emulab-docker")) {
$style = "basic";
}
......
#!/usr/bin/perl -wT
#
# Copyright (c) 2007-2017 University of Utah and the Flux Group.
# Copyright (c) 2007-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -831,6 +831,8 @@ sub new($$$$$)
"jacks_site" => undef,
"xen_settings" => undef,
"xen_ptype" => undef,
"docker_settings" => undef,
"docker_ptype" => undef,
"instantiate_on" => undef,
"services" => [],
"statements" => [],
......@@ -1101,6 +1103,18 @@ sub addNode($$$)
$node->{"xen_settings"} = $settings;
last SWITCH;
};
# Docker settings
/^docker$/i && do {
my $settings = GeniXML::GetDockerSettings($ref->parentNode);
fatal("Failed to get docker settings")
if (!defined($settings));
$node->{"docker_settings"} = $settings;
last SWITCH;
};
/^docker_ptype$/i && do {
$node->{"docker_ptype"} = GetTextOrFail("name", $child);
last SWITCH;
};
/^(sliver_type_shaping|firewall_config)$/i && do {
# We handled this above.
last SWITCH;
......@@ -1291,7 +1305,7 @@ sub Compare($$)
last SWITCH;
};
(/^(component_id|component_manager_id|disk_image)$/i ||
/^(hardware_type|jacks_site|xen_ptype|instantiate_on)$/i ||
/^(hardware_type|jacks_site|xen_ptype|docker_ptype|instantiate_on)$/i ||
/^(adb_target|failure_action)$/i ||
/^(isexptfirewall|firewall_style)$/i ||
/^(use_type_default_image|routable_control_ip)$/i) && do {
......@@ -1304,7 +1318,7 @@ sub Compare($$)
# Handled up above in CompareNodes.
last SWITCH;
};
/^(xen_settings|desires|pipes|blockstores|attributes)$/i && do {
/^(xen_settings|docker_settings|desires|pipes|blockstores|attributes)$/i && do {
return 1
if (APT_Rspec::CompareHashes("Node: $client_id: $key",
$val1, $val2));
......
......@@ -29,8 +29,10 @@ package APT_Utility;
use strict;
use English;
use Data::Dumper;
use Date::Parse;
use Carp;
use Exporter;
use POSIX qw(ceil);
use vars qw(@ISA @EXPORT);
@ISA = "Exporter";
......@@ -44,6 +46,7 @@ use Project;
use Group;
use GeniHRN;
use GeniUser;
use emutil;
# Configure variables
my $TB = "@prefix@";
......@@ -126,3 +129,88 @@ sub MapUserURN($)
return undef;
}
#
# Given a reservation details hash, calculate a utilization number
# from the history array.
#
sub ReservationUtilization($$)
{
my ($res, $active) = @_;
my $count = $res->{'nodes'};
my $resstart = str2time($res->{'start'});
my $resend = ($active ? time() :
(str2time(defined($res->{'deleted'}) ?
$res->{'deleted'} : $res->{'end'})));
my $reshours = (($resend - $resstart) / 3600) * $count;
my $usedhours = 0;
my $inuse = 0;
my $laststamp = $resstart;
my $type = $res->{'type'};
my $remote_uid = $res->{'remote_uid'};
my @tmp = @{$res->{'history'}};
# Init for web interface.
$res->{'reshours'} = $reshours;
$res->{'usedhours'} = undef;
$res->{'utilization'} = undef;
#
# Scan past timeline entries that are *before* the start of the
# reservation; these are experiments that were running when the
# reservation started, and provide the number of nodes allocated
# at the time the reservation starts.
#
while (@tmp) {
my $ref = $tmp[0];
my $stamp = $ref->{'t'};
my $allocated = $ref->{'allocated'};
$ref->{'tt'} = TBDateStringGMT($ref->{'t'});
last
if ($stamp >= $resstart);
# Watch for nothing allocated by the user at this time stamp
my $using = 0;
if (exists($allocated->{$remote_uid})) {
$using = $allocated->{$remote_uid}->{$type};
}
$inuse = $using;
$inuse = $count if ($inuse > $count);
shift(@tmp);
}
foreach my $ref (@tmp) {
my $stamp = $ref->{'t'};
my $reserved = $ref->{'reserved'};
my $allocated = $ref->{'allocated'};
$ref->{'tt'} = TBDateStringGMT($ref->{'t'});
# If this stamp is after the reservation, we can stop. The
# last entry will be the current number of nodes used till
# the end of the reservation. This entry is typically for the
# end of an experiment start before the end of the reservation.
last
if ($stamp > $resend);
# Watch for nothing allocated by the user at this time stamp
my $using = 0;
if (exists($allocated->{$remote_uid})) {
$using = $allocated->{$remote_uid}->{$type};
}
$usedhours += (($stamp - $laststamp) / 3600) * $inuse;
$laststamp = $stamp;
$inuse = $using;
$inuse = $count if ($inuse > $count);
}
# And then a final entry for usage until the end of the reservation.
if ($laststamp) {
$usedhours += (($resend - $laststamp) / 3600) * $inuse;
}
$res->{'reshours'} = $reshours;
$res->{'usedhours'} = $usedhours;
$res->{'utilization'} = POSIX::ceil(($usedhours/$reshours) * 100.0);
$res->{'utilization'} = 100 if ($res->{'utilization'} > 100);
return 0;
}
......@@ -2112,19 +2112,24 @@ sub DoMaxExtensionInternal($$)
#
$res->{'cluster'} = $aptagg->urn();
$res->{'cluster_id'} = $aptagg->nickname();
# Backwards compat
if (!exists($res->{'nodes'})) {
$res->{'nodes'} = $res->{'count'};
}
if ($res->{'approved'} eq "") {
# Maps to JSON NULL.
$res->{'approved'} = undef;
}
if (!exists($res->{'cancel'}) || $res->{'cancel'} eq "") {
# Maps to JSON NULL.
$res->{'cancel'} = undef;
# We need numbers.
$res->{'nodes'} = int($res->{'nodes'});
$res->{'using'} = int($res->{'using'});
#
# Hmm, undef/null is a pain with XMLRPC.
#
foreach my $key ("approved", "cancel", "deleted") {
if (!exists($res->{$key}) || $res->{$key} eq "") {
$res->{$key} = undef;
}
}
#
# Lets calculate a utilization numbers, this will update the
# res hash. Flag says not an active reservation.
#
APT_Utility::ReservationUtilization($res, 0);
}
$blob->{'reservations'}->{$aptagg->urn()} = $reslist;
}
......
......@@ -28,7 +28,7 @@ use XML::Simple;
use Data::Dumper;
use CGI;
use POSIX ":sys_wait_h";
use POSIX qw(:signal_h);
use POSIX qw(:signal_h ceil);
use Date::Parse;
#
......@@ -516,6 +516,10 @@ sub DoList()
$details->{'cluster_id'} = $aggregate->nickname();
$details->{'cluster_urn'} = $aggregate->urn();
# We need numbers.
$details->{'nodes'} = int($details->{'nodes'});
$details->{'using'} = int($details->{'using'});
# Backwards compat
if (!exists($details->{'nodes'})) {
$details->{'nodes'} = $details->{'count'};
......@@ -524,18 +528,19 @@ sub DoList()
#
# Hmm, undef/null is a pain with XMLRPC.
#
if ($details->{'approved'} eq "") {
# Maps to JSON NULL.
$details->{'approved'} = undef;
}
if ($details->{'cancel'} eq "") {
# Maps to JSON NULL.
$details->{'cancel'} = undef;
}
if (!exists($details->{'uuid'}) || $details->{'uuid'} eq "") {
$details->{'uuid'} = NewUUID();
foreach my $key ("approved", "cancel", "deleted") {
if (!exists($details->{$key}) || $details->{$key} eq "") {
$details->{$key} = undef;
}
}
#
# Lets calculate a utilization numbers, this will update the
# res hash. Flag says not an active reservation.
#
APT_Utility::ReservationUtilization($details, 1);
#
# If we have the history, then go through and map the
# experiments to local experiments so we can link to them in
......@@ -1141,6 +1146,10 @@ sub DoHistory()
$res->{'cluster_id'} = $aggregate->nickname();
$res->{'cluster_urn'} = $aggregate->urn();
# We need numbers.
$res->{'nodes'} = int($res->{'nodes'});
$res->{'using'} = int($res->{'using'});
# Backwards compat
if (!exists($res->{'nodes'})) {
$res->{'nodes'} = $res->{'count'};
......@@ -1148,22 +1157,17 @@ sub DoHistory()
#
# Hmm, undef/null is a pain with XMLRPC.
#
if (!exists($res->{'approved'}) ||
$res->{'approved'} eq "") {
$res->{'approved'} = undef;
}
if (!exists($res->{'cancel'}) ||
$res->{'cancel'} eq "") {
$res->{'cancel'} = undef;
}
if (!exists($res->{'deleted'}) ||
$res->{'deleted'} eq "") {
$res->{'deleted'} = undef;
}
if (!exists($res->{'uuid'}) ||
$res->{'uuid'} eq "") {
$res->{'uuid'} = NewUUID();
foreach my $key ("approved", "cancel", "deleted") {
if (!exists($res->{$key}) || $res->{$key} eq "") {
$res->{$key} = undef;
}
}
#
# Lets calculate a utilization numbers, this will update the
# res hash. Flag says not an active reservation.
#
APT_Utility::ReservationUtilization($res, 0);
}
done:
if (defined($webtask)) {
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2017 University of Utah and the Flux Group.
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -201,6 +201,45 @@ sub GenerateNodeStatements($)
$node->addTagStatement("InstantiateOn('$vhost')");
}
}
elsif ($ntype eq "emulab-docker") {
$node->addStatement("$ntag = request.DockerContainer('$client_id')");
#
# This is the only time we need to spit this out, since
# the default is False.
#
if (defined($node->{'exclusive'}) && $node->{'exclusive'}) {
$node->addTagStatement("exclusive = True");
}
if (defined($node->{'docker_settings'})) {
my $settings = $node->{'docker_settings'};
foreach my $setting (sort(keys(%{$settings}))) {
my $value = $settings->{$setting};
if ($setting eq "ram") {
$node->addTagStatement("ram = $value");
}
elsif ($setting eq "cores") {
$node->addTagStatement("cores = $value");
}
elsif ($setting eq "extimage") {
$node->addTagStatement("docker_extimage = $value");
}
elsif ($setting eq "dockerfile") {
$node->addTagStatement("docker_dockerfile = $value");
}
}
}
if (defined($node->{'docker_ptype'})) {
my $ptype = $node->{'docker_ptype'};
$node->addTagStatement("docker_ptype = '$ptype'");
}
if (defined($node->{'instantiate_on'})) {
my $vhost = $node->{'instantiate_on'};
$node->addTagStatement("InstantiateOn('$vhost')");
}
}
elsif ($ntype eq "delay") {
#
# Bridges are also special, see comment above for blockstore.
......
......@@ -174,6 +174,7 @@ my %xmlfields =
"global", => ["global", $SLOT_ADMINONLY, 0],
"mbr_version", => ["mbr_version", $SLOT_OPTIONAL],
"makedefault", => ["makedefault", $SLOT_ADMINONLY, 0],
"format", => ["format", $SLOT_ADMINONLY, "ndz"],
);
#
......@@ -490,14 +491,27 @@ elsif (! $isadmin) {
UserError("Path: Invalid Path");
}
}
if ($newimageid_args{"path"} =~ /\/$/) {
if (-e $newimageid_args{"path"} && ! -d $newimageid_args{"path"}) {
UserError("Path: invalid path, it should be a directory");
}
if (defined($newimageid_args{"format"})
&& $newimageid_args{"format"} eq "docker") {
#
# We only allow a specific path for docker images, since the storage
# backend relies on an ACL of repos a user can access. It's not a
# filesystem with UNIX permissions. With a docker registry,
# permissions are tied to specific paths. Don't even let admins
# override this for now; there is no point.
#
}
elsif (-d $newimageid_args{"path"} =~ /\/$/) {
UserError("Path: invalid path, its a directory");
else {
if ($newimageid_args{"path"} =~ /\/$/) {
if (-e $newimageid_args{"path"} && ! -d $newimageid_args{"path"}) {
UserError("Path: invalid path, it should be a directory");
}
}
elsif (-d $newimageid_args{"path"} =~ /\/$/) {
UserError("Path: invalid path, its a directory");
}
}
#
# See what node types this image will work on. Must be at least one!
#
......
......@@ -210,6 +210,7 @@ my %xmlfields =
"global", => ["global", $SLOT_OPTIONAL, 0],
"noexport", => ["noexport", $SLOT_OPTIONAL, 0],
"mbr_version", => ["mbr_version", $SLOT_OPTIONAL],
"format", => ["format", $SLOT_OPTIONAL],
"metadata_url", => ["metadata_url", $SLOT_ADMINONLY],
"imagefile_url", => ["imagefile_url", $SLOT_ADMINONLY],
"origin_uuid", => ["origin_uuid", $SLOT_ADMINONLY],
......
......@@ -516,7 +516,10 @@ docker-install: dir-install
$(BINDIR)/run/rcmanifest.d
$(INSTALL) -m 755 $(SRCDIR)/docker/create-docker-image $(LBINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/docker/emulabize-image $(LBINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/docker/analyze-image $(LBINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/docker/analyze-image-with-busybox $(LBINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/vnodectl $(BINDIR)/
$(INSTALL) -m 755 $(SRCDIR)/docker/cleanup-docker-vnodes.sh $(BINDIR)/
# Note that we install this, but do not enable it. It is dynamically
# started and stopped as needed by libvnode_docker.pm .
$(INSTALL) -m 644 -o root -g $(DIRGROUP) \
......
#!/usr/bin/perl
use strict;
use warnings;
use Getopt::Std;
use English;
use POSIX;
use Data::Dumper;
#
# A simple CLI wrapper around libvnode_docker::analyzeImage.
#
sub usage()
{
print "Usage: analyze-image [-d <level>] [-f] <image>\n".
"\n".
" -d <level> Debug mode\n".
" -f Analyze the image even if it has already been done\n";
exit(1);
}
my $optlist = "hd:f";
my $debug = 0;
my $force = 0;
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (!getopts($optlist,\%options)) {
usage();
}
if (defined($options{"h"})) {
usage();
}
if (defined($options{"d"})) {
$debug = $options{"d"};
}
if (defined($options{"f"})) {
$force = 1;
}
usage()
if (@ARGV > 1);
#
# Must be root.
#
if ($UID != 0) {
die("*** $0:\n".
" Must be root to run this script!\n");
}
#
# Turn off line buffering on output
#
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
use libsetup;
use libvnode_docker;
if ($debug) {
TBDebugTimeStampsOn();
libvnode_docker::setDebug($debug);
}
my %rethash = ();
my $image = $ARGV[0];
my $rc = libvnode_docker::analyzeImage($image,\%rethash,$force);
if ($rc) {
fatal("ERROR: failed to analyze $image!\n");
}
else {
print "Successfully analyzed $image:\n";
for my $k (keys(%rethash)) {
my $v = '';
if (defined($rethash{$k})) {
$v = $rethash{$k};
}
print "$k=$v\n";
}
}
exit(0);
#!/usr/bin/perl
use strict;
use warnings;
use Getopt::Std;
use English;
use POSIX;
use Data::Dumper;
use JSON::PP;
#
# A simple CLI wrapper around libvnode_docker::analyzeImageWithBusyboxCommand.
#
sub usage()
{
print "Usage: analyze-image-with-busybox [-d <level>] [-c] <image> [command... ]\n".
"\n".
" -d <level> Debug mode\n".
" -c <json> Customize the temporary container config according".
" to JSON passed to the Docker run API command.\n";
exit(1);
}
my $optlist = "hd:c:";
my $debug = 0;
my $config = {};
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (!getopts($optlist,\%options)) {
usage();
}
if (defined($options{"h"})) {
usage();
}
if (defined($options{"d"})) {
$debug = $options{"d"};
}
if (defined($options{"c"})) {
$config = decode_json($options{"c"});
}
usage()
if (@ARGV < 2);
#
# Must be root.
#
if ($UID != 0) {
die("*** $0:\n".
" Must be root to run this script!\n");
}
#
# Turn off line buffering on output
#
$| = 1;
# Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
use libsetup;
use libvnode_docker;
if ($debug) {
TBDebugTimeStampsOn();
libvnode_docker::setDebug($debug);
}
my $ret;
my $image = shift(@ARGV);
my $rc = libvnode_docker::analyzeImageWithBusyboxCommand(
$image,$config,\$ret,@ARGV);
if ($rc) {
fatal("ERROR: failed to analyze $image with busybox!\n");
}
else {
print "Successfully analyzed $image with busybox:\n$ret";
}
exit(0);
#!/bin/sh
#
# Cleans up *everything* related to the specified vnode, or to all
# vnodes if -a is specified instead of a vnode.
#
usage() {
if [ -n "$1" ]; then
echo "ERROR: $1"
fi
echo "USAGE: $0 [-h] [-a | vnode... ]"
echo " -a Remove all vnodes"
echo " vnode... A space-separated list of vnodes to remove"
exit 1
}
cleanupvnode() {
vnode=$1