All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

Commit 5e304942 authored by Leigh B Stoller's avatar Leigh B Stoller

Remove use of freenodes.csv download from clusters. Instead, use the

data collected by the portal_monitor (ListResources), which now updates
the apt_aggregate_nodetypes type with the count/free info for each of
the types found in the advertisement.
parent 6db95f98
......@@ -144,6 +144,17 @@ sub Refresh($)
$self->{'AGGREGATE'} = $query_result->fetchrow_hashref();
#
# Grab new status row.
#
$query_result =
DBQueryWarn("select * from apt_aggregate_status where urn=$safe_urn");
return -1
if (!$query_result);
if ($query_result->numrows) {
$self->{'STATUS'} = $query_result->fetchrow_hashref();
}
return 0;
}
......
......@@ -336,62 +336,44 @@ sub ReportLockdownExpired()
}
#
# Gather up aggregate info. At the moment, there is still a lot of
# Utah specific goings on here.
# Update the json files that are used to build the fancy cluster graphs.
#
sub UpdateAggregateGraphs()
{
my $freeblob = {};
my $query_result =
DBQueryWarn("select * from apt_aggregates where noupdate=0");
DBQueryWarn("select urn from apt_aggregates where noupdate=0");
return 0
if (!$query_result->numrows);
while (my $row = $query_result->fetchrow_hashref()) {
my $urn = $row->{'urn'};
my $url = $row->{'weburl'} . "/node_usage/freenodes.csv";
my $name = $row->{'nickname'};
system("$WGET --no-check-certificate ".
"--timeout=30 --waitretry=30 --retry-connrefused ".
"-q -O /tmp/freenodes.csv '$url'");
next
if ($?);
if (-z "/tmp/freenodes.csv") {
print STDERR "Zero length csv file for $urn\n";
next;
}
if (!open(CSV, "/tmp/freenodes.csv")) {
print STDERR "Could not open csv file for $urn\n";
while (my ($urn) = $query_result->fetchrow_array()) {
my $aggregate = APT_Aggregate->Lookup($urn);
if (!defined($aggregate)) {
print STDERR "Could not lookup aggregate: $urn\n";
next;
}
my $line = <CSV>;
if ($line !~ /^Type,Inuse,Free$/) {
print STDERR "Invalid first line in csv file for $urn\n";
close(CSV);
next;
}
while (<CSV>) {
if ($_ =~ /^([-\w]+),(\d+),(\d+)$/) {
my $count = $2 + $3;
$freeblob->{$name}->{$1} = {"inuse" => $2, "free" => $3};
my $name = $aggregate->nickname();
if ($impotent) {
print "$urn: $1, $count, $3\n";
}
else {
DBQueryWarn("replace into apt_aggregate_nodetypes set ".
" urn='$urn',type='$1',".
" count='$count',free='$3'");
}
}
#
# Load the nodetype info that the portal_monitor gathers.
#
my $types_result =
DBQueryWarn("select type,count,free from apt_aggregate_nodetypes ".
"where urn='$urn'");
next
if (!$types_result || !$types_result->numrows());
$freeblob->{$name} = {};
while (my ($type,$count,$free) = $types_result->fetchrow_array()) {
$freeblob->{$name}->{$type} =
{"count" => $count, "free" => $free};
}
close(CSV);
}
#print Dumper($freeblob);
if ($debug && $oneshot) {
print Dumper($freeblob);
}
#
# We are currently operating from two master json files. We make a
......@@ -427,8 +409,8 @@ sub UpdateAggregateGraphs()
print STDERR "$name: Did not get a count for $type\n";
next;
}
my $inuse = $freeblob->{$name}->{$type}->{'inuse'};
my $size = $freeblob->{$name}->{$type}->{'free'} + $inuse;
my $size = $freeblob->{$name}->{$type}->{'count'};
my $inuse = $size - $freeblob->{$name}->{$type}->{'free'};
$child->{'howfull'} = int($inuse);
$child->{'size'} = int($size);
......@@ -439,7 +421,12 @@ sub UpdateAggregateGraphs()
$cluster->{'howfull'} = int($total_size);
$cluster->{'size'} = int($total_size);
}
#print Dumper($obj);
if ($debug || $oneshot) {
print Dumper($obj);
}
next
if ($impotent);
#
# Write out new file for web ui.
#
......@@ -646,8 +633,7 @@ sub KillStaleDatasets()
}
if ($oneshot) {
KillInstances();
KillStaleDatasets();
UpdateAggregateGraphs();
exit(0);
}
......
......@@ -187,23 +187,33 @@ sub CheckAggregates()
my $markError = sub($$) {
my ($aggregate, $error) = @_;
$aggregate->last_error($error);
print STDERR $error . "\n";
#
# Decide if aggregate should be marked as down.
#
my $status;
if (!defined($aggregate->last_contact())) {
$aggregate->status("down");
return;
$status = "down";
}
my $last_attempt = str2time($aggregate->last_attempt());
my $last_contact = str2time($aggregate->last_contact());
else {
my $last_attempt = str2time($aggregate->last_attempt());
my $last_contact = str2time($aggregate->last_contact());
if ($last_attempt - $last_contact < 600) {
$aggregate->status("unknown");
if ($last_attempt - $last_contact < 600) {
$status = "unknown";
}
else {
$status = "down";
}
}
if ($impotent) {
print STDERR "Would mark $aggregate as $status\n";
}
else {
$aggregate->status("down");
$aggregate->status($status);
$aggregate->last_error($error);
}
};
......@@ -231,7 +241,8 @@ sub CheckAggregates()
#
# Mark as trying to contact.
#
$aggregate->last_attempt(time());
$aggregate->last_attempt(time())
if (!$impotent);
#
# Do a quick test to see if we can even get there.
......@@ -289,13 +300,20 @@ sub CheckAggregates()
# Mark that we could get the advertisement. Also mark it as up.
# Not sure about when to mark it down though.
#
$aggregate->last_success($aggregate->last_attempt());
$aggregate->status("up");
$aggregate->last_error("");
if ($impotent) {
print "Would mark $aggregate as up\n";
}
else {
$aggregate->last_success($aggregate->last_attempt());
$aggregate->status("up");
$aggregate->last_error("");
}
my $pcount = 0;
my $pavail = 0;
my $vcount = 0;
my %type_count = ();
my %type_avail = ();
foreach my $ref (GeniXML::FindNodes("n:node",
$manifest)->get_nodelist()) {
......@@ -311,6 +329,38 @@ sub CheckAggregates()
$pcount++;
$pavail++
if (GeniXML::IsAvailable($ref));
foreach my $htype (FindNodes("n:hardware_type",
$ref)->get_nodelist()) {
my $hname = GeniXML::GetText("name", $htype);
next
if (!defined($hname) || $hname eq "" ||
$hname eq "pcvm" || $hname eq "pc" ||
$hname =~ /^delay/ ||
# Protect DB.
$hname !~ /^[-\w]+$/);
my $ntype =
GeniXML::FindNodesNS("n:node_type",
$htype,
$GeniXML::EMULAB_NS)->pop();
next
if (!$ntype);
my $slots = GeniXML::GetText("type_slots", $ntype);
next
if (!defined($slots) || $slots !~ /^\d+$/);
next
if ($slots > 1);
if (!exists($type_count{$hname})) {
$type_count{$hname} = 0;
$type_avail{$hname} = 0;
}
$type_count{$hname} += 1;
$type_avail{$hname} += 1
if (GeniXML::IsAvailable($ref));
}
}
elsif ($name eq "emulab-xen") {
my $exclusive = GeniXML::GetExclusive($ref);
......@@ -352,9 +402,23 @@ sub CheckAggregates()
}
}
print "$nickname: pcount:$pcount, pfree:$pavail, vcount:$vcount\n";
$aggregate->pcount($pcount);
$aggregate->pfree($pavail);
$aggregate->vcount($vcount);
foreach my $type (keys(%type_count)) {
my $count = $type_count{$type};
my $avail = $type_avail{$type};
if ($debug || $impotent) {
print "$type $count:$avail\n";
}
if (!$impotent) {
DBQueryWarn("replace into apt_aggregate_nodetypes set ".
" urn='$urn',type='$type',".
" count='$count',free='$avail'");
}
}
if (!$impotent) {
$aggregate->pcount($pcount);
$aggregate->pfree($pavail);
$aggregate->vcount($vcount);
}
}
return 0;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment