Commit 91bf8a29 authored by Leigh Stoller's avatar Leigh Stoller

At Mike's request, refresh all datasets once a day to pick up changes in

state (say, from valid to grace). Also, change the refresh interval for
expired datasets to hourly instead of every five minues.
parent 64a35317
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2008-2018 University of Utah and the Flux Group. # Copyright (c) 2008-2019 University of Utah and the Flux Group.
# #
# {{{GENIPUBLIC-LICENSE # {{{GENIPUBLIC-LICENSE
# #
...@@ -63,6 +63,7 @@ my $SUDO = "/usr/local/bin/sudo"; ...@@ -63,6 +63,7 @@ my $SUDO = "/usr/local/bin/sudo";
my $WGET = "/usr/local/bin/wget"; my $WGET = "/usr/local/bin/wget";
my $SLEEP_INTERVAL = 300; my $SLEEP_INTERVAL = 300;
my $DAILY_INTERVAL = 24 * 3600; my $DAILY_INTERVAL = 24 * 3600;
my $HOURLY_INTERVAL = 3600;
my $OPENSTACK_INTERVAL = 600; my $OPENSTACK_INTERVAL = 600;
# un-taint path # un-taint path
...@@ -571,16 +572,19 @@ sub GatherOpenstackUtilization() ...@@ -571,16 +572,19 @@ sub GatherOpenstackUtilization()
} }
# #
# Kill off stale datasets (that were deleted at the cluster). They are # Refresh/Kill datasets. Once a day we check all datasets. Hourly we
# gone, no point in keeping them around. # check only expired datasets.
# #
sub KillStaleDatasets() sub RefreshDatasets($)
{ {
my ($doall) = @_;
my $query_result = my $query_result =
DBQueryWarn("select uuid,expires from apt_datasets as d ". DBQueryWarn("select uuid,expires from apt_datasets as d ".
"where (type='stdataset' or type='ltdataset') and ". "where (type='stdataset' or type='ltdataset') ".
" (UNIX_TIMESTAMP(now()) > ". ($doall ? "" :
" UNIX_TIMESTAMP(expires))"); " and (UNIX_TIMESTAMP(now()) > ".
" UNIX_TIMESTAMP(expires))"));
return return
if (!$query_result); if (!$query_result);
...@@ -590,11 +594,18 @@ sub KillStaleDatasets() ...@@ -590,11 +594,18 @@ sub KillStaleDatasets()
print STDERR "No such dataset $uuid\n"; print STDERR "No such dataset $uuid\n";
next; next;
} }
my $pid = $dataset->pid(); my $pid = $dataset->pid();
my $id = $dataset->dataset_id(); my $id = $dataset->dataset_id();
my $agg = $dataset->aggregate_urn(); my $agg = $dataset->aggregate_urn();
print STDERR "Dataset $pid/$id at $agg expired at $expires. ". my $state = $dataset->state();
"Asking for new info ...\n";
if ($dataset->IsExpired()) {
print STDERR "Dataset $pid/$id at $agg expired at $expires.\n";
}
if ($impotent) {
print STDERR "Would try to refresh $dataset (state:$state)\n";
next;
}
# #
# Try to refresh the dataset. We might get back a new expiration, # Try to refresh the dataset. We might get back a new expiration,
...@@ -608,12 +619,19 @@ sub KillStaleDatasets() ...@@ -608,12 +619,19 @@ sub KillStaleDatasets()
# #
if (!$?) { if (!$?) {
$dataset->Refresh(); $dataset->Refresh();
if ($dataset->IsExpired()) { if (!$doall) {
# Still expired, not sure what to do here. if ($dataset->IsExpired()) {
print STDERR "Dataset is still expired after refresh?\n"; # Still expired, not sure what to do here.
print STDERR "Dataset is still expired after refresh?\n";
}
else {
print STDERR "Dataset expires at ".
$dataset->expires() . "\n";
}
} }
else { if ($state ne $dataset->state()) {
print STDERR "Dataset is no longer expired after refresh.\n"; print STDERR "State changed from $state to ".
$dataset->state() . "\n";
} }
next; next;
} }
...@@ -638,32 +656,25 @@ sub KillStaleDatasets() ...@@ -638,32 +656,25 @@ sub KillStaleDatasets()
} }
if ($oneshot) { if ($oneshot) {
KillInstances(); RefreshDatasets(0);
KillStaleDatasets();
exit(0); exit(0);
} }
# Wait a bit before making a bunch of noise.
sleep($SLEEP_INTERVAL);
my $reportcounter = 0; my $reportcounter = 0;
my $openstackcounter = 0; my $lastdaily = 0;
my $lasthourly = 0;
# Do this once at startup
if (! NoLogins()) {
ReportLockdownExpired();
}
while (1) { while (1) {
if (NoLogins()) { if (NoLogins()) {
sleep(5); sleep(5);
next; next;
} }
$reportcounter += $SLEEP_INTERVAL;
$openstackcounter += $SLEEP_INTERVAL;
print "Running at ". print "Running at ".
POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n"; POSIX::strftime("20%y-%m-%d %H:%M:%S", localtime()) . "\n";
KillInstances(); KillInstances();
KillStaleDatasets();
FixFailedImaging(); FixFailedImaging();
ExpireInstances(); ExpireInstances();
if ($MAINSITE) { if ($MAINSITE) {
...@@ -671,13 +682,19 @@ while (1) { ...@@ -671,13 +682,19 @@ while (1) {
} }
PushUpdates(); PushUpdates();
if ($reportcounter >= $DAILY_INTERVAL) { if (time() - $lastdaily > $DAILY_INTERVAL) {
ReportLockdownExpired(); ReportLockdownExpired();
$reportcounter = 0; RefreshDatasets(1);
$lastdaily = time();
}
if (time() - $lasthourly > $HOURLY_INTERVAL) {
RefreshDatasets(0);
$lasthourly = time();
} }
exit(0) exit(0)
if ($oneshot); if ($oneshot);
print "Waiting for $SLEEP_INTERVAL seconds ...\n";
sleep($SLEEP_INTERVAL); sleep($SLEEP_INTERVAL);
} }
exit(0); exit(0);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment