Commit 11d792e3 authored by Leigh B. Stoller's avatar Leigh B. Stoller

Change the prerender code to run in the background so that Mike does

not have to wait 3 minutes for it to finish before he can watch his
experiment swapin fail for some other reason.

I adopted the same pid mechanism as in eventsys_control.in, which uses
a slot in the experiments table.

Running "prerender" puts the render into the background and stores
the pid. Running "prerender -r" kills a running prerender and removes
the existing info from the DB.

Fixed the problem with swapmod not restoring the old vis; swapmod now
kills any running prerender, and restarts one if the swapmod fails
(the prerun of the new NS file starts up another prerender in the
background).

Add setpriority() call in prerender to nice it and children to 15.
parent d44a0a58
......@@ -263,6 +263,7 @@ CREATE TABLE experiments (
canceled tinyint(4) NOT NULL default '0',
batchstate varchar(12) default NULL,
event_sched_pid int(11) default '0',
prerender_pid int(11) default '0',
uselinkdelays tinyint(4) NOT NULL default '0',
forcelinkdelays tinyint(4) NOT NULL default '0',
multiplex_factor smallint(5) default NULL,
......
......@@ -827,3 +827,10 @@ last_net_act,last_cpu_act,last_ext_act);
I contemplated putting this into the nodes table instead, but a
node has no role when its not reserved, so it seemed like the
wrong place for it.
1.169: Add a prerender pid so that it can run in the background, but
be killed if the experiment fails, terminates, or modifies.
alter table experiments add prerender_pid int(11) default '0' \
after event_sched_pid;
......@@ -59,7 +59,7 @@ my $action;
#
# Untaint the path
#
$ENV{'PATH'} = '/bin:/usr/bin';
$ENV{'PATH'} = "/bin:/usr/bin:$TB/libexec/vis";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
#
......@@ -470,6 +470,9 @@ elsif ($inout eq "modify") {
fatal("*** $0:\n".
" Could not backup experiment state; cannot safely continue!\n");
}
# Must deal with the prerender explicitly since it runs background.
system("prerender -r $pid $eid");
TBExptRemoveVirtualState($pid, $eid);
#
......@@ -512,8 +515,13 @@ elsif ($inout eq "modify") {
print STDERR "*** $0:\n".
" $modifyError\n";
print STDOUT "Recovering experiment state...\n";
# Must deal with the prerender explicitly since it runs background.
system("prerender -r $pid $eid");
TBExptRemoveVirtualState($pid, $eid );
if (0 == TBExptRestoreVirtualState($pid, $eid, $$)) {
# Must deal with the prerender explicitly since it runs background.
system("prerender -t $pid $eid");
fatal("*** Update aborted; old state restored.\n");
} else {
# Set state to NEW so experiment will get wiped.
......
......@@ -86,8 +86,9 @@ if (! SetExpState($pid, $eid, EXPTSTATE_TERMINATING)) {
print "Clearing out virtual state.\n";
$errors += TBExptRemoveVirtualState( $pid, $eid );
# This will kill a running prerender.
print "Removing visualization data...\n";
system("prerender -r -t $pid $eid");
system("prerender -r $pid $eid");
if ($errors == 0) {
SetExpState($pid, $eid, EXPTSTATE_TERMINATED) or
......
......@@ -107,6 +107,9 @@ if (! $modify) {
sub cleanup {
if (! $modify) {
print STDERR "Cleaning up after errors.\n";
# Must kill the prerender process first!
system("prerender -r $pid $eid");
print "Removing experiment state ... " . TBTimeStamp() . "\n";
TBExptRemoveVirtualState($pid, $eid );
print "Removal done! " . TBTimeStamp() . "\n";
......@@ -116,7 +119,6 @@ sub cleanup {
# going to "NEW" tells swapexp to fully terminate the experiment.
#
SetExpState($pid, $eid, EXPTSTATE_NEW);
system("prerender -r $pid $eid");
}
}
......@@ -133,8 +135,8 @@ TBDebugTimeStamp("parser finished");
print "Parser done! " . TBTimeStamp() . "\n";
TBDebugTimeStamp("prerender started in background");
print "Precomputing visualization (in background)...\n";
system("prerender -t $pid $eid &");
print "Precomputing visualization ...\n";
system("prerender -t $pid $eid");
TBDebugTimeStamp("routes started");
print "Setting up static routes (if requested) ... \n";
......
......@@ -2,7 +2,7 @@
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#
......@@ -162,7 +162,7 @@ if (defined($output)) {
$args .= "2> /dev/null";
}
if (system("$render $args") != 0) {
if (system("nice $render $args") != 0) {
exit(1);
}
......
......@@ -9,15 +9,18 @@
use English;
use Getopt::Std;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use BSD::Resource;
# Need this module to use mktemp (commented out below)
# use File::MkTemp;
# Configure variables
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
use lib '@prefix@/lib';
use libdb;
use libtestbed;
# Turn off line buffering on output
$| = 1;
......@@ -36,12 +39,13 @@ my $THUMB_CMD = "render -t 128";
sub dprint($);
sub usage {
die "Usage:\nprerender [-v] [-r] [-t] <pid> <eid>\n" .
die "Usage:\nprerender [-v] [-r] [-t] [-f] <pid> <eid>\n" .
" -f To stay in foreground (no fork).\n".
" -r Just remove vis info from DB\n".
" -t Generate (or remove) thumbnail\n";
}
my $optlist = "rvt";
my $optlist = "rvtf";
%options = ();
......@@ -50,6 +54,7 @@ if (@ARGV != 2) { usage; }
my $debug = 0;
my $thumb = 0;
my $foreground = 0;
if ( defined($options{"v"}) ) {
$debug++;
......@@ -60,6 +65,9 @@ if ( defined($options{"v"}) ) {
if ( defined($options{"t"}) ) {
$thumb++;
}
if ( defined($options{"f"}) ) {
$foreground++;
}
#my ($pid) = $ARGV[0];
#my ($eid) = $ARGV[1];
......@@ -69,6 +77,21 @@ my ($eid) = $ARGV[1] =~ /([0-9a-zA-Z_\-]+)/;
# for the mungeName function, below.
$mungeUID = 0;
# For process handling, below.
my $renderpid;
sub handler ($) {
my ($signame) = @_;
fatal("Caught a SIG${signame}!");
}
#
# See if another prerender is still running. If so, kill it and update the
# pid in the DB. We always kill a running prerender.
#
KillOldRender();
### If they specified -r, meaning they just wanted to remove vis info
### from the DB, we do it and quit.
if (defined($options{"r"})) {
......@@ -76,12 +99,46 @@ if (defined($options{"r"})) {
exit 0;
}
#
# Okay, go into the background.
#
if (! $foreground) {
if (TBBackGround("prerender.output")) {
#
# Parent exits normally
#
print STDOUT "Image rendering proceeding in background mode ...\n";
exit(0);
}
#
# Setup a handler so we can be killed later.
#
$SIG{TERM} = \&handler;
$SIG{INT} = \&handler;
#
# Enter our pid into the DB.
#
$renderpid = $PID;
DBQueryFatal("update experiments set ".
"prerender_pid=$renderpid ".
"where pid='$pid' and eid='$eid'");
}
# Lower priority since this whole thing consumes CPU like nuts
setpriority(PRIO_PROCESS, 0, 15);
### Now, read experiment virtual info from the DB.
# read nodes
my $result = DBQueryFatal("SELECT vname FROM virt_nodes " .
"WHERE pid='$pid' AND eid='$eid'");
my $result = DBQueryWarn("SELECT vname FROM virt_nodes " .
"WHERE pid='$pid' AND eid='$eid'");
fatal("Failed to get virt_nodes info from DB")
if (!$result);
my $nodes = ();
my $lans = ();
......@@ -94,9 +151,12 @@ while (my ($vname) = $result->fetchrow) {
# read lans
$result = DBQueryFatal("SELECT vname, member FROM virt_lans " .
$result = DBQueryWarn("SELECT vname, member FROM virt_lans " .
"WHERE pid='$pid' AND eid='$eid'");
fatal("Failed to get virt_lans info from DB")
if (!$result);
while (my ($vname, $member) = $result->fetchrow) {
$member =~ s/\:.*//;
$lans{$vname}{$member} = 1;
......@@ -112,7 +172,7 @@ foreach my $lan (keys %lans) {
$links{"$a $b"} = "pair";
} else {
# it's a lan; we need a visualizer node for it (of type "lan").
if (exists $nodes{$lan}) { die "LAN $lan name collided."; }
if (exists $nodes{$lan}) { fatal("LAN $lan name collided."); }
$nodes{$lan}{"type"} = "lan";
# make the appropriate links (from each member to the LAN)
......@@ -130,7 +190,8 @@ foreach my $lan (keys %lans) {
# Could use ($ENV{'TMP'} || "/tmp") but not sure if that is safe.
my $tempfile = "/tmp/prerender.$$-" . sprintf("%i",rand(65536));
open (TOPO, ">$tempfile" ) or die( "Couldn't open temporary file '$tempfile'\n" );
open (TOPO, ">$tempfile" ) or
fatal("Couldn't open temporary file '$tempfile'\n");
# write topology to neato
......@@ -140,22 +201,11 @@ foreach my $node (keys %nodes) {
print TOPO "{node [shape = box] " . mungeName($node) . "}\n";
}
#foreach my $a (keys %nodes) {
# foreach my $b (keys %nodes) {
# if (!exists $links{"$a $b"}) {
# print TOPO mungeName($a) . " -- " . mungeName($b) . " [len = 4];\n";
# print TOPO mungeName($a) . " -- " . mungeName($b) . " ;\n";
# }
# }
#}
foreach my $link (keys %links) {
my ($a, $b) = $link =~ /(\S+)\s(\S+)/;
if ($links{$link} eq "pair") {
# print TOPO mungeName($a) . " -- " . mungeName($b) . " [len = 3];\n";
print TOPO mungeName($a) . " -- " . mungeName($b) . " ;\n";
} else {
# print TOPO mungeName($a) . " -- " . mungeName($b) . " [len = 5];\n";
print TOPO mungeName($a) . " -- " . mungeName($b) . " ;\n";
}
}
......@@ -264,18 +314,18 @@ for ($attempt = 0; $attempt < 16; $attempt++) {
foreach my $linka (keys %links) {
my ($a1, $a2) = ($linka =~ /(\S+)\s(\S+)/);
defined $a1 or die;
defined $a2 or die;
exists $nodes{ $a1 } or die $a1;
exists $nodes{ $a2 } or die $a2;
defined $a1 or fatal("a1");
defined $a2 or fatal("a2");
exists $nodes{ $a1 } or fatal($a1);
exists $nodes{ $a2 } or fatal($a2);
my ($a1x,$a1y) = ($nodes{ $a1 }{"x"}, $nodes{ $a1 }{"y"});
my ($a2x,$a2y) = ($nodes{ $a2 }{"x"}, $nodes{ $a2 }{"y"});
defined $a1x or die "$linka $a1";
defined $a1y or die $a1;
defined $a2x or die $a2;
defined $a2y or die $a2;
defined $a1x or fatal("$linka $a1");
defined $a1y or fatal($a1);
defined $a2x or fatal($a2);
defined $a2y or fatal($a2);
foreach my $linkb (keys %links) {
# never check a link against itself
......@@ -283,16 +333,16 @@ for ($attempt = 0; $attempt < 16; $attempt++) {
my ($b1, $b2) = ($linkb =~ /(\S+)\s(\S+)/);
defined $b1 or die;
defined $b2 or die;
defined $b1 or fatal("b1");
defined $b2 or fatal("b2");
my ($b1x,$b1y) = ($nodes{ $b1 }{"x"}, $nodes{ $b1 }{"y"});
my ($b2x,$b2y) = ($nodes{ $b2 }{"x"}, $nodes{ $b2 }{"y"});
defined $b1x or die $b1;
defined $b1y or die $b1;
defined $b2x or die $b2;
defined $b2y or die $b2;
defined $b1x or fatal($b1);
defined $b1y or fatal($b1);
defined $b2x or fatal($b2);
defined $b2y or fatal($b2);
my $shared = "";
......@@ -371,57 +421,76 @@ for ($attempt = 0; $attempt < 16; $attempt++) {
dprint "Attempt = $attempt, overlaps = $overlaps.\n";
# if this is the best so far, stash it away.
#
# if this is the best so far, stash it in the DB, This gives us
# a picture quickly, even if its not the best picture. Also gives
# us something in case the experiment is terminated or modified before
# the render finishes, which is quite possible on giant experiments.
#
if (!defined $bestOverlaps || $overlaps < $bestOverlaps) {
$bestOverlaps = $overlaps;
# We have to lock the table so as not to cause grief to someone
# trying to vis this experiment during this window.
$result =
DBQueryWarn("lock tables vis_nodes write");
fatal("Failed to lock vis_nodes table.")
if (!$result);
$result =
DBQueryWarn("DELETE FROM vis_nodes ".
"WHERE pid='$pid' AND eid='$eid'");
fatal("Failed to clear old vis_nodes from DB")
if (!$result);
# Then stash new one.
foreach $node (keys %nodes) {
$nodes{ $node }{"x_best"} = $nodes{ $node }{"x"};
$nodes{ $node }{"y_best"} = $nodes{ $node }{"y"};
}
}
$result =
DBQueryWarn("INSERT INTO vis_nodes ".
"(vname, pid, eid, vis_type, x, y) VALUES".
" ('$node', '$pid', '$eid', " .
"'" . $nodes{ $node }{"type"} . "', " .
"'" . $nodes{ $node }{"x"} . "', " .
"'" . $nodes{ $node }{"y"} . "') ");
fatal("Failed to insert new vis_nodes into DB")
if (!$result);
}
$result =
DBQueryWarn("unlock tables");
fatal("Failed to unlock vis_nodes table.")
if (!$result);
#
# Generate thumbnail, if called for. The .png file is put into
# the work directory, so it is copied off to the users
# directory. We take that file and stash it in the DB, since
# that is where the system will read it from.
#
if ($thumb) {
my $workdir = TBExptWorkDir($pid, $eid);
my $fname = "$workdir/$eid.png";
if (system("$THUMB_CMD $pid $eid > $fname")) {
print STDERR "Error generating thumbnail.\n";
}
else {
TBExptSetThumbNail($pid, $eid, `cat $fname`);
}
}
}
# if there were no overlaps, we're done.
if ($overlaps == 0) { last; }
} # for $attempt
### Delete tempfile
unlink $tempfile;
### Put new data into db.
# First, wipe out the old layout.
DBQueryFatal("DELETE FROM vis_nodes WHERE pid='$pid' AND eid='$eid'");
foreach $node (keys %nodes) {
if (exists $nodes{ $node }{"type"} &&
exists $nodes{ $node }{"x_best"} &&
exists $nodes{ $node }{"y_best"}) {
DBQueryFatal("INSERT INTO vis_nodes (vname, pid, eid, vis_type, x, y) VALUES".
" ('$node', '$pid', '$eid', " .
"'" . $nodes{ $node }{"type"} . "', " .
"'" . $nodes{ $node }{"x_best"} . "', " .
"'" . $nodes{ $node }{"y_best"} . "') ");
}
}
# Generate thumbnail, if called for. The .png file is put into the work
# directory, so it is copied off to the users directory. We take that
# file and stash it in the DB, since that is where the system will read
# it from.
if ($thumb) {
my $workdir = TBExptWorkDir($pid, $eid);
my $fname = "$workdir/$eid.png";
if (system("nice $THUMB_CMD $pid $eid > $fname")) {
print STDERR "Error generating thumbnail.\n";
}
TBExptSetThumbNail($pid, $eid, `cat $fname`);
}
### Success!!
cleanup();
exit 0;
......@@ -448,3 +517,54 @@ sub dprint($) {
my $n = shift;
if ($debug > 0) { print STDERR $n; }
}
sub cleanup()
{
# Clear our pid from the DB since we are about to exit.
if ($renderpid) {
DBQueryFatal("update experiments set prerender_pid=0 ".
"where pid='$pid' and eid='$eid'");
}
}
sub fatal($)
{
my $msg = shift;
cleanup();
DBQueryFatal("DELETE FROM vis_nodes WHERE pid='$pid' AND eid='$eid'");
die("*** $0:\n".
" $msg\n");
}
sub KillOldRender()
{
DBQueryFatal("lock tables experiments write");
my $query_result =
DBQueryFatal("select prerender_pid from experiments ".
"where pid='$pid' and eid='$eid'");
DBQueryFatal("update experiments set ".
"prerender_pid=-prerender_pid ".
"where pid='$pid' and eid='$eid'");
DBQueryWarn("unlock tables");
my @row = $query_result->fetchrow_array();
my $procid = $row[0];
if ($procid > 0 &&
! kill('TERM', $procid)) {
DBQueryWarn("update experiments set ".
"prerender_pid=-prerender_pid ".
"where pid='$pid' and eid='$eid'");
SENDMAIL($TBOPS,
"Failed to stop prerender for $pid/$eid",
"Could not kill(TERM) process $procid: $? $!");
die("*** $0:\n".
"Failed to stop prerender for $pid/$eid!\n");
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment