Commit 6f0b2081 authored by David Johnson's avatar David Johnson

Allow rc.topomap/linktest to use gzip'd versions of ltmap/ltpmap files.

If $ETCDIR/ltmap-gzip exists on a clientside node, rc.topomap will only
download the gzip'd versions of ltmap/ltpmap into $BOOTDIR, and
linktest.pl will use them.  Those files are TMLTMAPGZ() and TMLTPMAPGZ().

This is important for multi-thousand node exps, where the lt*map files
easily grow to 250MB or more (and are compressible to 25:1 or so!);
saves CoW virtual disk blocks and raw disk space.  And now that commit
67cd8518 means nodetype no longer uses ltpmap, linktest is the only
consumer of lt*map files.
parent 2d10e5e1
#!/usr/bin/perl -w -T #!/usr/bin/perl -w -T
# #
# Copyright (c) 2000-2017 University of Utah and the Flux Group. # Copyright (c) 2000-2018 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -195,8 +195,6 @@ use constant FALSE => 0; ...@@ -195,8 +195,6 @@ use constant FALSE => 0;
# Globals # Globals
############################################################################## ##############################################################################
my $topology_file; # location of the topology input file.
my $ptopology_file; # location of the physical topology input file.
my $synserv; # synch server node my $synserv; # synch server node
my $rtproto; # routing protocol my $rtproto; # routing protocol
my $hostname; # this hosts name my $hostname; # this hosts name
...@@ -353,6 +351,8 @@ my $PATH_SCHEDFILE = "$VARDIR/logs/linktest.sched"; ...@@ -353,6 +351,8 @@ my $PATH_SCHEDFILE = "$VARDIR/logs/linktest.sched";
my $PATH_SYNCSERVER = "$VARDIR/boot/syncserver"; my $PATH_SYNCSERVER = "$VARDIR/boot/syncserver";
my $PATH_TOPOFILE = "$VARDIR/boot/ltmap"; my $PATH_TOPOFILE = "$VARDIR/boot/ltmap";
my $PATH_PTOPOFILE = "$VARDIR/boot/ltpmap"; my $PATH_PTOPOFILE = "$VARDIR/boot/ltpmap";
my $PATH_TOPOFILE_GZ = "${PATH_TOPOFILE}.gz";
my $PATH_PTOPOFILE_GZ = "${PATH_PTOPOFILE}.gz";
my $PATH_WINPING = "/cygdrive/c/windows/system32/ping.exe"; my $PATH_WINPING = "/cygdrive/c/windows/system32/ping.exe";
my $schedfile = $PATH_SCHEDFILE; my $schedfile = $PATH_SCHEDFILE;
...@@ -396,8 +396,6 @@ $EVENTID = "$proj_id/$exp_id" ...@@ -396,8 +396,6 @@ $EVENTID = "$proj_id/$exp_id"
# the current ns file and the output file for topology info. # the current ns file and the output file for topology info.
# #
$linktest_path = "$LOGDIR/linktest"; $linktest_path = "$LOGDIR/linktest";
$topology_file = $PATH_TOPOFILE;
$ptopology_file = $PATH_PTOPOFILE;
# #
# Determine what OS we are. Used for handling the occasional difference # Determine what OS we are. Used for handling the occasional difference
...@@ -415,7 +413,7 @@ sleep(int(rand(5))); ...@@ -415,7 +413,7 @@ sleep(int(rand(5)));
if (! $STANDALONE) { if (! $STANDALONE) {
&my_system($PATH_RCTOPO, "reconfig"); &my_system($PATH_RCTOPO, "reconfig");
} }
&get_topo($topology_file, $ptopology_file); &get_topo();
&debug_top(); &debug_top();
# #
...@@ -2246,14 +2244,20 @@ sub print_edge { ...@@ -2246,14 +2244,20 @@ sub print_edge {
# Handles reading the topology map(s). # Handles reading the topology map(s).
# Note: the topo map file can be huge, so we read it line at a time # Note: the topo map file can be huge, so we read it line at a time
# rather than en masse with read_file. # rather than en masse with read_file.
# We will also try to read from the gzip'd variants if they exist.
# #
sub get_topo { sub get_topo {
my ($top_file, $ptop_file) = @_; if (-e $PATH_TOPOFILE) {
open TOPO, $PATH_TOPOFILE || die ("Could not open $PATH_TOPOFILE\n");
die "Attempted to open missing file $top_file\n" }
unless -e $top_file; elsif (-e $PATH_TOPOFILE_GZ) {
open TOPO, "gzip -c -d $PATH_TOPOFILE_GZ |"
|| die ("Could not open $PATH_TOPOFILE_GZ\n");
}
else {
die "Attempted to open missing file $PATH_TOPOFILE\n";
}
open TOPO, $top_file || die ("Could not open $top_file\n");
while(<TOPO>) { while(<TOPO>) {
# load the output from ns. # load the output from ns.
# the file format is simple: # the file format is simple:
...@@ -2305,7 +2309,16 @@ sub get_topo { ...@@ -2305,7 +2309,16 @@ sub get_topo {
# #
# Augment with physical info if present # Augment with physical info if present
# #
if (open(TOPO, $ptop_file)) { my $have_ptop = 0;
if (-e $PATH_PTOPOFILE) {
$have_ptop = 1
if (open TOPO, $PATH_PTOPOFILE);
}
elsif (-e $PATH_PTOPOFILE_GZ) {
$have_ptop = 1
if (open TOPO, "gzip -c -d $PATH_PTOPOFILE_GZ |");
}
if ($have_ptop) {
my $vers = 1; my $vers = 1;
while(<TOPO>) { while(<TOPO>) {
my @row = split; my @row = split;
......
#!/usr/bin/perl -w #!/usr/bin/perl -w
# #
# Copyright (c) 2004-2012 University of Utah and the Flux Group. # Copyright (c) 2004-2018 University of Utah and the Flux Group.
# #
# {{{EMULAB-LICENSE # {{{EMULAB-LICENSE
# #
...@@ -40,6 +40,8 @@ $| = 1; ...@@ -40,6 +40,8 @@ $| = 1;
# Drag in path stuff so we can find emulab stuff. # Drag in path stuff so we can find emulab stuff.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; } BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }
my $LTGZIPONLY_FILE = "$ETCDIR/ltmap-gzip";
# #
# Load the OS independent support library. It will load the OS dependent # Load the OS independent support library. It will load the OS dependent
# library and initialize itself. # library and initialize itself.
...@@ -227,9 +229,17 @@ sub dotopomap() ...@@ -227,9 +229,17 @@ sub dotopomap()
} }
if (-s $templt) { if (-s $templt) {
system("cat ${templt} | gunzip > " . TMLTMAP()); if (-e $LTGZIPONLY_FILE) {
if ($?) { system("mv ${templt} " . TMLTMAPGZ());
fatal("Could not unzip ltmap"); if ($?) {
fatal("Could not mv ${templt} to " . TMLTMAPGZ());
}
}
else {
system("cat ${templt} | gunzip > " . TMLTMAP());
if ($?) {
fatal("Could not unzip ltmap");
}
} }
} }
else { else {
...@@ -237,9 +247,17 @@ sub dotopomap() ...@@ -237,9 +247,17 @@ sub dotopomap()
} }
if (-s $templtp) { if (-s $templtp) {
system("cat ${templtp} | gunzip > " . TMLTPMAP()); if (-e $LTGZIPONLY_FILE) {
if ($?) { system("mv ${templtp} " . TMLTPMAPGZ());
fatal("Could not unzip ltpmap"); if ($?) {
fatal("Could not mv ${templtp} to " . TMLTPMAPGZ());
}
}
else {
system("cat ${templtp} | gunzip > " . TMLTPMAP());
if ($?) {
fatal("Could not unzip ltpmap");
}
} }
} }
else { else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment