Commit f8ed2e3b authored by Leigh B. Stoller's avatar Leigh B. Stoller
Browse files

Back out previous revision that changed the XML output format to RPC

format instead of plain old well formed XML. Note this change does not
affect the parser in any way; the parser continues to spit out its own
brand of XML, and xmlconvert treats that special when it reads it.

The XML I spit out now is better formed and makes more sense when you
look at it. It also helps that its much faster to generate, and the
resulting output is much smaller. For example:

	time /usr/testbed/libexec/xmlconvert testbed jail-2440
	real    2m54.229s
	user    1m7.492s
	sys     1m15.927s

	time /usr/testbed/devel/stoller/libexec/xmlconvert testbed jail-2440
	real    0m14.738s
	user    0m5.755s
	sys     0m5.156s

	/tmp/bar*.xml
	-rw-rw-r--  1 stoller  wheel  15595161 Jun  4 11:16 /tmp/rpc.xml
	-rw-rw-r--  1 stoller  wheel   5285690 Jun  4 11:17 /tmp/plain.xml

Thats for a big experiment. For a small experiment:

	time /usr/testbed/libexec/xmlconvert testbed jail-416
	real    0m9.346s
	user    0m5.083s
	sys     0m2.675s

	time /usr/testbed/devel/stoller/libexec/xmlconvert testbed jail-416
	real    0m2.751s
	user    0m1.377s
	sys     0m1.183s

	ll /tmp/bar*.xml
	-rw-rw-r--  1 stoller  wheel  3135518 Jun  4 11:19 /tmp/rpc.xml
	-rw-rw-r--  1 stoller  wheel  1015745 Jun  4 11:19 /tmp/plain.xml

As you can see, the RPC code is rather non-linear in its performance!

Since this affects Tim and netlab-client, I have added an alternate
XMLRPC routine called virtual_topology_xml() that returns the new format,
since Tim will need to transition to this new format (parsing it into a
Java data structure). The old format is left in the RPC server until that
is done.
parent d51614da
#!/usr/bin/perl -wT #!/usr/bin/perl -w
# #
# EMULAB-COPYRIGHT # EMULAB-COPYRIGHT
...@@ -9,8 +9,8 @@ ...@@ -9,8 +9,8 @@
use English; use English;
use Getopt::Std; use Getopt::Std;
use XML::Parser; use XML::Parser;
use RPC::XML; #use RPC::XML;
use RPC::XML::Parser; #use RPC::XML::Parser;
# #
# Convert between XML and DB representation of a virtual experiment. # Convert between XML and DB representation of a virtual experiment.
...@@ -55,43 +55,57 @@ my $XMLHEADER = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>"; ...@@ -55,43 +55,57 @@ my $XMLHEADER = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>";
# #
my %virtual_tables = my %virtual_tables =
("experiments" => { rows => undef, ("experiments" => { rows => undef,
tag => "experiment", tag => "settings",
# Indicates a single row.
row => undef,
attrs => [ ] }, attrs => [ ] },
"virt_nodes" => { rows => undef, "virt_nodes" => { rows => undef,
tag => "nodes", tag => "nodes",
row => "node",
attrs => [ "vname" ]}, attrs => [ "vname" ]},
"virt_lans" => { rows => undef, "virt_lans" => { rows => undef,
tag => "lans", tag => "lans",
row => "lan",
attrs => [ "vname" ]}, attrs => [ "vname" ]},
"virt_lan_settings" => { rows => undef, "virt_lan_settings" => { rows => undef,
tag => "lan_settings", tag => "lan_settings",
row => "lan_setting",
attrs => [ "vname", "capkey" ]}, attrs => [ "vname", "capkey" ]},
"virt_lan_member_settings" => { rows => undef, "virt_lan_member_settings" => { rows => undef,
tag => "lan_member_settings", tag => "lan_member_settings",
row => "lan_member_setting",
attrs => [ "vname", "member", "capkey" ]}, attrs => [ "vname", "member", "capkey" ]},
"virt_trafgens" => { rows => undef, "virt_trafgens" => { rows => undef,
tag => "trafgens", tag => "trafgens",
row => "trafgen",
attrs => [ "vname", "vnode" ]}, attrs => [ "vname", "vnode" ]},
"virt_agents" => { rows => undef, "virt_agents" => { rows => undef,
tag => "agents", tag => "agents",
row => "agent",
attrs => [ "vname", "vnode" ]}, attrs => [ "vname", "vnode" ]},
"virt_node_desires" => { rows => undef, "virt_node_desires" => { rows => undef,
tag => "node_desires", tag => "node_desires",
row => "node_desire",
attrs => [ "vname", "desire" ]}, attrs => [ "vname", "desire" ]},
"virt_routes" => { rows => undef, "virt_routes" => { rows => undef,
tag => "routes", tag => "routes",
row => "route",
attrs => [ "vname", "src", "dst" ]}, attrs => [ "vname", "src", "dst" ]},
"virt_vtypes" => { rows => undef, "virt_vtypes" => { rows => undef,
tag => "vtypes", tag => "vtypes",
row => "vtype",
attrs => [ "name" ]}, attrs => [ "name" ]},
"virt_programs" => { rows => undef, "virt_programs" => { rows => undef,
tag => "programs", tag => "programs",
row => "program",
attrs => [ "vname", "vnode" ]}, attrs => [ "vname", "vnode" ]},
"nseconfigs" => { rows => undef, "nseconfigs" => { rows => undef,
tag => "nseconfigs", tag => "nseconfigs",
row => "nseconfig",
attrs => [ "vname" ]}, attrs => [ "vname" ]},
"eventlist" => { rows => undef, "eventlist" => { rows => undef,
tag => "events", tag => "events",
row => "event",
attrs => [ "vname" ]}); attrs => [ "vname" ]});
...@@ -118,6 +132,14 @@ my %experiment_fields = ("multiplex_factor" => 1, ...@@ -118,6 +132,14 @@ my %experiment_fields = ("multiplex_factor" => 1,
"use_ipassign" => 1, "use_ipassign" => 1,
"ipassign_args" => 1); "ipassign_args" => 1);
# New parsing code state machine control.
my $PARSING_NOTYET = 0;
my $PARSING_EXPERIMENT = 1;
my $PARSING_TABLE = 2;
my $PARSING_ROW = 3;
my $PARSING_SLOT = 4;
my $parserstate = $PARSING_NOTYET;
# #
# Turn off line buffering on output # Turn off line buffering on output
# #
...@@ -198,7 +220,7 @@ if ($fromxml) { ...@@ -198,7 +220,7 @@ if ($fromxml) {
readXML($pid, $eid, $xmlfile, $fromparser); readXML($pid, $eid, $xmlfile, $fromparser);
} }
else { else {
writeXML($pid, $eid); writeXML_XML($pid, $eid);
} }
exit(0); exit(0);
...@@ -229,8 +251,8 @@ sub readXML($$$$) { ...@@ -229,8 +251,8 @@ sub readXML($$$$) {
# Create a parser. # Create a parser.
# #
my $parser = new XML::Parser(Style => 'Tree'); my $parser = new XML::Parser(Style => 'Tree');
$parser->setHandlers('Start' => \&StartElement, $parser->setHandlers('Start' => \&StartElement_FromParser,
'End' => \&EndElement, 'End' => \&EndElement_FromParser,
'Char' => \&ProcessElement); 'Char' => \&ProcessElement);
fatal($@) fatal($@)
...@@ -238,29 +260,15 @@ sub readXML($$$$) { ...@@ -238,29 +260,15 @@ sub readXML($$$$) {
} }
else { else {
# #
# Convert data structure into table rows. # Create a parser.
# #
my $resp = RPC::XML::Parser->new()->parse(STDIN); my $parser = new XML::Parser(Style => 'Tree');
if (!ref($resp)) { $parser->setHandlers('Start' => \&StartElement,
fatal("Could not parse XML input!"); 'End' => \&EndElement,
} 'Char' => \&ProcessElement);
my $exp = $resp->value->value;
if (!exists($exp->{"experiment"}) ||
!exists($exp->{"experiment"}->{"settings"})) {
fatal("Missing experiment settings structure!");
}
$virtual_tables{"experiments"}->{"rows"} =
[ $exp->{"experiment"}->{"settings"} ];
foreach my $table (keys(%virtual_tables)) {
my $tag = $virtual_tables{$table}{"tag"};
if (exists($exp->{"experiment"}->{$tag})) { fatal($@)
$virtual_tables{$table}->{"rows"} = if (eval { $parser->parse(*STDIN); return 1; } != 1);
$exp->{"experiment"}->{$tag};
}
}
} }
# If these are the results of parsing the nse specifications, # If these are the results of parsing the nse specifications,
...@@ -330,7 +338,7 @@ sub readXML($$$$) { ...@@ -330,7 +338,7 @@ sub readXML($$$$) {
foreach my $key (keys(%experiments_table)) { foreach my $key (keys(%experiments_table)) {
my $val = $experiments_table{$key}; my $val = $experiments_table{$key};
if ($val eq "NULL" || $val eq "") { if (!defined($val) || $val eq "NULL" || $val eq "") {
push(@setlist, "$key=NULL"); push(@setlist, "$key=NULL");
} }
else { else {
...@@ -408,7 +416,7 @@ sub readXML($$$$) { ...@@ -408,7 +416,7 @@ sub readXML($$$$) {
foreach my $key (keys(%rowhash)) { foreach my $key (keys(%rowhash)) {
my $val = $rowhash{$key}; my $val = $rowhash{$key};
if ($val eq "NULL") { if (!defined($val) || $val eq "NULL") {
push(@values, "NULL"); push(@values, "NULL");
} }
elsif ($val eq "") { elsif ($val eq "") {
...@@ -459,7 +467,7 @@ sub readXML($$$$) { ...@@ -459,7 +467,7 @@ sub readXML($$$$) {
# #
# Start an element. # Start an element.
# #
sub StartElement ($$$) sub StartElement_FromParser ($$$)
{ {
my ($expat, $element, %attrs) = @_; my ($expat, $element, %attrs) = @_;
...@@ -518,7 +526,7 @@ sub StartElement ($$$) ...@@ -518,7 +526,7 @@ sub StartElement ($$$)
# #
# End an element. # End an element.
# #
sub EndElement ($$) sub EndElement_FromParser ($$)
{ {
my ($expat, $element) = @_; my ($expat, $element) = @_;
...@@ -586,11 +594,268 @@ sub ProcessElement ($$) ...@@ -586,11 +594,268 @@ sub ProcessElement ($$)
} }
} }
#
# Start an element.
#
sub StartElement ($$$)
{
my ($expat, $element, %attrs) = @_;
#
# First element must be the experiment tag; It starts the process.
#
if ($parserstate == $PARSING_NOTYET) {
fatal("Must start with an experiment tag!")
if ($element ne "experiment");
fatal("Out of sync at experiment start: $element")
if (defined($current_expt) ||
defined($current_table) ||
defined($current_row) ||
defined($current_slot));
$current_expt = "$pid/$eid";
#
# Sanity check pid/eid.
#
if ((exists($attrs{'pid'}) && $attrs{'pid'} ne $pid) ||
(exists($attrs{'eid'}) && $attrs{'eid'} ne $eid)) {
fatal("pid/eid mismatch!");
}
print "Starting new experiment $pid/$eid\n"
if ($debug);
$parserstate = $PARSING_EXPERIMENT;
}
elsif ($parserstate == $PARSING_EXPERIMENT) {
#
# Need to find the right table.
#
my $table;
foreach my $key (keys(%virtual_tables)) {
if ($virtual_tables{$key}->{"tag"} eq $element) {
$table = $key;
last;
}
}
fatal("Unknown table: $element")
if (!defined($table));
fatal("Out of sync at table start: $element")
if (!defined($current_expt) ||
defined($current_table) ||
defined($current_row) ||
defined($current_slot));
if (! defined($virtual_tables{$table}->{"rows"})) {
$virtual_tables{$table}->{"rows"} = [];
}
$current_table = $table;
$parserstate = $PARSING_TABLE;
print "Starting new table: $table\n"
if ($debug);
# Skip to parsing a row.
if (!defined($virtual_tables{$current_table}->{"row"})) {
$current_row = {};
$parserstate = $PARSING_ROW;
}
}
elsif ($parserstate == $PARSING_TABLE) {
#
# A row in a table. row tag must match table tag.
#
fatal("Out of sync at row start: $element")
if ((!defined($current_expt) ||
!defined($current_table) ||
defined($current_row) ||
defined($current_slot)) ||
$virtual_tables{$current_table}->{"row"} ne $element);
print "Starting new row at $element in table: $current_table\n"
if ($debug);
$current_row = {};
$parserstate = $PARSING_ROW;
}
elsif ($parserstate == $PARSING_ROW) {
#
# A slot in a row.
#
fatal("Out of sync at slot start: $element")
if (!defined($current_expt) ||
!defined($current_table) ||
!defined($current_row) ||
defined($current_slot));
print "Starting new slot $element in table: $current_table\n"
if ($debug);
$parserstate = $PARSING_SLOT;
$current_slot = $element;
$current_data = "";
}
else {
fatal("Out of sync at element: $element");
}
}
#
# End an element.
#
sub EndElement ($$)
{
my ($expat, $element) = @_;
if ($parserstate == $PARSING_EXPERIMENT) {
fatal("Out of sync at experiment start: $element")
if ($element ne "experiment" ||
(!defined($current_expt) ||
defined($current_table) ||
defined($current_row) ||
defined($current_slot)));
undef($current_expt);
$parserstate = $PARSING_NOTYET;
}
elsif ($parserstate == $PARSING_TABLE) {
#
# A table termination.
#
fatal("Out of sync at element end: $element")
if ((!defined($current_expt) ||
!defined($current_table) ||
defined($current_row) ||
defined($current_slot)) ||
$element ne $virtual_tables{$current_table}->{"tag"});
undef($current_table);
$parserstate = $PARSING_EXPERIMENT;
}
elsif ($parserstate == $PARSING_ROW) {
#
# A row termination.
#
fatal("Out of sync at row end: $element")
if ((!defined($current_expt) ||
!defined($current_table) ||
!defined($current_row) ||
defined($current_slot)) ||
(defined($virtual_tables{$current_table}->{"row"}) &&
$element ne $virtual_tables{$current_table}->{"row"}));
print "Adding new row $element to table $current_table\n"
if ($debug);
push(@{ $virtual_tables{$current_table}->{"rows"} }, $current_row);
undef($current_row);
$parserstate = $PARSING_TABLE;
# Skip to parsing an experiment
if (!defined($virtual_tables{$current_table}->{"row"})) {
undef($current_table);
$parserstate = $PARSING_EXPERIMENT;
}
}
elsif ($parserstate == $PARSING_SLOT) {
#
# A slot termination.
#
fatal("Out of sync at slot end: $element")
if (!defined($current_expt) ||
!defined($current_table) ||
!defined($current_row) ||
!defined($current_slot));
#
# Always ignore pid/eid.
#
if ($current_slot ne "pid" && $current_slot ne "eid") {
print " Entering new slot: $current_slot: $current_data\n"
if ($debug);
$current_row->{$current_slot} =
($current_data ne "__NULL__" ? $current_data : undef);
}
undef($current_slot);
undef($current_data);
$parserstate = $PARSING_ROW;
}
}
# #
# Convert a virtual experiment representation into XML and spit it out. # Convert a virtual experiment representation into XML and spit it out.
# The DB holds the data of course. # The DB holds the data of course.
# #
sub writeXML($$) { sub writeXML_XML($$) {
my ($pid, $eid) = @_;
my $query_result =
DBQueryFatal("select * from experiments ".
"where eid='$eid' and pid='$pid'");
if (! $query_result->numrows) {
fatal("No such experiment $pid/$eid exists!");
}
spitxml_header();
spitxml_opentag("experiment pid='$pid' eid='$eid'", 0);
spitxml_opentag("settings", 1);
spitxml_spaces(2);
my $settings = $query_result->fetchrow_hashref();
foreach my $key (keys(%{ $settings })) {
my $data = $settings->{$key};
spitxml_entity($key, $data, 0);
}
spitxml_closetag("settings", 1);
#
# Read in a set of tables that live at top level.
#
foreach my $table (keys(%virtual_tables)) {
next
if ($table eq "experiments");
my $tabletag = $virtual_tables{$table}->{"tag"};
my $rowtag = $virtual_tables{$table}->{"row"};
$query_result =
DBQueryFatal("select * from $table ".
"where eid='$eid' and pid='$pid'");
next
if (! $query_result->numrows);
spitxml_opentag($tabletag, 1);
while (my $rowref = $query_result->fetchrow_hashref()) {
spitxml_opentag($rowtag, 2);
spitxml_spaces(3);
foreach my $key (keys(%{ $rowref })) {
my $data = $rowref->{$key};
next
if ($key eq "pid" || $key eq "eid");
spitxml_entity($key, $data, 0);
}
print "\n";
spitxml_closetag($rowtag, 2);
}
spitxml_closetag($tabletag, 1);
}
spitxml_closetag("experiment", 0);
return 0;
}
#
# This is the old version; I will eventually remove it.
#
sub writeXML_RPC($$) {
my ($pid, $eid) = @_; my ($pid, $eid) = @_;
my $query_result = my $query_result =
...@@ -639,6 +904,57 @@ sub writeXML($$) { ...@@ -639,6 +904,57 @@ sub writeXML($$) {
return 0; return 0;
} }
#
# Utility functions to pretty print XML output, with specified indentation.
#
sub spitxml_spaces($)
{
my ($level) = @_;
my $spaces = $level * 1;
printf("%${spaces}s", "");
}
sub spitxml_opentag($$)
{
my ($tag, $level) = @_;
spitxml_spaces($level);
print "<${tag}>\n";
}
sub spitxml_closetag($$)
{
my ($tag, $level) = @_;
spitxml_spaces($level);
print "</${tag}>\n";
}
sub spitxml_header()
{
print "$XMLHEADER\n";
}
sub spitxml_entity($$$)
{
my ($tag, $data, $level) = @_;
$data = "__NULL__"
if (!defined($data));
spitxml_spaces($level)
if ($level);
if ($data eq "") {
print "<${tag}/>";
}
else {
print "<${tag}>" . xmlencode($data) . "</${tag}>";
}
}
# #
# Convert from/to XML special chars. Not many of them ... # Convert from/to XML special chars. Not many of them ...
# #
......
...@@ -1453,6 +1453,61 @@ class experiment: ...@@ -1453,6 +1453,61 @@ class experiment:
return EmulabResponse(RESPONSE_SUCCESS, value=result, output="") return EmulabResponse(RESPONSE_SUCCESS, value=result, output="")
def virtual_topology_xml(self, version, argdict):
if version != self.VERSION:
return EmulabResponse(RESPONSE_BADVERSION,
output="Client version mismatch!");
pass
try:
checknologins()
pass
except NoLoginsError, e:
return EmulabResponse(RESPONSE_REFUSED, output=str(e))
argerror = CheckRequiredArgs(argdict, ("proj", "exp"))
if (argerror):
return argerror
if not (re.match("^[-\w]*$", argdict["proj"]) and
re.match("^[-\w]*$", argdict["exp"])):
return EmulabResponse(RESPONSE_BADARGS,
output="Illegal characters in arguments!")
#
# Check permission. This needs to be a lib routine!
#
res = DBQueryFatal("SELECT gid FROM experiments "
"WHERE pid=%s and eid=%s",