xmlconvert.in 14.8 KB
Newer Older
1
2
3
4
#!/usr/bin/perl -wT

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
6
7
8
9
10
# All rights reserved.
#

use English;
use Getopt::Std;
11
use XML::Parser;
12
13
14
15
16
    
#
# Convert between XML and DB representation of a virtual experiment.
# Very simple, no DTDs, DOMs, XSLs, etc. Just the facts ...
#
17
18
19
20
# XXX We do not regex the data carefully enough before inserting it into
# the DB. We run quotemeta() over it, but we should be more careful about
# per-slot checks.
#
21
22
23
24
25
26
sub usage()
{
    print STDOUT "Usage: xmlconvert [-x <xmlfile> [-n]] [-d] pid eid\n";
 
    exit(-1);
}
27
my $optlist  = "x:nds";
28
my $fromxml  = 0;
29
my $impotent = 0;
30
my $debug    = 0;
31
32
33
# Results of parsing nse specifications. Therefore different treatment.
# In particular, we don't expect updates to the experiments table
my $simparse = 0;
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

#
# Configure variables
#
my $TB       = "@prefix@";
my $TBOPS    = "@TBOPSEMAIL@";

# Locals
my $xmlfile;
my $pid;
my $eid;

# This goes at the beginning of the output.
my $XMLHEADER = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>";

#
# These are the virtual tables that make up an experiment.
# Each one could have multiple rows, each of which will be a
# hash table. 
53
54
55
56
57
58
59
60
my %virtual_tables = ("experiments"		=> undef,
		      "virt_nodes"		=> undef,
		      "virt_lans"		=> undef,
		      "virt_trafgens"		=> undef,
		      "virt_agents"		=> undef,
		      "virt_node_desires"	=> undef,
		      "virt_routes"		=> undef,
		      "virt_vtypes"		=> undef,
61
		      "virt_programs"		=> undef,
62
63
		      "nseconfigs"		=> undef,
		      "eventlist"		=> undef);
64
65
66
67

# XXX
# The experiment table is special. Only certain fields are allowed to
# be updated. Not sure what the right approach for this is.
68
# Note that I regex the data before inserting it below.
69
#
70
71
72
73
74
75
76
my %experiment_fields = ("multiplex_factor"		=> 1,
			 "forcelinkdelays"		=> 1,
			 "uselinkdelays"		=> 1,
			 "usewatunnels"			=> 1,
			 "uselatestwadata"		=> 1,
			 "wa_delay_solverweight"	=> 1,
			 "wa_bw_solverweight"		=> 1,
77
			 "wa_plr_solverweight"		=> 1,
78
79
			 "cpu_usage"			=> 1,
			 "mem_usage"			=> 1,
80
			 "allowfixnode"			=> 1,
81
			 "veth_encapsulate"		=> 1,
82
83
			 "jail_osname"			=> 1,
			 "delay_osname"			=> 1,
84
85
86
			 "sync_server"			=> 1,
		         "use_ipassign"			=> 1,
		         "ipassign_args"		=> 1);
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

#
# Turn off line buffering on output
#
$| = 1;

#
# Untaint the path
# 
$ENV{'PATH'} = "/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;

#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
117
118
119
if (defined($options{"s"})) {
    $simparse = 1;
}
120
121
122
123
124
125
126
127
128
129
130
if (defined($options{"x"})) {
    $fromxml = 1;
    $xmlfile = $options{"x"};

    if ($xmlfile =~ /^([-\w\/\.]+)$/) {
	$xmlfile = $1;
    }
    else {
	fatal("Bad data in argument: $xmlfile.");
    }
    if (defined($options{"n"})) {
131
	$impotent = 1;
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
    }    
}
if (@ARGV != 2) {
    usage();
}
$pid   = $ARGV[0];
$eid   = $ARGV[1];

# Taint Check.
if ($pid =~ /^([-\w]+)$/) {
    $pid = $1;
}
else {
    fatal("Bad data in argument: $pid.");
}
if ($eid =~ /^([-\w]+)$/) {
    $eid = $1;
}
else {
    fatal("Bad data in argument: $eid.");
}

154
155
156
157
158
159
my %event_objtypes;
my $query_result = DBQueryFatal("select idx,type from event_objecttypes");
while (my ($idx,$type) = $query_result->fetchrow_array()) {
    $event_objtypes{$type} = $idx;
}

160
161
162
163
164
165
166
167
168
169
170
171
172
173
# Do it
if ($fromxml) {
    readXML($pid, $eid, $xmlfile);
}
else {
    writeXML($pid, $eid);
}
exit(0);

#
# Read in XML and convert to DB representation, doing lots of checks!
# This code is silly. Overly stylized (one tag per line!). Should
# use the XML::Parser package instead. But this was easy and fun for a
# first cut. 
174
175
176
177
178
179
180
181
#
# State variables for parsing code below.
my $current_expt;
my $current_table;
my $current_row;
my $current_slot;
my $current_data;

182
183
184
185
186
187
188
189
190
191
sub readXML($$$) {
    my ($pid, $eid, $xmlfile) = @_;
    my %experiment;

    if ($xmlfile ne "-") {
	open(STDIN, "< $xmlfile")
	    or fatal("opening $xmlfile for STDIN: $!");
    }

    #
192
    # Create a parser.
193
    #
194
195
196
197
    my $parser = new XML::Parser(Style => 'Tree');
    $parser->setHandlers('Start'   => \&StartElement,
			 'End'     => \&EndElement,
			 'Char'    => \&ProcessElement);
198

199
200
    fatal($@)
	if (eval { $parser->parse(*STDIN); return 1; } != 1);
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

    # If these are the results of parsing the nse specifications,
    # we don't expect updates to the experiments table
    my %experiments_table;
    if ( ! $simparse ) {

	#
	# Verify. 
	#
	# Must be exactly one experiments table row, and we prune out lots
	# of stuff that is not allowed. Note that we never insert a
	# experiment, but only allow updates of certain values. 
	#
	if (scalar(@{$virtual_tables{"experiments"}}) != 1) {
	    fatal("Must be exactly one experiments table row!");
	}
	%experiments_table = %{@{$virtual_tables{"experiments"}}[0]};
	foreach my $key (keys(%experiments_table)) {
	    delete($experiments_table{$key})
		if (!exists($experiment_fields{$key}));
	}
222
223
224
225
226
227
228
229
230
231
232
    }

    #
    # Okay, a hokey DoS check. Do not allow more than 10000 total rows!
    # Why so many? Well, Rob likes to generate lots of events!
    #
    my $count = 0;
    foreach my $table (keys(%virtual_tables)) {
	$count += scalar(@{$virtual_tables{$table}})
	    if (defined($virtual_tables{$table}));
    }
233
    if ($count > 100000) {
234
235
236
237
238
	fatal("Too many rows of data!");
    }

    #
    # Okay, thats all the checking we do! There is not much that can
239
240
    # screw up the DB just by inserting rows into the allowed set of
    # virtual experiment tables, since we ignore the pid/eid in the xml. 
241
    #
242
243
    # First the experiments table, which gets an update statement, if there
    # is anything to update.
244
    #
245
    if ( (! $simparse) && scalar(keys(%experiments_table))) {
246
	my @setlist = ();
247

248
249
	foreach my $key (keys(%experiments_table)) {
	    my $val = $experiments_table{$key};
250

251
252
253
254
	    if ($val eq "NULL") {
		push(@setlist, "$key=NULL");
	    }
	    else {
255
256
257
		# Sanity check the fields.
		if (TBcheck_dbslot($val, "experiments", $key,
				TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
258
259
260
		    push(@setlist, "$key='$val'");
		}
		else {
261
262
		    fatal("Illegal characters in table data: ".
			  "experiments:$key - $val");
263
		}
264
	    }	    
265
	}
266
267
268
269
270
271
272
273
	my $query = "update experiments ".
	            "set " . join(",", @setlist) . " " .
		    "where eid='$eid' and pid='$pid'";

	print "$query\n"
	    if ($debug);
	DBQueryFatal($query)
	    if (!$impotent);
274
275
276
277
278
279
280
    }

    #
    # Now all the other tables, which get inserts. Need to delete all the
    # old info too.
    #
    foreach my $table (keys(%virtual_tables)) {
281
282
283
284
	# Don't want to muck with this table! Done above. 
	next
	    if ($table eq "experiments");

285
286
287
288
289
290
	# Delete only during the initial parsing and not
	# during parsing of nse specifications
	if ( ! $simparse ) {
	    DBQueryFatal("delete from $table ".
			 "where eid='$eid' and pid='$pid'")
		if (!$impotent);
291
292
293
294
295
296
297
298
299
300
301
302
	} else {
	    # The nseconfigs table is special. During a
	    # simparse, we need delete all rows for the
	    # experiment except the one with the vname
	    # 'fullsim'. This row is essentially virtual
	    # info and does not change across swapins
	    # where as the other rows depend on the
	    # mapping
	    if ( !$impotent && ($table eq "nseconfigs") ) {
		DBQueryFatal("delete from $table ". 
		             "where eid='$eid' and pid='$pid' and ".
			     "vname!='fullsim'")
303
304
305
306
307
308
309
310
311
312
	    } elsif ( !$impotent && (($table eq "eventlist") ||
				       ($table eq "virt_agents")) ) {
	        # Both eventlist and virt_agents need to be cleared
		# for NSE event objecttype since entries in this
		# table depend on the particular mapping
		my $nse_objtype = $event_objtypes{"NSE"};
		DBQueryFatal("delete from $table ". 
		             "where pid='$pid' and eid='$eid' and ".
			     "objecttype='$nse_objtype'");
	    } 
313
	}
314
315
316
317
318
319
320
321
	next
	    if (!defined($virtual_tables{$table}));

	foreach my $rowref (@{$virtual_tables{$table}}) {
	    my %rowhash = %{ $rowref };
	    my @fields  = ("pid", "eid");
	    my @values  = ("'$pid'", "'$eid'");

322
323
324
325
	    # If no actual rows, then skip. Might happen.
	    last
		if (! scalar(keys(%rowhash)));

326
	    foreach my $key (keys(%rowhash)) {
327
		my $val = $rowhash{$key};
328
329
330
331

		if ($val eq "NULL") {
		    push(@values, "NULL");
		}
332
333
334
		elsif ($val eq "") {
		    push(@values, "''");
		}
335
		else {
336
337
338
339
340
341
342
343
344
345
		    # Sanity check the fields.
		    if (TBcheck_dbslot($val, $table, $key,
				 TBDB_CHECKDBSLOT_WARN|TBDB_CHECKDBSLOT_ERROR)) {
			push(@values, DBQuoteSpecial($val));
		    }
		    else {
			fatal("Illegal characters in table data: ".
			      "$table:$key - $val");
		    }
		}
346
347
		push(@fields, $key);
	    }
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
	    # If we are called after an nseparse, we need to
	    # use replace coz some of the tables such as 
	    # virt_agents and eventlist are not truly
	    # virtual tables. That is coz they contain the
	    # vnode field which is the same as the vname
	    # field in the reserved table. For simulated
	    # nodes, the mapping may change across swapins
	    # and the event may have to be delivered to a
	    # different simhost
	    if ( $simparse ) {
		$query = "replace into $table (" . join(",", @fields) . ") ".
		"values (" . join(",", @values) . ") ";
	    } else {
		$query = "insert into $table (" . join(",", @fields) . ") ".
		"values (" . join(",", @values) . ") ";
	    }
364
365
366
367

	    print "$query\n"
		if ($debug);
	    DBQueryFatal($query)
368
		if (!$impotent);
369
370
371
372
373
	}
    }
    return 0;
}

374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
#
# XML::Parser routines.
#
#
# Start an element.
# 
sub StartElement ($$$)
{
    my ($expat, $element, %attrs) = @_;

    if ($element eq "virtual_experiment") {
	fatal("Out of sync at experiment start: $element")
	    if (defined($current_expt) ||
		defined($current_table) ||
		defined($current_row) ||
		defined($current_slot));
	$current_expt = "$pid/$eid";
	
	#
	# Sanity check pid/eid.
	#
	if ((exists($attrs{'pid'}) && $attrs{'pid'} ne $pid) ||
	    (exists($attrs{'eid'}) && $attrs{'eid'} ne $eid)) {
	    fatal("pid/eid mismatch!");
	}
    }
    elsif (exists($virtual_tables{$element})) {
	#
	# A new table start.
	#
	fatal("Out of sync at element start: $element")
	    if (!defined($current_expt) ||
		defined($current_table) ||
		defined($current_row) ||
		defined($current_slot));
	$current_table = $element;

	if (! defined($virtual_tables{$element})) {
	    $virtual_tables{$element} = [];
	}
	print "Starting new table: $element\n"
	    if ($debug);
    }
    elsif ($element eq "row") {
	fatal("Out of sync at row start: $element")
	    if (!defined($current_expt) ||
		!defined($current_table) ||
		defined($current_row) ||
		defined($current_slot));
	$current_row = {};
    }
    else {
	fatal("Out of sync at slot start: $element")
	    if (!defined($current_expt) ||
		!defined($current_table) ||
		!defined($current_row) ||
		defined($current_slot));
	$current_slot = $element;
	$current_data = "";
    }
}

#
# End an element.
# 
sub EndElement ($$)
{
    my ($expat, $element) = @_;

    if ($element eq "virtual_experiment") {
	fatal("Out of sync at experiment start: $element")
	    if (!defined($current_expt) ||
		defined($current_table) ||
		defined($current_row) ||
		defined($current_slot));
	undef($current_expt);
    }
    elsif (exists($virtual_tables{$element})) {
	#
	# A table termination.
	#
	fatal("Out of sync at element end: $element")
	    if (!defined($current_expt) ||
		!defined($current_table) ||
		defined($current_row) ||
		defined($current_slot));
	undef($current_table);
    }
    elsif ($element eq "row") {
	fatal("Out of sync at row end: $element")
	    if (!defined($current_expt) ||
		!defined($current_table) ||
		!defined($current_row) ||
		defined($current_slot));

	print "Adding new row to table $current_table\n"
	    if ($debug);
	
	push(@{$virtual_tables{$current_table}}, $current_row);
	undef($current_row);
    }
    else {
	fatal("Out of sync at slot end: $element")
	    if (!defined($current_expt) ||
		!defined($current_table) ||
		!defined($current_row) ||
		!defined($current_slot));
    
	#
	# Always ignore pid/eid.
	#
	if ($current_slot ne "pid" && $current_slot ne "eid") {
	    print "    Entering new slot: $current_slot: $current_data\n"
		if ($debug);
	    $current_row->{$current_slot} = $current_data;
	}
	undef($current_slot);
	undef($current_data);
    }
}

#
# Process stuff inside a slot.
# 
sub ProcessElement ($$)
{
    my ($expat, $string) = @_;

    if (defined($current_slot)) {
	$current_data .= xmldecode($string);
    }
}

507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
#
# Convert a virtual experiment representation into XML and spit it out.
# The DB holds the data of course.
#
sub writeXML($$) {
    my ($pid, $eid) = @_;

    my $query_result =
	DBQueryFatal("select * from experiments ".
		     "where eid='$eid' and pid='$pid'");

    if (! $query_result->numrows) {
	fatal("No such experiment $pid/$eid exists!");
    }

    spitxml_header();
    spitxml_opentag("virtual_experiment pid='$pid' eid='$eid'", 0);

    # Spit out all of the tables.
    foreach my $table (keys(%virtual_tables)) {
	spitxml_dbrows($table, 1, 
		       "select * from $table ".
		       "where eid='$eid' and pid='$pid'");
    }
    spitxml_closetag("virtual_experiment", 0);

    return 0;
}

#
# Utility functions to pretty print XML output, with specified indentation.
#
sub spitxml_opentag($$)
{
    my ($tag, $level) = @_;

    my $spaces = $level * 2;

    printf("%${spaces}s<%s>\n", "", $tag);
}

sub spitxml_closetag($$)
{
    my ($tag, $level) = @_;

    my $spaces = $level * 2;

    printf("%${spaces}s</%s>\n", "", $tag);
}

sub spitxml_header()
{
    print "$XMLHEADER\n";
}

sub spitxml_entity($$$)
{
    my ($tag, $data, $level) = @_;
    my $spaces = $level * 2;

    $data = "NULL"
	if (!defined($data));

    printf("%${spaces}s<%s>%s</%s>\n", "", $tag, xmlencode($data), $tag);
}

sub spitxml_dbrow($%)
{
    my($level, %dbrow) = @_;

    spitxml_opentag("row", $level);
    
    foreach my $tag (keys(%dbrow)) {
	my $data = $dbrow{$tag};
	spitxml_entity($tag, $data, $level + 1);

    }
    spitxml_closetag("row", $level);
}

sub spitxml_dbrows($$$)
{
    my($tag, $level, $query) = @_;
    my $query_result = DBQueryFatal($query);
    
    if ($query_result->numrows) {
	spitxml_opentag($tag, $level);
	while (my %hashrow = $query_result->fetchhash()) {
	    spitxml_dbrow($level + 1, %hashrow);
	}
	spitxml_closetag($tag, $level);
    }
}

#
# Convert from/to XML special chars. Not many of them ...
# 
sub xmlencode($)
{
    my ($string) = @_;

    my %specialchars = ('&' => '&amp;',
			'<' => '&lt;',
			'>' => '&gt;',
611
612
613
			"'" => '&#39;',
			"]" => '&#93;',
			'"' => '&#34;');
614
615
616
617
618
619
620
621
622
623
624
625

    $string =~ s/([&<>"'])/$specialchars{$1}/ge;
    return $string;
}

sub xmldecode($)
{
    my ($string) = @_;

    my %specialchars = ('&amp;'  => '&',
			'&lt;'   => '<',
			'&gt;'   => '>',
626
627
628
			'&#39;'  => "'",
			'&#93;'  => ']',
			'&#34;'  => '"');
629
630
631
632
633
634
635
636
637
638
639
640
641

    $string =~ s/(&\w+;)/$specialchars{$1}/ge;
    return $string;
}

# Die
sub fatal($)
{
    my ($msg) = @_;

    die("*** $0:\n".
	"    $msg\n");
}