Merge branch 'master' of git-public.flux.utah.edu:/flux/git/emulab-devel

.gitignore

 event/trafgen/tg2.0
+event/linktest/iperf/iperf-2.0.2
+event/linktest/rude/rude-0.70
 sensors/nfstrace/nfsdump2/Makefile
 sensors/nfstrace/nfsdump2/config.h
 sensors/nfstrace/nfsdump2/config.log

db/Experiment.pm.in

@@ -108,6 +108,7 @@ my $EXPT_RESOURCESHOSED = 0;
 		   "event_groups",
 		   "virt_firewalls",
 		   "firewall_rules",
+		   "elabinelab_attributes",
 		   "virt_tiptunnels",
 		   "ipsubnets");
 

db/Node.pm.in

@@ -2478,5 +2478,32 @@ sub LookupWideArea($$)
     return Node->Lookup($node_id);
 }
 
+#
+# Return the partition that an OSID is loaded on. 
+#
+sub IsOSLoaded($$)
+{
+    my ($self, $osinfo) = @_;
+
+    if (!ref($osinfo)) {
+	my $tmp = OSinfo->Lookup($osinfo);
+	if (!defined($tmp)) {
+	    print STDERR "Cannot lookup osinfo for $osinfo\n";
+	    return -1;
+	}
+	$osinfo = $tmp;
+    }
+    my $osid   = $osinfo->osid();
+    my $nodeid = $self->node_id();
+
+    my $query_result =
+	DBQueryWarn("select osid from partitions as p ".
+		    "where p.node_id='$nodeid' and p.osid='$osid'");
+    return -1
+	if (!$query_result);
+
+    return $query_result->numrows;
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

db/OSinfo.pm.in

 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2007-2009 University of Utah and the Flux Group.
+# Copyright (c) 2007-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package OSinfo;
@@ -19,6 +19,7 @@ use libdb;
 use libtestbed;
 use Project;
 use User;
+use Image;
 use English;
 use Data::Dumper;
 use overload ('""' => 'Stringify');
@@ -433,6 +434,17 @@ sub ValidOpMode($$)
     return (exists($OPMODES{$opmode}) ? 1 : 0);
 }
 
+#
+# Is the osinfo "generic"
+#
+sub IsGeneric($)
+{
+    my ($self)  = @_;
+    my $version = $self->version();
+    
+    return ((defined($version) && $version ne "") ? 0 : 1);
+}
+
 #
 # Boot command like. The caller supplies the default in $pref.
 #
@@ -618,5 +630,28 @@ sub RunsOnParent($$)
     return 1;
 }
 
+#
+# Map an osinfo and node type to the actual image.
+#
+sub MapToImage($$)
+{
+    my ($self, $type) = @_;
+
+    if (ref($type)) {
+	$type = $type->type();
+    }
+    my $osid = $self->osid();
+
+    my $query_result =
+	DBQueryWarn("select imageid from osidtoimageid ".
+		    "where type='$type' and osid='$osid'");
+    return undef
+	if (!defined($query_result) || !$query_result->numrows);
+
+    my ($imageid) = $query_result->fetchrow_array();
+
+    return Image->Lookup($imageid);
+}
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

db/VirtExperiment.pm.in

 #!/usr/bin/perl -wT
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2009 University of Utah and the Flux Group.
+# Copyright (c) 2009-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 package VirtExperiment;
@@ -69,6 +69,7 @@ my $debug	      = 0;
      "event_groups"		=> [ "group_name", "agent_name" ],
      "virt_firewalls"		=> [ "fwname", "type", "style" ],
      "firewall_rules"		=> [ "fwname", "ruleno", "rule" ],
+     "elabinelab_attributes"	=> [ "role", "attrkey", "ordering" ],
      "virt_tiptunnels"		=> [ "host", "vnode" ],
      "virt_parameters"          => [ "name", "value" ],
 );
@@ -627,7 +628,7 @@ sub NewRow($$)
 		$argref->{$key} = $self->{'COUNTER'}++;
 	    }
 	    else {
-		carp("Missing table key for new table in $tablename");
+		carp("Missing table key $key for new table in $tablename");
 		return undef;
 	    }
 	}
@@ -1178,5 +1179,10 @@ use vars qw(@ISA);
 @ISA = "VirtExperiment::VirtTableRow";
 use VirtExperiment;
 
+package VirtExperiment::VirtTableRow::elabinelab_attributes;
+use vars qw(@ISA);
+@ISA = "VirtExperiment::VirtTableRow";
+use VirtExperiment;
+
 # _Always_ make sure that this 1 is at the end of the file...
 1;

db/xmlconvert.in

@@ -2,7 +2,7 @@
 
 #
 # EMULAB-COPYRIGHT
-# Copyright (c) 2000-2009 University of Utah and the Flux Group.
+# Copyright (c) 2000-2010 University of Utah and the Flux Group.
 # All rights reserved.
 #
 
@@ -110,6 +110,9 @@ my %virtual_tables =
      "firewall_rules"		=> { rows  => undef, 
 				     tag   => "firewall_rules",
 				     row   => "firewall_rule"},
+     "elabinelab_attributes"	=> { rows  => undef, 
+				     tag   => "elabinelab_attributes",
+				     row   => "attribute"},
      "virt_tiptunnels"		=> { rows  => undef, 
 				     tag   => "tiptunnels",
 				     row   => "tiptunnel"},

event/simple-agent/compile.sh

+gcc -DELVIN_COMPAT -c ../lib/event.c -o event.o -I../lib/ -I../../lib/libtb/ \
+  -I../../.. -L../../../pubsub
+gcc -DELVIN_COMPAT -c ../lib/util.c -o util.o -I../lib/ -I../../lib/libtb/ \
+  -I../../.. -L../../../pubsub
+ar crv libevent.a event.o util.o
+ranlib libevent.a
+g++ -DELVIN_COMPAT -o simple-agent -Wall -I../lib/ -I../../lib/libtb/ \
+  -I../../.. -L../../../pubsub main.cc libevent.a -lpubsub -lssl
+

event/simple-agent/main.cc

+// main.cc
+
+#include <cassert>
+#include <cmath>
+#include <ctime>
+
+#include <string>
+#include <map>
+#include <list>
+#include <vector>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+
+using namespace std;
+
+// For getopt
+#include <unistd.h>
+
+#include "event.h"
+#include "tbdefs.h"
+
+enum { EVENT_BUFFER_SIZE = 50 };
+
+namespace g
+{
+  std::string experimentName;
+  bool debug = false;
+}
+
+void readArgs(int argc, char * argv[]);
+void usage(char * name);
+// Reads the map file, initializes the pipe and pipeVault data
+// structure, and sets up the two subscription strings for events.
+void writePidFile(string const & pidFile);
+void initEvents(string const & server, string const & port,
+                string const & keyFile, string const & subscription);
+void subscribe(event_handle_t handle, address_tuple_t eventTuple,
+               string const & subscription);
+void callback(event_handle_t handle,
+              event_notification_t notification, void *data);
+
+int main(int argc, char * argv[])
+{
+  cerr << "Beginning program" << endl;
+  readArgs(argc, argv);
+  return 0;
+}
+
+void readArgs(int argc, char * argv[])
+{
+  cerr << "Processing arguments" << endl;
+  string server;
+  string port;
+  string keyFile;
+  string subscription;
+
+  // Prevent getopt from printing an error message.
+  opterr = 0;
+
+  /* get params from the optstring */
+  char const * argstring = "s:p:dE:k:u:";
+  int option = getopt(argc, argv, argstring);
+  while (option != -1)
+  {
+    switch (option)
+    {
+    case 'd':
+      g::debug = true;
+      break;
+    case 's':
+      server = optarg;
+      break;
+    case 'p':
+      port = optarg;
+      break;
+    case 'E':
+      g::experimentName = optarg;
+      break;
+    case 'k':
+      keyFile = optarg;
+      break;
+    case 'u':
+      subscription = optarg;
+      break;
+    case '?':
+    default:
+      usage(argv[0]);
+      break;
+    }
+    option = getopt(argc, argv, argstring);
+  }
+
+  /*Check if all params are specified, otherwise, print usage and exit*/
+  if(server == "" || g::experimentName == "")
+      usage(argv[0]);
+
+
+  initEvents(server, port, keyFile, subscription);
+}
+
+void usage(char * name)
+{
+  cerr << "Usage: " << name << " -E proj/exp -s server [-d] [-p port] "
+       << "[-i pidFile] [-k keyFile] [-u subscription]" << endl;
+  exit(-1);
+}
+
+void initEvents(string const & server, string const & port,
+                string const & keyFile, string const & subscription)
+{
+  cerr << "Initializing event system" << endl;
+  string serverString = "elvin://" + server;
+  event_handle_t handle;
+  if (port != "")
+  {
+    serverString += ":" + port;
+  }
+  cerr << "Server string: " << serverString << endl;
+  if (keyFile != "")
+  {
+    handle = event_register_withkeyfile(const_cast<char *>(serverString.c_str()), 0,
+                                        const_cast<char *>(keyFile.c_str()));
+  }
+  else
+  {
+    handle = event_register_withkeyfile(const_cast<char *>(serverString.c_str()), 0, NULL);
+  }
+  if (handle == NULL)
+  {
+    cerr << "Could not register with event system" << endl;
+    exit(1);
+  }
+
+  address_tuple_t eventTuple = address_tuple_alloc();
+
+  subscribe(handle, eventTuple, subscription);
+
+  address_tuple_free(eventTuple);
+
+  event_main(handle);
+}
+
+void subscribe(event_handle_t handle, address_tuple_t eventTuple,
+               string const & subscription)
+{
+  cerr << "Link subscription names: " << subscription << endl;
+  eventTuple->objname = const_cast<char *>(subscription.c_str());
+//  eventTuple->objtype = TBDB_OBJECTTYPE_LINK;
+  eventTuple->objtype = ADDRESSTUPLE_ANY;
+//  eventTuple->eventtype = TBDB_EVENTTYPE_MODIFY;
+  eventTuple->eventtype = ADDRESSTUPLE_ANY;
+  eventTuple->expt = const_cast<char *>(g::experimentName.c_str());
+  eventTuple->host = ADDRESSTUPLE_ANY;
+  eventTuple->site = ADDRESSTUPLE_ANY;
+  eventTuple->group = ADDRESSTUPLE_ANY;
+  eventTuple->scheduler = 1;
+  if (event_subscribe(handle, callback, eventTuple, NULL) == NULL)
+  {
+    cerr << "Could not subscribe to " << eventTuple->eventtype << " event" << endl;
+    exit(1);
+  }
+}
+
+void callback(event_handle_t handle,
+              event_notification_t notification,
+              void * data)
+{
+  char name[EVENT_BUFFER_SIZE];
+  char type[EVENT_BUFFER_SIZE];
+  char args[EVENT_BUFFER_SIZE];
+  time_t basicTime = time(NULL);
+  struct tm * structTime = gmtime(&basicTime);
+  char timestamp[1024];
+  strftime(timestamp, 1024, "%Y-%m-%dT%H:%M:%S", structTime);
+
+  if (event_notification_get_string(handle, notification, "OBJNAME", name, EVENT_BUFFER_SIZE) == 0)
+  {
+    cerr << timestamp << ": ERROR: Could not get the object name" << endl;
+    return;
+  }
+
+  if (event_notification_get_string(handle, notification, "EVENTTYPE", type, EVENT_BUFFER_SIZE) == 0)
+  {
+    cerr << timestamp << ": ERROR: Could not get the event type" << endl;
+    return;
+  }
+
+  if (event_notification_get_string(handle, notification,
+                                    "ARGS", args, EVENT_BUFFER_SIZE) == 0)
+  {
+    cerr << timestamp << ": ERROR: Could not get event arguments" << endl;
+    return;
+  }
+
+  istringstream line(args);
+  int sequence = 0;
+  line >> sequence;
+
+  cerr << timestamp << " name=" << name << " type=" << type
+       << " sequence=" << sequence << endl;
+}

install/boss-install.in

@@ -846,6 +846,8 @@ Phase "database", "Setting up database", sub {
 	if (!ExecQuiet("$MYSQLADMIN ping")) {
 	    PhaseSkip("mysqld already running");
 	}
+	# XXX make sure client startup was done (setting ldpath)
+	ExecQuietFatal("$RCDIR/1.mysql-client.sh start");
 	ExecQuietFatal("$RCDIR/2.mysql-server.sh start");
 	# Give mysqld some time to start, then make sure it did
 	sleep 5;

protogeni/lib/GeniAggregate.pm.in

@@ -51,6 +51,7 @@ my $VERIFYCRED	   = "$TB/sbin/verifygenicred";
 my $NODEREBOOT	   = "$TB/bin/node_reboot";
 my $VNODESETUP     = "$TB/sbin/vnode_setup";
 my $POWER          = "$TB/bin/power";
+my $OSLOAD         = "$TB/bin/os_load";
 my $SNMPIT         = "$TB/bin/snmpit";
 my $NAMEDSETUP     = "$TB/sbin/named_setup";
 my $EXPORTS_SETUP  = "$TB/sbin/exports_setup";
@@ -745,6 +746,7 @@ sub Start($$$)
     my %reboots  = ();
     my %vnodes   = ();
     my %poweron  = ();
+    my %reloads  = ();
 
     foreach my $sliver (@slivers) {
 	next
@@ -770,20 +772,87 @@ sub Start($$$)
 	    
 	    if ($node->isvirtnode()) {
 		$vnodes{$node->node_id} = $node;
+
+		# A virtnode on a shared physical node needs nothing else.
+		next
+		    if ($node->sharing_mode());
+
+		# But if non-shared, have to make sure that the phys node
+		# gets loaded.
+		my $physnodeid = $node->phys_nodeid();
+		next
+		    if (exists($poweron{$physnodeid}) ||
+			exists($reboots{$physnodeid}) ||
+			exists($reloads{$physnodeid}));
+		$node = Node->Lookup($physnodeid);
+		if (!defined($node)) {
+		    print STDERR "Could not lookup $physnodeid\n";
+		    return -1;
+		}
+	    }
+	    #
+	    # Look to see if local physical node was stopped (powered off).
+	    #
+	    if (!$node->isremotenode() &&
+		$sliver->state() eq "stopped") {
+		$poweron{$node->node_id} = $node;
 	    }
 	    else {
+		# node_reboot is smart enough to know that if a pnode
+		# is rebooted it can ignore the vnodes on it, so do
+		# not optimize this here.
+		$reboots{$node->node_id} = $node;
+	    }
+	    next
+		if (!$node->imageable());
+
+	    my $osinfo = OSinfo->Lookup($node->def_boot_osid());
+	    if (!defined($osinfo)) {
+		print STDERR "Could not get osinfo for $node\n";
+		return -1;
+	    }
+	    print STDERR "$node wants to boot $osinfo.\n";
+	    if ($osinfo->IsGeneric()) {
 		#
-		# Look to see if local physical node was stopped (powered off).
+		# Map generic OSID to the specific one.
 		#
-		if (!$node->isremotenode() &&
-		    $sliver->state() eq "stopped") {
-		    $poweron{$node->node_id} = $node;
+		my $tmp = $osinfo->ResolveNextOSID($experiment);
+		if (!defined($tmp)) {
+		    print STDERR "No next mapping for $osinfo on $node!\n";
+		    return -1;
+		}
+		print STDERR "  Mapping $osinfo on $node to $tmp\n";
+		$osinfo = $tmp;
+	    }
+	    #
+	    # Make sure this OSID is actually loaded on the machine.
+	    #
+	    my $isloaded = $node->IsOSLoaded($osinfo);
+	    if ($isloaded < 0) {
+		print STDERR
+		    "Error determining if $osinfo is loaded on $node\n";
+		return -1;
+	    }
+	    if (! $isloaded) {
+		print STDERR "  Setting up a reload for $node\n";
+		
+		my $image = $osinfo->MapToImage($node->type());
+		if (!defined($image)) {
+		    print STDERR "  No image for $osinfo on $node\n";
+		    return -1;
 		}
-		else {
-		    # node_reboot is smart enough to know that if a pnode
-		    # is rebooted it can ignore the vnodes on it, so do
-		    # not optimize this here.
-		    $reboots{$node->node_id} = $node;
+		if (!exists($reloads{$image->imageid()})) {
+		    $reloads{$image->imageid()} = [ ];
+		} 
+		push(@{ $reloads{$image->imageid()} }, $node);
+	    }
+	    else {
+		#
+		# Make sure boot is set correctly.
+		#
+		if ($node->OSSelect($osinfo, "def_boot_osid", 0)) {
+		    print STDERR "  Could not os_select $node to $osinfo\n";
+		    return -1;
 		}
 	    }
 	}
@@ -806,6 +875,21 @@ sub Start($$$)
     }
     %vnodes = %tmp;
 
+    #
+    # Setup the reloads. We do not reboot the nodes until below.
+    #
+    if (keys(%reloads)) {
+	foreach my $imageid (keys(%reloads)) {
+	    my @nodes = @{ $reloads{$imageid} };
+	    my @node_ids = map { $_->node_id() } @nodes;
+
+	    # No wait, no reboot. reload runs completely in the background.
+	    system("$OSLOAD -s -r -m $imageid @node_ids");
+	    return -1
+		if ($?);
+	}
+    }
+
     if ($version >= 2) {
 	if (system("$GENTOPOFILE $pid $eid")) {
 	    print STDERR "$GENTOPOFILE failed\n";
@@ -836,7 +920,7 @@ sub Start($$$)
 		}
 	    }
 	}
-	system("$SNMPIT -t $pid $eid");
+	system("$SNMPIT -q -t $pid $eid");
 	if ($?) {
 	    print STDERR "$SNMPIT failed\n";
 	    return -1;
@@ -965,7 +1049,7 @@ sub Stop($$)
     %vnodes = %tmp;
 
     if ($version >= 2) {
-	system("$SNMPIT -r $pid $eid");
+	system("$SNMPIT -q -r $pid $eid");
 	if ($?) {
 	    print STDERR "$SNMPIT failed\n";
 	    return -1;

protogeni/lib/GeniCMV2.pm.in

@@ -92,7 +92,15 @@ my $API_VERSION = 2;
 #
 sub GetVersion()
 {
-    return GeniResponse->Create( GENIRESPONSE_SUCCESS, $API_VERSION );
+    my @input_rspec_versions = ( "0.1", "2" );
+    my $blob = {
+	"api" => $API_VERSION,
+	"level" => 1,
+	"input_rspec" => \@input_rspec_versions,
+	"output_rspec" => "0.1"
+    };
+
+    return GeniResponse->Create( GENIRESPONSE_SUCCESS, $blob );
 }
 
 #

protogeni/lib/GeniSliver.pm.in

@@ -764,14 +764,6 @@ sub Provision($;$)
 	if ($node->ModifyReservation({"genisliver_idx" => $self->idx()})) {
 	    return -1;
 	}
-
-	#
-	# Terrible place to do this. 
-	#
-	if ($node->OSSelect("<DEFAULT>", "def_boot_osid", 0) != 0) {
-	    print STDERR "Could not os_select $node to default\n";
-	    return -1;
-	}
     }
     return 0;
 }

protogeni/scripts/genclrcredentials.in

@@ -7,6 +7,10 @@
 use strict;
 use lib '@prefix@/lib';
 
+# Do this early so that we talk to the right DB.
+use vars qw($GENI_DBNAME $GENI_ISCLRHOUSE);
+BEGIN { $GENI_DBNAME = "geni-ch"; $GENI_ISCLRHOUSE = 1; }
+
 use GeniCredential;
 use GeniCertificate;
 use GeniAuthority;
@@ -67,7 +71,7 @@ if($numArgs !=2) {
 	print "Please specify <user-urn> <ch-urn> as command line arguments\n\n";
 }else{
 my $val =  CreateSpecialCredential @ARGV; 
-print STDERR $val->{"code"};
-print STDERR $val->{"value"};
-print $val->{"output"};
+#print $val->{"output"};
+#print $val->{"code"};
+print $val->{"value"};
 }

protogeni/test/clrhouselist.py → protogeni/test/list-ch.py

@@ -26,29 +26,31 @@ from M2Crypto import X509
 
 execfile( "test-common.py" )
 
+TYPE = None
+
+if len(REQARGS) == 1:
+    TYPE = REQARGS[0]
+else:
+    print "You must supply a TYPE (users|slices|authorities) to list"
+    sys.exit(1)
+    pass
+
+# sanity check on TYPE
+if TYPE not in ["users", "slices", "authorities"]:
+    print "TYPE must be one of users|slices|authorities"
+    sys.exit(1)
+
 #
 # Get a credential for myself, that allows me to do things at the SA.
 #
 mycredential = get_self_credential()
 
 #
-# Ask the clearinghouse for a list of slices
-#
-params = {}
-params["credential"] = mycredential
-params["type"]       = "slices"