Attempt to deal with hotplug errors.

clientside/tmcc/common/libutil.pm

@@ -27,7 +27,7 @@
 package libutil;
 use Exporter;
 @ISA    = "Exporter";
-@EXPORT = qw( ipToMac macAddSep fatal mysystem mysystem2
+@EXPORT = qw( ipToMac macAddSep fatal mysystem mysystem2 ExecQuiet
               findDNS setState isRoutable findDomain convertToMebi
               ipToNetwork CIDRmask untaintNumber untaintHostname
 	      GenFakeMac
@@ -279,6 +279,42 @@ sub mysystem2($)
     }
 }
 
+sub ExecQuiet($)
+{
+    my ($command) = @_;
+
+    print STDERR "ExecQuiet: '$command'\n";
+
+    my $output    = "";
+    
+    #
+    # This open implicitly forks a child, which goes on to execute the
+    # command. The parent is going to sit in this loop and capture the
+    # output of the child. We do this so that we have better control
+    # over the descriptors.
+    #
+    my $pid = open(PIPE, "-|");
+    if (!defined($pid)) {
+	print STDERR "ExecQuiet Failure; popen failed!\n";
+	return undef;
+    }
+    if ($pid) {
+	while (<PIPE>) {
+	    $output .= $_;
+	}
+	close(PIPE);
+    }
+    else {
+	open(STDERR, ">&STDOUT");
+	exec($command);
+    }
+    print STDERR $output;
+    if ($?) {
+	print STDERR "Command failed: $?\n";
+    }
+    return $output;
+}
+
 #
 # Generate a hopefully unique mac address that is suitable for use
 # on a shared node where uniqueness matters.

clientside/tmcc/linux/xen/libvnode_xen.pm

@@ -1549,7 +1549,23 @@ sub vnodeBoot($$$$)
     libutil::setState("BOOTING");
 
     # and finally, create the VM
-    mysystem("xm create $config");
+    my $output = ExecQuiet("xm create $config");
+    return -1
+	if (!defined($output));
+
+    # We have a problem with intermittent failures. We retry if we
+    # get a hotplug error after a small wait.
+    if ($? && $output =~ /hotplug/i) {
+	print "Hotplug Error: Retrying after a short wait.\n";
+	sleep(10);
+	
+	my $output = ExecQuiet("xm create $config");
+	return -1
+	    if (!defined($output));
+    }
+    return -1
+	if ($?);
+
     print "Created virtual machine $vnode_id\n";
     return 0;
 }