libosload_mlnx_sn2410.pm.in 13.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2018 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#
package libosload_mlnx_sn2410;

use strict;
use English;
use Data::Dumper;
use Expect;

#
# Configure stuff
#
my $TB		= "@prefix@";
my $TESTMODE    = @TESTMODE@;

use libosload_switch;
use base qw(libosload_switch);
39
use libdb;
40
use libEmulab;
41
use Lan;
42 43
use emutil;
use EmulabConstants;
44
use libtblog_simple;
45 46 47 48 49 50 51

# Initial switch user/password. We change this as soon as we can.
my $ADMIN_USER    = "admin";
my $ADMIN_PSWD    = "admin";
my $USER_PROMPT   = '[standalone: master] > ';
my $ADMIN_PROMPT  = "[standalone: master] # ";
my $CONFIG_PROMPT = "[standalone: master] (config) # ";
52
my $CONN_TIMEOUT  = 20;
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
my $EXPECT_DEBUG_LOG    = "/tmp/osload_switch.log";

sub New($$$)
{
    my ($class, $parent, $type) = @_;

    my $self = $class->SUPER::New($parent, $type);
    bless($self, $class);

    return $self;
}

#
# Create an Expect object that spawns the ssh process to the switch.
#
68
sub createExpectObject($$)
69
{
70 71 72 73 74 75 76 77
    my ($self, $nodeobject) = @_;
    my $node_id = $nodeobject->node_id();

    # First try with protouser (the normal state). If that fails, it
    # is possible the switch is still in its post load state, with a
    # default password.
    return 0
	if ($self->createExpectObjectAux($nodeobject, 0) == 0);
78 79 80 81
    # Sleep for a while, sometimes sshd locks out the next login,
    # not really sure what is going on yet.
    $self->dprint(0,"$self createExpectObject($node_id): sleeping for a bit\n");
    sleep(20);
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
    return -1
	if ($self->createExpectObjectAux($nodeobject, 1));

    #
    # We got in on the default password. Lets change it!
    #
    my $cleartext = $self->DefaultPassword();
    my $salt      = "\$5\$" . substr(emutil::GenHash(), 0, 16) . "\$";
    my $encrypted = crypt($cleartext, $salt);
    my $protouser = PROTOUSER();

    my @config = (
	"username admin password 7 $encrypted",
	"username monitor password 7 $encrypted",
	# Need this account for snmpit.
	"username xmladmin password 7 $encrypted",
	"username xmluser password 7 $encrypted",
	"username monitor disable",
	"username xmluser disable",
	"username $protouser capability admin",
	"username $protouser full-name \"\"",
	"username $protouser password 7 $encrypted",
	# Need this to allow write
	"no zero-touch suppress-write",
	);

    # Install boss' root key into elabman for easier login.
    my $pubkey;
    GetSiteVar("node/ssh_pubkey", \$pubkey);
    if (defined($pubkey) && $pubkey ne '') {
	push(@config,
	     "ssh client user $protouser authorized-key sshv2 \"$pubkey\"");
    }

    if ($self->writeConfig($nodeobject, @config)) {
	return -1;
    }
    return 0;
}

sub createExpectObjectAux($$$)
{
    my ($self, $nodeobject, $initial) = @_;
125 126 127 128 129
    my $node_id = $nodeobject->node_id();
    my ($admin_user, $admin_pswd);
    my $prompt = "$node_id $USER_PROMPT";
    my $error = 0;

130
    if ($initial) {
131 132 133 134 135 136 137 138 139 140 141 142
	$admin_user = $ADMIN_USER;
	$admin_pswd = $ADMIN_PSWD;
    }
    else {
	$admin_user = PROTOUSER();
	$admin_pswd = $self->DefaultPassword();
	return -1
	    if (!defined($admin_pswd));
    }
    # Host keys change every reload, do not want to save them.
    my $spawn_cmd = "ssh -o userknownhostsfile=/dev/null ".
	"-l $admin_user $node_id";
143 144

    $self->dprint(0,"$self createExpectObject($node_id): $spawn_cmd\n");
145 146 147 148 149 150 151
    
    # Create Expect object and initialize it:
    my $exp = new Expect();
    if (!$exp) {
        # upper layer will check this
        return -1;
    }
152 153 154 155 156 157 158
    $exp->log_stdout(0);
    
    if ($self->debug() > 3) {
	$exp->log_stdout(1);
	# Copious debugging.
	$exp->exp_internal(1);
    }
159 160 161 162 163 164 165 166 167 168 169 170 171

    if (!$exp->spawn($spawn_cmd)) {
	warn "$self createExpectObject($node_id): ".
	    "Cannot spawn $spawn_cmd: $!\n";
	return -1;
    }
    $exp->raw_pty(1);

    $exp->expect($CONN_TIMEOUT,
		 ["Password:" =>
		  sub { my $e = shift;
			$e->send("$admin_pswd\n");
			exp_continue;}],
172
		 ["wizard for initial configuration" =>
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
		  sub { my $e = shift;
			$e->send("no\n");
			exp_continue;}],
		 ["Permission denied" =>
		  sub { $error = "password incorrect!";} ],
		 [timeout =>
		  sub { $error = "Timeout connecting to switch!";} ],
		 ["-ex", "$node_id $USER_PROMPT"  =>
		   sub { my $e = shift;
			 $e->send("enable\n");
			 exp_continue;}],
		 ["-ex", "$node_id $ADMIN_PROMPT"  =>
		   sub { my $e = shift;
			 $e->send("configure terminal\n");
			 exp_continue;}],
		 "$node_id $CONFIG_PROMPT"
		);

    if (!$error && $exp->error()) {
	$error = $exp->error();
    }

    if ($error) {
	warn "$self createExpectObject($node_id): ".
	    "Could not connect to switch: $error\n";
	return -1;
    }
200
    $self->expectobj($nodeobject, $exp);
201 202 203 204 205 206
    return 0;
}

#
# Generate initial switch config for MLNX-OS. 
#
207
sub generateConfig($$)
208
{
209
    my ($self,$nodeobject) = @_;
210 211
    my $node_id       = $nodeobject->node_id();
    my $experiment    = $nodeobject->Reservation();
212
    my @config        = ();
213 214

    #
215 216 217
    # Once a switch is dragged into an experiment, we are going to add a
    # user account set to the current node root_password. We leave the
    # elabman account alone, that is our way back in.
218
    #
219
    my $cleartext = $nodeobject->root_password();
220 221 222 223
    my $salt      = "\$5\$" . substr(emutil::GenHash(), 0, 16) . "\$";
    my $encrypted = crypt($cleartext, $salt);

    #
224 225
    # Experiment creator account. Not going to create accounts for all
    # users at this time, does not seem necessary.
226
    #
227
    if (defined($experiment)) {
228 229 230 231 232 233
	my $user = $experiment->GetSwapper();
	if (!defined($user)) {
	    print STDERR "generateConfig: No swapper defined for $experiment\n";
	}
	else {
	    my $uid  = $user->uid();
234
	    push(@config,
235 236
		 "username $uid capability admin",
		 "username $uid password 7 $encrypted");
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
	}

	#
	# Find all the ports that are in use on this switch. We want to enable
	# those ports and put them into vlan 1. This will give the user a basic
	# single broadcast domain as a starting point. We will turn on STP to
	# avoid loops created by the user (cause that is gonna happen!).
	#
	my @vlanports = ();
	my @portlines = ();
	my @vlans;
	if (VLan->ExperimentVLans($experiment, \@vlans) != 0) {
	    print STDERR "Could not get list of all vlans for $experiment\n";
	    return -1;
	}
	foreach my $vlan (@vlans) {
	    my @members;
254
	    
255 256 257 258 259 260
	    next
		if ($vlan->type() ne "wire");

	    if ($vlan->MemberList(\@members)) {
		print STDERR "Could not get member list of members for $vlan\n";
		return -1;
261
	    }
262 263 264 265 266 267 268 269 270 271 272 273 274 275
	    foreach my $member (@members) {
		my ($membnode_id, $membiface);
		if ($member->GetNodeIface(\$membnode_id, \$membiface)) {
		    print STDERR "Could not get node/iface for $member\n";
		    return -1;
		}
		next
		    if ($membnode_id->node_id() ne $node_id);

		my $interface = Interface->LookupByIface($nodeobject,
							 $membiface);
		if (!defined($interface)) {
		    print STDERR "Could not lookup interface for $member\n";
		    return -1;
276
		}
277 278 279
		my $card  = $interface->card();
		my $port  = $interface->port();

280 281 282 283 284 285 286 287 288 289 290
		my @pconfig = (
		    "interface ethernet $card/$port",
		    "  shutdown",
		    "  switchport mode access",
		    "  speed 10G no-autoneg");

		if ($vlan->GetRole() eq "trunk") {
		    push(@pconfig, 
			 "  spanning-tree guard loop");
		}
		push(@pconfig, 
291 292
		     "  no shutdown",
		     "  exit");
293 294
		
		push(@portlines, @pconfig);
295 296
	    }
	}
297 298
	if (@portlines) {
	    push(@config,
299
		 "spanning-tree mode rst",
300 301
		 @portlines);
	}
302
    }
303 304
    push(@config, 
	 "write memory",
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
	);

    if ($self->debug()) {
	foreach my $cl (@config) {
	    $self->dprint(4,"Reconfigure($node_id): config line: $cl\n");
	}
    }
    return @config;
}

#
# Write a configuration to the switch using the expect object.
#
sub writeConfig($$@)
{
    my ($self, $nodeobject, @config) = @_;
    my $node_id = $nodeobject->node_id();
322
    my $exp     = $self->expectobj($nodeobject);
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364

    $self->dprint(0,"$self writeConfig($node_id): \n");

    #
    # Write each line, look for some that resembles an error.
    #
    foreach my $line (@config) {
	my $output;
	my $error;
	
	$exp->clear_accum(); # Clean the accumulated output, as a rule.
	$exp->send($line . "\n");
	
	$exp->expect($CONN_TIMEOUT,
		     [qr/% (.*)\n/,
		      sub { my $e = shift;
			    $error = ($e->matchlist)[0];}],
		     ["-ex", "$node_id $CONFIG_PROMPT"  =>
		      sub { my $e = shift;
			    $output = $e->before();}],
		     );

	if (!$error && $exp->error()) {
	    $error = $exp->error();
	}
	if ($error) {
	    $self->dprint(0, "$self writeConfig($node_id): ".
			 "cmd: '$line', error: '$error'\n");
	    return (1, $error);
	}
    }
    return (0);
}

#
# It takes about 30 minutes to write MLNX to the flash drive. Add some
# padding of course.
#
sub MaxWaitTime($$)
{    
    my ($self, $nodeobject) = @_;

365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
    return 60 * 60;
}

#
# Reconfigure.
#
sub DeviceReconfigure($$)
{
    my ($self, $nodeobject) = @_;
    my $node_id    = $nodeobject->node_id();
    my $experiment = $nodeobject->Reservation();

    my @config = $self->generateConfig($nodeobject);	
    if (!@config) {
 	tberror "$self: Reconfigure($node_id): could not generate config!\n";
 	return -1;
    }
382 383 384 385

    # Sleep for a few seconds to let switch settle down after booting.
    sleep(10);
    
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
    if ($self->createExpectObject($nodeobject)) {
	return -1;
    }
    if ($self->writeConfig($nodeobject, @config)) {
	return -1;
    }
    if (!defined($experiment)) {
	$self->expectobj($nodeobject)->soft_close();
	return 0;
    }
    #
    # Deal with SSH keys.
    #
    my $nonlocal = 0;
    my @keys = ();
    my $user;

    #
    # Nonlocal (geni experiments) are a little more trouble, since the
    # swapper is "geniuser" and the keys are in nonlocal_user_pubkeys.
    #
    if ($experiment->geniflags()) {
	#
	# Look at the project, if its a nonlocal project then we need
	# the nonlocal keys. But with PROTOGENI_LOCALUSER, we might
	# have a local project (and thus a local user).
	#
	my $project = $experiment->GetProject();
	
	if (!defined($project)) {
	    print STDERR "generateConfig: No project for $experiment\n";
	    return 0;
	}
	$nonlocal = $project->IsNonLocal();
	$user = $experiment->GetCreator();
    }
    else {
	$user = $experiment->GetSwapper();
    }
    if (!defined($user)) {
	print STDERR "generateConfig: No swapper defined for $experiment\n";
	return 0;
    }
    my $uid = $user->uid();

    # Now the keys.
    if ($nonlocal) {
	#
	# Grab all the nonlocal users and then find the keys for the
	# experiment creator.
	#
	my $tmp;
	$experiment->NonLocalUsers(\$tmp);
	if (defined($tmp)) {
	    foreach my $ref (@{$tmp}) {
		if ($ref->{'urn'} eq $user->nonlocal_id()) {
		    foreach my $key (@{ $ref->{'keys'} }) {
			push(@keys, $key->{'key'})
		    }
		    $uid = $ref->{'login'};
		    last;
		}
	    }
	}
    }
    else {
	$user->GetSSHKeys(\@keys);
    }

    #
    # ssh-rsa keys only, need to have at least one.
    #
    my @tmp = ();
    foreach my $key (@keys) {
	next
	    if ($key !~ /^ssh-rsa/);
	push(@tmp, $key);
    }
    @keys = @tmp;
    if (!@keys) {
	print STDERR "No ssh keys for $uid\n";
	return 0;
    }
    @config = ();

    #
    # For geni experiments, we have to create the user since the uid
    # is different (see above).
    #
    if ($nonlocal) {
	@config = ("username $uid capability admin");
    }
    foreach my $key (@keys) {
	push(@config,
	     "ssh client user $uid authorized-key sshv2 \"$key\"");
    }
    push(@config, "write memory");

    if ($self->writeConfig($nodeobject, @config)) {
	print STDERR "Could not send ssh keys to $node_id\n";
	return -1;
    }
    return 0;
489 490 491 492 493 494 495 496
}

#
# Try to reboot a switch, first looking to see if its running the NOS,
# and if it is doing it the NOS way. If not running NOS, call up a level
# for a standard reboot, which will work fine if its runnng ONIE, failing
# back to a power cycle. 
#
497
sub Reboot($$)
498 499 500 501 502 503 504 505 506 507 508 509
{
    my ($self, $nodeobject) = @_;
    my $node_id  = $nodeobject->node_id();
    my $error;

    #
    # If its dead, use normal reboot which will power cycle.
    #
    if (! $self->Pingable($nodeobject)) {
	goto reboot;
    }
    # Running something else, we use normal reboot.
510
    if (! $self->RunningNOS($nodeobject)) {
511 512 513 514 515
	goto reboot;
    }
    if ($self->createExpectObject($nodeobject)) {
	goto reboot;
    }
516
    my $exp = $self->expectobj($nodeobject);
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548

    $exp->clear_accum();  # Clean the accumulated output, as a rule.
    $exp->send("\n");     # Get us a prompt.
	
    $exp->expect($CONN_TIMEOUT,
		 [qr/% (.*)\n/,
		  sub { my $e = shift;
			$error = ($e->matchlist)[0];}],
		 ["-ex", "$node_id $CONFIG_PROMPT"  =>
		  sub { my $e = shift;
			$e->send("exit\n");
			exp_continue;}],
		 ["-ex", "$node_id $USER_PROMPT"  =>
		   sub { my $e = shift;
			 $e->send("enable\n");
			 exp_continue;}],
		 ["-ex", "$ADMIN_PROMPT"  =>
		   sub { my $e = shift;
			 $e->send("reload\n");
			 exp_continue;}],
		 [qr/save first\? \[yes\] / =>
		   sub { my $e = shift;
			 $e->send("no\n");}],
		);

    if (!$error && $exp->error()) {
	$error = $exp->error();
    }
    if ($error) {
	$self->dprint(0, "$self RebootNOS($node_id): error: '$error'\n");
	goto reboot;
    }
549
    TBSetNodeEventState($node_id, TBDB_NODESTATE_SHUTDOWN);
550 551 552
    return 0;

  reboot:
553
    if (0) {
554 555
	return 0;
    }
556
    return $self->SUPER::Reboot($nodeobject);
557 558 559
}

1;