Commit ea496d44 authored by David Johnson's avatar David Johnson

Add per-experiment switch support.

Per-experiment switch stacks only come into being if the experiment
actually has a switch allocated to it.  If not, tbswap and snmpit
should function unchanged.  If there is a per-experiment stack that needs
configuration, we first invoke normal snmpit in the normal place, but we
use the new snmpit option `--skip-supplied' in combination with -S to skip
the per-experiment stack.  We then configure the per-experiment stack by
itself with -S after os_setup has completed.

There are some new functions in the db backend stuff to create, modify,
and remove per-experiment switches.

There is some new code in snmpit to do the --skip-supplied filtering.  I
also put all the -S, -i, and --skip-supplied stuff into portstats...
because we also can't call portstats on a per-experiment switch in tbswap;
otherwise it will hang and/or fail.
parent c75427ef
......@@ -2696,6 +2696,56 @@ sub NodeList($;$$)
return @nodes;
}
#
# Return list of experiment switches (objects or just names)
#
sub SwitchList($;$$)
{
my ($self, $namesonly, $includevirtual) = @_;
my @nodenames = ();
# Must be a real reference.
return undef
if (! ref($self));
$includevirtual = 0
if (!defined($includevirtual));
$namesonly = 0
if (!defined($namesonly));
my $pid = $self->pid();
my $eid = $self->eid();
my $query_result =
DBQueryWarn("select r.node_id,nt.isvirtnode from reserved as r ".
"left join nodes as n on n.node_id=r.node_id ".
"left join node_types as nt on nt.type=n.type ".
"where r.pid='$pid' and r.eid='$eid' and nt.isswitch=1");
return undef
if (!$query_result);
return ()
if (!$query_result->numrows);
while (my ($nodeid,$isvirt) = $query_result->fetchrow_array()) {
next
if ($isvirt && !$includevirtual);
push(@nodenames, $nodeid);
}
return @nodenames
if ($namesonly);
my @nodes = ();
foreach my $nodeid (@nodenames) {
my $node = Node->Lookup($nodeid);
if (!defined($node)) {
print STDERR "*** Could not map $nodeid to its object\n";
return undef;
}
push(@nodes, $node);
}
return @nodes;
}
#
# Copy log files to long term storage.
#
......
......@@ -59,6 +59,9 @@ use vars qw(@ISA @EXPORT);
TBWideareaNodeID TBTipServers
TBSiteVarExists TBGetSiteVar TBSetSiteVar
TBActivityReport GatherAssignStats
AddPerExperimentSwitchStack UpdatePerExperimentSwitchStack
DeletePerExperimentSwitchStack GetPerExperimentSwitchStack
GetPerExperimentSwitchStackName
TBAvailablePCs
max min
hash_recurse array_recurse hash_recurse2 array_recurse2
......@@ -2523,6 +2526,185 @@ sub GatherAssignStats($$%)
"where idx=$rsrcidx");
}
sub GetPerExperimentSwitchStackName($) {
my ($expt) = @_;
return "ExpStack" . $expt->idx();
}
#
# Get per-experiment switch stack id, the leader, and the switches.
#
sub GetPerExperimentSwitchStack($) {
my ($expt) = @_;
my $leader;
my @switches;
my $stack_id = GetPerExperimentSwitchStackName($expt);
my $qres = DBQueryFatal("select leader from switch_stack_types" .
" where stack_id='$stack_id'");
if (!defined($qres) || $qres->numrows() == 0) {
if (wantarray) {
return ();
}
else {
return undef;
}
}
($leader) = $qres->fetchrow_array();
$qres = DBQueryFatal("select node_id from switch_stacks" .
" where stack_id='$stack_id'");
while (my ($node_id) = $qres->fetchrow_array()) {
push @switches, $node_id
if ($node_id ne $leader);
}
return ($stack_id,$leader,@switches);
}
#
# Add a per-experiment switch stack type, and add switches to that stack.
#
sub AddPerExperimentSwitchStack($@;$$$) {
my ($expt,@switches,$leader,$snmp_community,$min_vlan,$max_vlan) = @_;
my $scomm = $snmp_community;
my $stack_id = GetPerExperimentSwitchStackName($expt);
return 1
if (!@switches);
if (!defined($leader)) {
$leader = $switches[0];
}
if (!defined($snmp_community)) {
$scomm = int(rand(100000000));
}
if (!defined($min_vlan)) {
$min_vlan = 256;
}
if (!defined($max_vlan)) {
$max_vlan = 999;
}
my $query = "replace into switch_stack_types" .
" (stack_id,stack_type,supports_private,single_domain," .
" snmp_community,min_vlan,max_vlan,leader)" .
" values ( '$stack_id','generic',0,0," .
" '$scomm',$min_vlan,$max_vlan,'$leader')";
DBQueryFatal($query);
foreach my $switch (@switches) {
my $is_primary = 0;
if ($switch eq $leader) {
$is_primary = 1;
}
# for each switch, if the caller didn't supply snmp_community, then
# check node_type_attributes to see if there is a fixed community we
# must use -- i.e., if we can't reconfig this switch with a generated
# one.
if (!defined($snmp_community)) {
my $qres = DBQueryFatal("select nta.attrvalue" .
" from nodes as n " .
" left join node_type_attributes as nta" .
" on n.type=nta.type " .
" where n.node_id='$switch'" .
" and nta.attrkey='snmp_community'");
if ($qres && $qres->numrows()) {
($scomm,) = $qres->fetchrow_array();
}
}
DBQueryFatal("replace into switch_stacks (node_id,stack_id,is_primary,snmp_community)" .
" values ('$switch','$stack_id',$is_primary,'$scomm')");
}
return 0;
}
#
# Update a per-experiment switch stack type (i.e., during an expt modify).
#
sub UpdatePerExperimentSwitchStack($@) {
my ($expt,@switches) = @_;
my $stack_id = GetPerExperimentSwitchStackName($expt);
if (!@switches) {
DBQueryFatal("delete from switch_stacks where stack_id='$stack_id'");
return 0;
}
my $qres = DBQueryFatal("select leader,snmp_community from switch_stack_types" .
" where stack_id='$stack_id'");
if (!$qres || !$qres->numrows) {
tbwarn("No such switch stack id '$stack_id'!");
return 1;
}
my ($leader,$scomm) = $qres->fetchrow_array;
my $need_new_leader = 1;
foreach my $switch (@switches) {
if ($switch eq '$leader') {
$need_new_leader = 0;
last;
}
}
if ($need_new_leader) {
DBQueryFatal("delete from switch_stacks" .
" where stack_id='$stack_id' and node_id='$leader'");
$leader = $switches[0];
}
# delete them all, and add them all back -- easier than querying and
# diff'ing
DBQueryFatal("delete from switch_stacks where stack_id='$stack_id'");
foreach my $switch (@switches) {
my $is_primary = 0;
if (1 || $switch eq $leader) {
$is_primary = 1;
}
# for each switch, if the caller didn't supply snmp_community, then
# check node_type_attributes to see if there is a fixed community we
# must use -- i.e., if we can't reconfig this switch with a generated
# one.
if (!defined($scomm) || $scomm eq '') {
my $qres = DBQueryFatal("select nta.attrvalue" .
" from nodes as n " .
" left join node_type_attributes as nta" .
" on n.type=nta.type " .
" where n.node_id='$switch'" .
" and nta.attrkey='snmp_community'");
if ($qres && $qres->numrows()) {
($scomm,) = $qres->fetchrow_array();
}
}
DBQueryFatal("replace into switch_stacks (node_id,stack_id,is_primary)" .
" values ('$switch','$stack_id',$is_primary)");
}
return 0;
}
#
# Delete a per-experiment switch stack type.
#
sub DeletePerExperimentSwitchStack($) {
my ($expt) = @_;
my $stack_id = GetPerExperimentSwitchStackName($expt);
DBQueryFatal("delete from switch_stacks where stack_id='$stack_id'");
DBQueryFatal("delete from switch_stack_types where stack_id='$stack_id'");
return 0;
}
sub max ( $$ ) {
return ($_[0] > $_[1] ? $_[0] : $_[1]);
}
......
......@@ -49,6 +49,15 @@ Usage: $0 [-h] <-p | <pid> <eid> > [vname ...] [vname:port ...]
eid should be given when using this option.
-s Ports are specified in switch.port syntax
-C List control net, rather than experimental net, ports
-i <device>
Operate on <device>, overriding default device list. Can be
given multiple times
-S <stack>
Operate on the given stack, instead of the default of the
experimental network
--skip-supplied
If the -S option was given, start with the default stacks,
but skip any switches in the stacks listed in -S.
If only pid and eid are given, prints out information about all ports in the
experiment. Otherwise, output is limited to the nodes and/or ports given.
......@@ -69,7 +78,8 @@ END
#
my %opt = ();
Getopt::Long::Configure("no_ignore_case");
GetOptions(\%opt,'h','a','e','p','b','z','q','c','s','C');
GetOptions(\%opt,'h','a','e','p','b','z','q','c','s','C','i=s@','S=s@',
'skip-supplied');
if ($opt{h}) {
exit &usage;
......@@ -286,8 +296,55 @@ elsif ($opt{p}) {
#
my %portMap = mapPortsToDevices(@ports);
#
# User-supplied switch lists
#
# See if we should we flip the behavior of -S and -i ?
my $skip_supplied = 0;
if ($opt{'skip-supplied'} && $opt{S}) {
$skip_supplied = 1;
}
my @supplied_switches = ();
my @supplied_stacks = ();
my $supplied_switches = 0; # Whether -i or -S was given
if ($opt{i}) {
$supplied_switches = 1;
push @supplied_switches, @{$opt{i}};
}
if ($opt{S}) {
foreach my $stack (@{$opt{S}}) {
$supplied_switches = 1;
my @switches = getSwitchesInStack($stack);
if (@switches) {
push @supplied_stacks, $stack;
push @supplied_switches, @switches;
}
else {
print STDERR "Warning: no such switch stack $stack, skipping!";
}
}
}
@supplied_switches = uniq(@supplied_switches);
@supplied_stacks = uniq(@supplied_stacks);
my %supplied_switch_map = ();
foreach my $ss (@supplied_switches) {
$supplied_switch_map{$ss} = $ss;
}
my @stats;
DEVICE: foreach my $name (keys %portMap) {
if ($supplied_switches && !$skip_supplied) {
next
if (!exists($supplied_switch_map{$name}));
}
elsif ($skip_supplied) {
next
if (exists($supplied_switch_map{$name}));
}
my @ports = @{$portMap{$name}};
my %oidwarned = ();
......
......@@ -36,6 +36,7 @@ use snmpit_remote;
# Protos
sub parseStatusString($);
sub filterList($$);
sub debug($);
sub doListVlans($);
sub doListPorts($);
......@@ -107,6 +108,8 @@ General:
given multiple times
-S <stack> Operate on the given stack, instead of the default of the
experimental network
--skip-supplied If the -S option was given, start with the default stacks,
but skip any switches in the stacks listed in -S.
--redirect-err Redirect STDERR to STDOUT, for easier capturing in logfiles
VLAN Control:
......@@ -187,7 +190,7 @@ GetOptions(\%opt,
'y=s','x=s','z=s','F','L=s','O', 'D', 'R', 'f', 'X', 'Z', 'vlan_tag=i',
'of-disable=s', 'of-enable=s', 'of-controller=s', 'of-listener=s',
'o=s@{1,1}', 'redirect-err', 'blockmode', 'syncvlans', 'impotent',
'shadow');
'shadow', 'skip-supplied');
if ($opt{h}) {
exit &usage;
......@@ -525,6 +528,13 @@ if (!@commands) {
#
# User-supplied switch lists
#
# See if we should we flip the behavior of -S and -i ?
my $skip_supplied = 0;
if ($opt{'skip-supplied'} && $opt{S}) {
$skip_supplied = 1;
}
my @supplied_switches = ();
my @supplied_stacks = ();
my $supplied_switches = 0; # Whether -i or -S was given
......@@ -677,17 +687,18 @@ COMMAND: foreach my $command (@commands) {
SWITCH: for ($operation) {
(/listvlans/ || /getstats/ || /vlannumber/ || /synchleader/ ||
/syncvlans/) && do {
@devicenames = $supplied_switches?
@devicenames = ($supplied_switches && !$skip_supplied)?
@supplied_switches : getTestSwitches();
last;
};
(/listports/) && do {
@devicenames = $supplied_switches? @supplied_switches :
(@ports? getDeviceNames(@ports) : getTestSwitches());
@devicenames = ($supplied_switches && !$skip_supplied)?
@supplied_switches :
(@ports? getDeviceNames(@ports) : getTestSwitches());
last;
};
( /make/ ) && do {
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
@devicenames = @supplied_switches;
} elsif (@ports) {
# Have to operate on whole stacks so that trunks work
......@@ -704,7 +715,7 @@ COMMAND: foreach my $command (@commands) {
# vlan from the db
# if others fail, default to experimental switches
my ($vlan_name) = @args;
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
debug("Remove: using supplied switches\n");
@devicenames = @supplied_switches;
}
......@@ -739,7 +750,7 @@ COMMAND: foreach my $command (@commands) {
if (!$quiet);
next COMMAND;
}
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
debug("Tables: using supplied switches\n");
@devicenames = @supplied_switches;
@vlans = filterVlansBySwitches(\@supplied_switches, @vlans);
......@@ -754,6 +765,17 @@ COMMAND: foreach my $command (@commands) {
debug("Tables: list from database: " . join(",",@devicenames) .
"\n");
}
if ($skip_supplied) {
debug("Tables: skipping supplied switches\n");
@devicenames = filterList(\@devicenames,\@supplied_switches);
@vlans = filterVlansBySwitches(\@devicenames, @vlans);
if (!@devicenames || !@vlans) {
print "snmpit: $pid/$eid has no VLANs to create ".
"on skip-filtered device list, skipping\n"
if (!$quiet);
next COMMAND;
}
}
if (scalar(@devicenames == 0)) {
debug("Tables: falling back to test swtiches\n");
@devicenames = getTestSwitches();
......@@ -785,7 +807,7 @@ COMMAND: foreach my $command (@commands) {
"created on switches\n ... skipping\n";
next COMMAND;
}
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
debug("Reset: using supplied switches\n");
@devicenames = @supplied_switches;
@vlans = filterVlansBySwitches(\@supplied_switches, @vlans);
......@@ -801,6 +823,18 @@ COMMAND: foreach my $command (@commands) {
"\n");
}
if ($skip_supplied) {
debug("Reset: skipping supplied switches\n");
@devicenames = filterList(\@devicenames,\@supplied_switches);
@vlans = filterVlansBySwitches(\@devicenames, @vlans);
if (!@devicenames || !@vlans) {
print "snmpit: $pid/$eid has no VLANs to reset ".
"on skip-filtered device list, skipping\n"
if (!$quiet);
next COMMAND;
}
}
# Fallthrough - if we haven't found any switches yet, operate
# on the whole experimetnal net
if (scalar(@devicenames) == 0) {
......@@ -816,7 +850,7 @@ COMMAND: foreach my $command (@commands) {
my @oldvlans;
VLan->StaleVlanList($experiment,\@oldvlans);
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
die("-X and -i cannot be used together\n");
}
else {
......@@ -825,6 +859,19 @@ COMMAND: foreach my $command (@commands) {
debug("Sync: list from database: " . join(",",@devicenames) .
"\n");
}
if ($skip_supplied) {
debug("Sync: skipping supplied switches (dangerous!)\n");
@devicenames = filterList(\@devicenames,\@supplied_switches);
@vlans = filterVlansBySwitches(\@devicenames, @vlans);
if (!@devicenames || !@vlans) {
print "snmpit: $pid/$eid has no VLANs to sync ".
"on skip-filtered device list, skipping\n"
if (!$quiet);
next COMMAND;
}
}
# Fallthrough - if we haven't found any switches yet, operate
# on the whole experimental net
if (scalar(@devicenames) == 0) {
......@@ -834,7 +881,7 @@ COMMAND: foreach my $command (@commands) {
last;
};
(/portcontrol/ || /trunkdisable/ || /portstatus/) && do {
@devicenames = $supplied_switches?
@devicenames = ($supplied_switches && !$skip_supplied)?
@supplied_switches : getDeviceNames(@ports);
last;
};
......@@ -847,14 +894,14 @@ COMMAND: foreach my $command (@commands) {
# Safety check - cannot be used with -i . We have to operate on
# all experimental switches
#
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
die "-c and -i or -S cannot be used together\n";
}
@devicenames = getTestSwitches();
last;
};
(/trunkenable/) && do {
@devicenames = $supplied_switches?
@devicenames = ($supplied_switches && !$skip_supplied)?
@supplied_switches : getDeviceNames(@ports);
@vlans = @optvlanids;
last;
......@@ -871,7 +918,7 @@ COMMAND: foreach my $command (@commands) {
if ($args{vlan}) {
@vlans = ($args{vlan});
}
@devicenames = $supplied_switches?
@devicenames = ($supplied_switches && !$skip_supplied)?
@supplied_switches : getDeviceNames(@ports);
@args = %args;
last;
......@@ -881,7 +928,7 @@ COMMAND: foreach my $command (@commands) {
# Same to 'remove', we find stacks by VLAN
my ($vlanobj, $stack);
if ($supplied_switches) {
if ($supplied_switches && !$skip_supplied) {
debug("Openflow operations: using supplied switches\n");
@devicenames = @supplied_switches;
} elsif (defined($experiment) &&
......@@ -897,6 +944,13 @@ COMMAND: foreach my $command (@commands) {
};
}
#
# No matter what operation we're doing, we're safe to filter the
# devicenames list now!
#
@devicenames = filterList(\@devicenames,\@supplied_switches)
if ($skip_supplied);
debug("Device names: " . join(",",@devicenames) . "\n");
debug("Ports: " . join(",",@ports) . "\n");
......@@ -3350,7 +3404,28 @@ sub doTrunkDisable($$) {
if (!$errors);
return $errors;
}
#
# Simple little list filtering util function.
#
sub filterList($$) {
my ($srclist,$filterlist) = @_;
my @retval = ();
foreach my $item (@$srclist) {
my $found = 0;
foreach my $filter (@$filterlist) {
if ($item eq $filter) {
$found = 1;
last;
}
}
push @retval, $item
if (!$found);
}
return @retval;
}
#
# Openflow lock and functions
......
......@@ -524,6 +524,17 @@ sub doSwapout($) {
}
}
#
# Grab our per-experiment switch stack name.
#
my @expswitches = $experiment->SwitchList(1);
my ($perexpstack,$leader,@curswitches) = \
GetPerExperimentSwitchStack($experiment);
my $stackarg = "";
if (defined($perexpstack) && (@expswitches || @curswitches)) {
$stackarg = "-S $perexpstack --skip-supplied";
}
#
# Clean up any VLANs in experiment.
#
......@@ -536,7 +547,7 @@ sub doSwapout($) {
my $tagopt = ($type != MODIFY ? "-C" : "");
TBDebugTimeStamp("snmpit started");
print STDERR "Removing VLANs.\n";
if (system("snmpit --redirect-err $tagopt -r $pid $eid")) {
if (system("snmpit $stackarg --redirect-err $tagopt -r $pid $eid")) {
tberror({type => 'secondary', severity => SEV_SECONDARY,
error => ['vlan_reset_failed']},
"Failed to reset VLANs");
......@@ -558,7 +569,7 @@ sub doSwapout($) {
}
if (@stale) {
print "Removing stale vlans @stale\n";
system("snmpit --redirect-err -f -C ".
system("snmpit $stackarg --redirect-err -f -C ".
join(" ", map("-o $_", @stale)));
if ($?) {
tberror({type => 'summary', severity => SEV_SECONDARY,
......@@ -568,6 +579,15 @@ sub doSwapout($) {
}
}
}
#
# Remove per-experiment switch stacks -- don't bother to check if there
# are any; it's cheap.
#
if ($type >= RETRY && defined($perexpstack)) {
print STDERR "Removing per-experiment switch stack.\n";
DeletePerExperimentSwitchStack($experiment);
}
}
if ($type >= MODIFY) {
......@@ -956,6 +976,30 @@ sub doSwapin($) {
return 1
}
#
# Create/update per-experiment switch stacks if necessary.
# Still haven't done any vlan synch for modify.
#
my @expswitches = $experiment->SwitchList(1);
my ($perexpstack,$leader,@curswitches) = \
GetPerExperimentSwitchStack($experiment);
my $stackarg = "";
if (($type > MODIFY && @expswitches)
|| ($type == MODIFY && (!defined($perexpstack) && @expswitches))) {
print STDERR "Creating per-experiment switch stack.\n";
AddPerExperimentSwitchStack($experiment,@expswitches);
$perexpstack = GetPerExperimentSwitchStackName($experiment);
}
elsif ($type == MODIFY && defined($perexpstack)
&& (@expswitches || @curswitches)) {
print STDERR "Updating per-experiment switch stack.\n";
UpdatePerExperimentSwitchStack($experiment,@expswitches);
}
if (defined($perexpstack) && (@expswitches || @curswitches)) {
print STDERR "Will configure per-experiment switch stack ($perexpstack) later.\n";
$stackarg = "-S $perexpstack --skip-supplied";
}
#
# When doing a modify, we have to compare vlans to determine which
# vlans actually changed and need to be deleted, before processing
......@@ -986,7 +1030,7 @@ sub doSwapin($) {
if ($ELABINELAB || !$syncvlans) {
if (@diff) {
print "Removing obsolete vlans @diff\n";
system("snmpit --redirect-err -f -C ".
system("snmpit $stackarg --redirect-err -f -C ".
join(" ", map("-o $_", @diff)));
if ($?) {
tberror({type => 'summary', severity => SEV_SECONDARY,
......@@ -994,17 +1038,36 @@ sub doSwapin($) {
"Failed to remove obsolete VLANs.");
return 1;
}
if (defined($perexpstack)) {
system("snmpit -S $perexpstack --redirect-err -f -C ".
join(" ", map("-o $_", @diff)));
if ($?) {
tberror({type => 'summary', severity => SEV_SECONDARY,
error => ['vlan_setup_failed']},
"Failed to remove obsolete per-experiment VLANs.");
return 1;
}
}
}
}
else {
print "Synchronizing VLANs.\n";
system("snmpit --redirect-err -X $pid $eid");
system("snmpit $stackarg --redirect-err -X $pid $eid");
if ($?) {
tberror({type => 'summary', severity => SEV_SECONDARY,
error => ['vlan_setup_failed']},
"Failed to synchronize VLANs.");
return 1;
}
if (defined($perexpstack)) {
system("snmpit -S $perexpstack --redirect-err -X $pid $eid");
if ($?) {
tberror({type => 'summary', severity => SEV_SECONDARY,
error => ['vlan_setup_failed']},
"Failed to synchronize per-experiment VLANs.");
return 1;
}
}
}
}
......@@ -1335,7 +1398,7 @@ sub doSwapin($) {
($type != MODIFY || ($ELABINELAB || !$syncvlans))) {
print "Setting up VLANs.\n";
TBDebugTimeStamp("snmpit started");
if (system("snmpit --redirect-err -t $pid $eid")) {
if (system("snmpit $stackarg --redirect-err -t $pid $eid")) {
tberror({type => 'summary', severity => SEV_SECONDARY,
error => ['vlan_setup_failed']},
"Failed to set up VLANs.");
......@@ -1364,7 +1427,7 @@ sub doSwapin($) {
if ($type >= RETRY) {
print "Clearing port counters.\n";
TBDebugTimeStamp("portstats started");
if (system("portstats -z -a -q $pid $eid")) {
if (system("portstats $stackarg -z -a -q $pid $eid")) {
tbwarn "Failed to clear port counters.";
#
# This is a non-fatal error.
......@@ -1418,6 +1481,22 @@ sub doSwapin($) {
return 1;
}
#
# Run snmpit on the per-experiment switch stacks if os_setup didn't fail!
#
if (defined($perexpstack) && !$experiment->skipvlans() &&
($type != MODIFY || ($ELABINELAB || !$syncvlans))) {
print "Setting up per-experiment VLANs.\n";
TBDebugTimeStamp("snmpit started");
if (system("snmpit -S $perexpstack --redirect-err -t $pid $eid")) {
tberror({type => 'summary', severity => SEV_SECONDARY,
error => ['vlan_setup_failed']},
"Failed to set up per-experiment VLANs.");
return 1;
}
TBDebugTimeStamp("snmpit finished");
}
#
# Okay, start the event system now that we know all the nodes have
# rebooted (os_setup is done). This only takes a moment (puts itself
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment