Support adding and removing blockstores across reboot.

This is the client-side of our current "swapmod" goal. You can add and
remove blockstores, but you cannot modify the attributes of an existing
blockstore.

clientside/tmcc/common/config/rc.storage

@@ -131,19 +131,86 @@ sub doboot()
 	fatal("Error grabbing storage config!");
     }
 
-    if (!@cmds) {
+    #
+    # We could have just rebooted as the result of a swapmod operation.
+    # We read in any old config so we can see if we have added or removed
+    # any blockstores.
+    #
+    my $ocmdref = [];
+    if (-r "$OLDCONFIG") {
+	$ocmdref = eval { Storable::retrieve($OLDCONFIG); };
+	if ($@ || !$ocmdref) {
+	    warn "*** Could not read old config, ignoring...\n";
+	    unlink($OLDCONFIG);
+	    $ocmdref = [];
+	}
+    }
+
+    #
+    # No blockstores old or new.
+    #
+    if (!@cmds && !@$ocmdref) {
 	#warn("*** No storageconfig output - nothing to do");
 	return;
     }
 
     #
-    # Process each command in turn.  Already sorted by
-    # getstorageconfig().
+    # Added blockstores will get created as a natural consequence of
+    # processing code. However, we need to check explicitly for removed
+    # blockstores and remove them up front, in case new blockstores are
+    # counting on reusing their space.
     #
+    if (@$ocmdref > 0) {
+	my @dcmds = ();
+
+	#
+	# For each element of the old list, see if it exists in the new.
+	#
+	OUTER: foreach my $ohref (@$ocmdref) {
+	    foreach my $href (@cmds) {
+		next if ($ohref->{'VOLNAME'} ne $href->{'VOLNAME'});
+
+		#
+		# Names are the same. In theory, the blockstore could have
+		# changed in other ways, but right now they don't and we
+		# are relying on the parser to prevent that from happening.
+		# So we just declare that the blockstores are identical
+		# and move on.
+		#
+		next OUTER;
+	    }
+
+	    #
+	    # Found an old blockstore that is no longer present, take note.
+	    #
+	    push(@dcmds, $ohref);
+	}
+
+	#
+	# Remove the blockstores that are no longer present.
+	#
+	if (@dcmds > 0) {
+	    my $so = os_init_storage(\@dcmds);
+	    if (!$so) {
+		fatal("Could not initialize storage subsystem!");
+	    }
+	    foreach my $cmd (@dcmds) {
+		if (!process($so, $cmd, 0, 1)) {
+		    fatal("Could not process storage commands!");
+		}
+	    }
+	}
+    }
+
     my $so = os_init_storage(\@cmds);
     if (!$so) {
 	fatal("Could not initialize storage subsystem!");
     }
+
+    #
+    # Process each command in turn.  Already sorted by
+    # getstorageconfig().
+    #
     foreach my $cmd (@cmds) {
 	if (!process($so, $cmd, 1, 0)) {
 	    fatal("Could not process storage commands!");

clientside/tmcc/freebsd/liblocstorage.pm

@@ -663,17 +663,22 @@ sub os_init_storage($)
 	# gvinum: put module load in /boot/loader.conf so that /etc/fstab
 	# mounts will work.
 	#
-	elsif (mysystem("grep -q 'geom_vinum_load=\"YES\"' /boot/loader.conf")) {
-	    if (!open(FD, ">>/boot/loader.conf")) {
-		warn("*** storage: could not enable gvinum in /boot/loader.conf\n");
-		return undef;
+	else {
+	    if (is_lvm_initialized(0)) {
+		$so{'VINUM_DRIVES'} = 1;
 	    }
-	    print FD "# added by $BINDIR/rc/rc.storage\n";
-	    print FD "geom_vinum_load=\"YES\"\n";
-	    close(FD);
+	    if (mysystem("grep -q 'geom_vinum_load=\"YES\"' /boot/loader.conf")) {
+		if (!open(FD, ">>/boot/loader.conf")) {
+		    warn("*** storage: could not enable gvinum in /boot/loader.conf\n");
+		    return undef;
+		}
+		print FD "# added by $BINDIR/rc/rc.storage\n";
+		print FD "geom_vinum_load=\"YES\"\n";
+		close(FD);
 
-	    # and do a one-time start
-	    mysystem("$GVINUM start");
+		# and do a one-time start
+		mysystem("$GVINUM start");
+	    }
 	}
 
 	#
@@ -1234,39 +1239,48 @@ sub os_create_storage_slice($$$)
 	    if (!exists($so->{'SPACEMAP'})) {
 		my %spacemap = ();
 
-		if ($bsid eq "ANY") {
-		    $spacemap{$bdisk}{'pnum'} = 4;
-		}
-		foreach my $dev (keys %$dinfo) {
-		    if ($dinfo->{$dev}->{'type'} eq "DISK" &&
-			$dinfo->{$dev}->{'inuse'} == 0) {
-			$spacemap{$dev}{'pnum'} = 0;
+		if (exists($so->{'VINUM_DRIVES'})) {
+		    foreach my $vdev (get_vinum_drives()) {
+			if ($vdev =~ /^(.*)s(\d+)$/) {
+			    $spacemap{$1}{'pnum'} = $2;
+			}
 		    }
 		}
-		if (keys(%spacemap) == 0) {
-		    warn("*** $lv: no space found\n");
-		    return 0;
-		}
-
-		#
-		# Create partitions on each disk
-		#
-		foreach my $disk (keys %spacemap) {
-		    my $pnum = $spacemap{$disk}{'pnum'};
+		else {
+		    if ($bsid eq "ANY") {
+			$spacemap{$bdisk}{'pnum'} = 4;
+		    }
+		    foreach my $dev (keys %$dinfo) {
+			if ($dinfo->{$dev}->{'type'} eq "DISK" &&
+			    $dinfo->{$dev}->{'inuse'} == 0) {
+			    $spacemap{$dev}{'pnum'} = 0;
+			}
+		    }
+		    if (keys(%spacemap) == 0) {
+			warn("*** $lv: no space found\n");
+			return 0;
+		    }
 
 		    #
-		    # If pnum==0, we need an MBR first
+		    # Create partitions on each disk
 		    #
-		    if ($pnum == 0) {
-			if (mysystem("$GPART create -s mbr $disk $redir")) {
-			    warn("*** $lv: could not create MBR on $disk$logmsg\n");
+		    foreach my $disk (keys %spacemap) {
+			my $pnum = $spacemap{$disk}{'pnum'};
+
+			#
+			# If pnum==0, we need an MBR first
+			#
+			if ($pnum == 0) {
+			    if (mysystem("$GPART create -s mbr $disk $redir")) {
+				warn("*** $lv: could not create MBR on $disk$logmsg\n");
+				return 0;
+			    }
+			    $pnum = $spacemap{$disk}{'pnum'} = 1;
+			}
+			if (mysystem("$GPART add -i $pnum -t freebsd $disk $redir")) {
+			    warn("*** $lv: could not create ${disk}s${pnum}$logmsg\n");
 			    return 0;
 			}
-			$pnum = $spacemap{$disk}{'pnum'} = 1;
-		    }
-		    if (mysystem("$GPART add -i $pnum -t freebsd $disk $redir")) {
-			warn("*** $lv: could not create ${disk}s${pnum}$logmsg\n");
-			return 0;
 		    }
 		}