Commit 9bf09981 authored by David Johnson's avatar David Johnson

Add Docker container blockstore support.

Docker containers may be (and default to, and in the shared host case,
must be) deprivileged; thus, they cannot mount devices, much less tell
the kernel (via iscsi userspace tools, etc) to make devices.

Therefore, we must setup any storage backing devices (temp LVs, iscsi
attachments) outside the container.  This commit makes that possible for
rc.storage and linux liblocstorage.  Basically, rc.storage now supports
(for the Linux liblocstorage and Docker) the -j vnodeid calling
convention; and if it's being called on behalf of a vnodeid, it uses
per-vnodeid fstab for any mounts, storage.conf for its state; etc.

I modified libvnode_docker to *not* create virtual networks for
remote blockstore links, because those are pinned to /30s, and thus I
have no client blockstore link address to place on a device in the root
context.  However, I (ab)used the existing Docker network setup for the
blockstore links, and that all happens the same as it used to; we just
no longer create the Docker virtual network nor attach the container to
it.

Finally, I modified tmcd dostorageconfig slightly to return
HOSTIP/HOSTMASK for remote blockstores; and now
libsetup::getstorageconfig will use HOSTIP in preference to its own
HOSTID->HOSTIP translation.  I had to do this so that libvnode_docker in
the root context would not have to go through the mess of translating
HOSTID on behalf of a vnode.
parent ef517168
......@@ -121,6 +121,13 @@ elsif (WINDOWS()) {
} elsif (STORAGEHOST()) {
@bootscripts = ("rc.misc","rc.localize","rc.keys");
}
elsif (INDOCKERVM()) {
@bootscripts = ("rc.misc", "rc.localize", "rc.keys",
"rc.blobs", "rc.topomap", "rc.accounts",
"rc.route", "rc.tunnels",
"rc.trace", "rc.syncserver", "rc.trafgen",
"rc.tarfiles", "rc.rpms", "rc.progagent", "rc.linkagent"
);
else {
@bootscripts = ("rc.firewall", "rc.tpmsetup",
"rc.misc", "rc.localize", "rc.keys",
......
......@@ -38,6 +38,7 @@ my $dolocal = 1;
my $doremote = 1;
my $typestr = "";
my $bsapidfile = "/var/run/bsagent.pid";
my $vnodeid;
# Turn off line buffering on output
$| = 1;
......@@ -53,11 +54,11 @@ if ($EUID != 0) {
#
# Load the OS independent support library. It will load the OS dependent
# library and initialize itself.
# library and initialize itself.
# (NB: liblocstorage must be imported after argument processing.)
#
use libsetup;
use liblocsetup;
use liblocstorage;
use libtmcc;
use librc;
......@@ -85,8 +86,9 @@ if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{'j'})) {
my $vnodeid = $options{'j'};
$vnodeid = $options{'j'};
libsetup_setvnodeid($vnodeid);
$OLDCONFIG .= ".$vnodeid";
}
if (defined($options{'L'})) {
$dolocal = 1;
......@@ -103,6 +105,12 @@ if (@ARGV) {
$action = $ARGV[0];
}
#
# NB: finally import liblocstorage, now that we have processed args (and
# thus might have called libsetup_setvnodeid()!
#
use liblocstorage;
# Execute the action.
SWITCH: for ($action) {
/^boot$/i && do {
......@@ -618,7 +626,7 @@ sub process($$$$)
if ($href->{'CMD'} eq "ELEMENT") {
# look up the host name and convert to IP
if (exists($href->{'HOSTID'})) {
if (!exists($href->{'HOSTIP'}) && exists($href->{'HOSTID'})) {
my $hostip = gethostbyname($href->{'HOSTID'});
if (!defined($hostip)) {
warn("*** Cannot resolve hostname '" . $href->{'HOSTID'} . "'\n");
......
......@@ -3887,6 +3887,8 @@ sub getstorageconfig($;$) {
'VOLSIZE' => '\d+',
'DATASET' => '[-\w\/\.:]+',
'SERVER' => '[-\w\.]+',
'HOSTIP' => '|(\d+\.\d+\.\d+\.\d+)',
'HOSTMASK'=> '|(\d+\.\d+\.\d+\.\d+)',
);
my @ops = ();
......
......@@ -2290,6 +2290,29 @@ sub rootPreConfigNetwork($$$$)
my @node_ifs = @{ $vnconfig->{'ifconfig'} };
my @node_lds = @{ $vnconfig->{'ldconfig'} };
#
# See if we have we have blockstore links. We do not add them as
# virtual docker networks; instead we handle the iscsi stuff outside
# the container. We just bind mount into the container stuff from
# the root context. We have to do this because parts of the iscsi
# layer in the kernel are not network-namespace-aware, so we cannot
# run the iscsi userspace tools in the network namespace, and do the
# mount ourselves. This way, we also reuse almost all the
# rc.storage*/liblocstorage code, too.
#
my %blockstoreIPs = ();
if (exists($vnconfig->{'storageconfig'})
&& defined($vnconfig->{"storageconfig"})) {
foreach my $bsref (@{$vnconfig->{'storageconfig'}}) {
if (exists($bsref->{"HOSTIP"})) {
$blockstoreIPs{$bsref->{"HOSTIP"}} =
inet_aton($bsref->{"HOSTIP"});
}
}
TBDebugTimeStamp("blockstoreIPs: ".Dumper(%blockstoreIPs)."\n")
if ($debug > 1);
}
#
# If we're using veths, figure out what bridges we need to make:
# we need a bridge for each physical iface that is a multiplex pipe,
......@@ -2312,6 +2335,24 @@ sub rootPreConfigNetwork($$$$)
print "$vnode_id interface " . Dumper($ifc) . "\n"
if ($debug > 1);
my $isblockstorelink = 0;
if (keys(%blockstoreIPs) > 0
&& exists($ifc->{IPMASK}) && exists($ifc->{IPADDR})) {
my ($nip,$nmask) =
(inet_aton($ifc->{IPADDR}),inet_aton($ifc->{IPMASK}));
foreach my $k (keys(%blockstoreIPs)) {
if (($nip & $nmask) eq ($blockstoreIPs{$k} & $nmask)) {
$isblockstorelink = 1;
last;
}
}
}
if ($isblockstorelink) {
print "$vnode_id interface $ifc->{IPMASK} is a blockstore link;".
" we will not create a virtual network for it.\n"
if ($debug > 1);
}
#
# In the era of shared nodes, we cannot name the bridges
# using experiment local names (e.g., the link name).
......@@ -2408,6 +2449,11 @@ sub rootPreConfigNetwork($$$$)
$ifc->{'BRIDGE'} = $brname
if (defined($brname));
if ($isblockstorelink) {
$brs{$brname}{ISBLOCKSTORELINK} = 1;
$ifc->{ISBLOCKSTORELINK} = 1;
}
#
# Docker networks require a subnet (and a gateway; i.e.
# https://github.com/docker/libnetwork/issues/1447#issuecomment-247368397).
......@@ -2465,6 +2511,10 @@ sub rootPreConfigNetwork($$$$)
foreach my $k (keys(%brs)) {
my $cidr = $brs{$k}{CIDR};
my $gw = $brs{$k}{GW};
my $isblockstorelink = 0;
if (exists($brs{$k}{ISBLOCKSTORELINK})) {
$isblockstorelink = $brs{$k}{ISBLOCKSTORELINK};
}
if (!$USE_MACVLAN) {
#
......@@ -2531,26 +2581,40 @@ sub rootPreConfigNetwork($$$$)
}
#
# Now that the bridge exists, make the Docker network atop it.
# If this is a blockstore link, we just IP the bridge we
# just created; we do not expose this as a Docker virtual
# network. This way, we can reuse all the existing network
# teardown code.
#
TBDebugTimeStamp("checking existence of docker network $k");
($code,$content,$resp) = getClient()->network_inspect($k);
if ($code) {
my $ourdocker_extra_args = undef;
if ($ISOURDOCKER) {
$ourdocker_extra_args = {
"Options" => { "com.docker.network.bridge.layer2_mode"
=> "true" },
"IPAM" => { "Options" => { "PrivatePoolId" => $k } },
};
if ($isblockstorelink) {
my ($bsa,$bsm) =
($brs{$k}{IFC}->{IPADDR},$brs{$k}{IFC}->{IPMASK});
mysystem2("$IP addr replace $bsa/$bsm dev $k");
mysystem2("$IP link set $k up");
}
else {
#
# Now that the bridge exists, make the Docker network atop it.
#
TBDebugTimeStamp("checking existence of docker network $k");
($code,$content,$resp) = getClient()->network_inspect($k);
if ($code) {
my $ourdocker_extra_args = undef;
if ($ISOURDOCKER) {
$ourdocker_extra_args = {
"Options" => { "com.docker.network.bridge.layer2_mode"
=> "true" },
"IPAM" => { "Options" => { "PrivatePoolId" => $k } },
};
}
TBDebugTimeStamp("creating docker network $k");
($code,$content,$resp) = getClient()->network_create_bridge(
$k,$cidr,$gw,$k,$ourdocker_extra_args);
goto bad
if ($code);
}
TBDebugTimeStamp("creating docker network $k");
($code,$content,$resp) = getClient()->network_create_bridge(
$k,$cidr,$gw,$k,$ourdocker_extra_args);
goto bad
if ($code);
$private->{'dockernets'}->{$k} = $k;
}
$private->{'dockernets'}->{$k} = $k;
}
else {
my $basedev;
......@@ -2574,21 +2638,29 @@ sub rootPreConfigNetwork($$$$)
$private->{'dummys'}->{$k} = $basedev;
}
#
# Make the docker network if necessary.
#
TBDebugTimeStamp("checking existence of docker network $k");
($code,$content,$resp) = getClient()->network_inspect($k);
if ($code) {
# Now that the dummy device exists, make the Docker
# network atop it.
TBDebugTimeStamp("creating docker network $k");
($code,$content,$resp) = getClient()->network_create_macvlan(
$k,$cidr,$gw,$basedev);
goto bad
if ($code);
if ($isblockstorelink) {
my ($bsa,$bsm) =
($brs{$k}{IFC}->{IPADDR},$brs{$k}{IFC}->{IPMASK});
mysystem2("$IP addr replace $bsa/$bsm dev $k");
mysystem2("$IP link set $k up");
}
else {
#
# Make the docker network if necessary.
#
TBDebugTimeStamp("checking existence of docker network $k");
($code,$content,$resp) = getClient()->network_inspect($k);
if ($code) {
# Now that the dummy device exists, make the Docker
# network atop it.
TBDebugTimeStamp("creating docker network $k");
($code,$content,$resp) = getClient()->network_create_macvlan(
$k,$cidr,$gw,$basedev);
goto bad
if ($code);
}
$private->{'dockernets'}->{$k} = $k;
}
$private->{'dockernets'}->{$k} = $k;
}
}
......@@ -2783,6 +2855,31 @@ sub rootPreConfigNetwork($$$$)
return -1;
}
sub _docker_get_ext_trans_mountpoint($)
{
my ($mpoint,) = @_;
my $confdir = CONFDIR();
my $tmpoint = `readlink -f $confdir/mountpoints`;
chomp($tmpoint);
if ($mpoint =~ /^$tmpoint/) {
return $mpoint;
}
else {
$mpoint = $tmpoint.$mpoint;
}
if (! -e $mpoint) {
my @dirs = split(/\//,$mpoint);
my $tpath = CONFDIR()."/mountpoints";
foreach my $dir (@dirs) {
$tpath .= "/$dir";
mkdir($tpath);
}
}
return $mpoint;
}
#
# Create the basic context for the VM and give it a unique ID for identifying
# "internal" state. If $raref is set, then we are in a RELOAD state machine
......@@ -2967,6 +3064,30 @@ sub vnodeCreate($$$$)
#
addMounts($vnode_id,\%mounts);
#
# Handle blockstores/datasets.
#
my %blockstoreMounts = ();
if (exists($vnconfig->{"storageconfig"})
&& defined($vnconfig->{"storageconfig"})) {
foreach my $bsref (@{$vnconfig->{'storageconfig'}}) {
if (exists($bsref->{"MOUNTPOINT"})) {
my $src = _docker_get_ext_trans_mountpoint($bsref->{"MOUNTPOINT"});
$blockstoreMounts{$src} = $bsref->{"MOUNTPOINT"};
}
}
TBDebugTimeStamp("blockstoreMounts: ".Dumper(%blockstoreMounts)."\n")
if ($debug > 1);
TBDebugTimeStamp("starting rc.storage")
if ($debug > 1);
if (mysystem2("/usr/local/etc/emulab/rc/rc.storage -j $vnode_id boot")) {
fatal("Failed to setup storage in rc.storage; aborting!");
}
TBDebugTimeStamp("rc.storage finished successfully")
if ($debug > 1);
$private->{'blockstores'} = scalar(keys(%blockstoreMounts));
}
#
# Start building the 'docker create' args.
# (NB: see note below about why we have to put the container on the
......@@ -2997,6 +3118,16 @@ sub vnodeCreate($$$$)
push(@{$args{"HostConfig"}{"Binds"}},$bind);
}
#
# Add blockstore mounts.
#
$args{"HostConfig"}{"Binds"} = [];
foreach my $src (keys(%blockstoreMounts)) {
my $dst = $blockstoreMounts{$src};
my $bind = "${src}:${dst}";
push(@{$args{"HostConfig"}{"Binds"}},$bind);
}
#
# Add some Emulab-specific mount points that contain information:
# /var/emulab/boot/{tmcc,tmcc.<vnodeid>}.
......@@ -3546,6 +3677,12 @@ sub vnodePreConfigExpNetwork($$$$)
Dumper($ifc))
if ($debug > 1);
if (exists($ifc->{ISBLOCKSTORELINK}) && $ifc->{ISBLOCKSTORELINK}) {
TBDebugTimeStamp("vnodePreConfigExpNetwork: $vnode_id skipping blockstore interface!")
if ($debug > 1);
next;
}
my $br = $ifc->{"BRIDGE"};
my $physdev = $ifc->{"PHYSDEV"};
my $ldinfo;
......@@ -4116,6 +4253,16 @@ sub vnodeDestroy($$$$)
TBDebugTimeStamp(" got global lock")
if ($lockdebug);
if (exists($private->{'blockstores'})) {
TBDebugTimeStamp("starting rc.storage")
if ($debug > 1);
if (mysystem2("/usr/local/etc/emulab/rc/rc.storage -j $vnode_id fullreset")) {
fatal("Failed to remove storage in rc.storage; aborting!");
}
TBDebugTimeStamp("rc.storage finished successfully")
if ($debug > 1);
}
#
# Remove mounts.
#
......
......@@ -58,9 +58,14 @@ BEGIN
$VARDIR = "/etc/rc.d/testbed";
$BOOTDIR = "/etc/rc.d/testbed";
}
my $genvmtype = `cat $ETCDIR/genvmtype`;
chomp($genvmtype);
$VGNAME = "emulab";
if (GENVNODEHOST() && !SHAREDHOST()) {
if (GENVNODEHOST() && GENVNODETYPE() eq 'docker') {
$VGNAME = "docker";
} elsif (GENVNODEHOST() && !SHAREDHOST()) {
$VGNAME = "xen-vg";
} elsif (INXENVM() && -r "$VARDIR/boot/vmname") {
my $vname = `cat $VARDIR/boot/vmname`;
......@@ -71,6 +76,14 @@ BEGIN
}
}
sub ISFORDOCKERVM() {
if (defined(libsetup_getvnodeid())
&& GENVNODEHOST() && GENVNODETYPE() eq 'docker') {
return 1;
}
return 0;
}
my $MOUNT = "/bin/mount";
my $UMOUNT = "/bin/umount";
my $MKDIR = "/bin/mkdir";
......@@ -89,6 +102,8 @@ my $PPROBE = "/sbin/partprobe";
my $FRISBEE = "/usr/local/bin/frisbee";
my $HDPARM = "/sbin/hdparm";
my $FSTAB = "/etc/fstab";
#
#
# To find the block stores exported from a target portal:
......@@ -534,8 +549,8 @@ sub get_diskinfo()
close(FD);
# XXX watch out for mounted disks/partitions (DOS type may be 0)
if (!open(FD, "/etc/fstab")) {
warn("*** get_diskinfo: could not get mount info from /etc/fstab\n");
if (!open(FD, "$FSTAB")) {
warn("*** get_diskinfo: could not get mount info from $FSTAB\n");
return undef;
}
while (<FD>) {
......@@ -779,6 +794,20 @@ sub os_init_storage($)
my %so = ();
#
# If we are running on the outside of a Docker container, but on its
# behalf, we want to do some things differently.
#
if (ISFORDOCKERVM()) {
$FSTAB = CONFDIR()."/fstab-storage";
$MOUNT .= " --fstab $FSTAB";
#$UMOUNT .= " --fstab $FSTAB";
if (! -e $FSTAB) {
open(FD,">$FSTAB");
close(FD);
}
}
foreach my $href (@{$lref}) {
if ($href->{'CMD'} eq "ELEMENT") {
$gotelement++;
......@@ -953,6 +982,31 @@ sub os_check_storage($$)
return -1;
}
sub _docker_get_ext_trans_mountpoint($)
{
my ($mpoint,) = @_;
my $confdir = CONFDIR();
my $tmpoint = `readlink -f $confdir/mountpoints`;
chomp($tmpoint);
if ($mpoint =~ /^$tmpoint/) {
return $mpoint;
}
else {
$mpoint = $tmpoint.$mpoint;
}
if (! -e $mpoint) {
my @dirs = split(/\//,$mpoint);
my $tpath = CONFDIR()."/mountpoints";
foreach my $dir (@dirs) {
$tpath .= "/$dir";
mkdir($tpath);
}
}
return $mpoint;
}
sub os_check_storage_element($$)
{
my ($so,$href) = @_;
......@@ -1032,8 +1086,16 @@ sub os_check_storage_element($$)
# do for local blockstores. Thus, if the blockstore device is not
# mounted, we do it here.
#
# NB: also, for the Docker case where we setup blockstore access
# outside the container, we make alternative $FSTAB entries!
#
my $mpoint = $href->{'MOUNTPOINT'};
if ($mpoint) {
if (ISFORDOCKERVM()) {
$mpoint = $href->{'MOUNTPOINT'} =
_docker_get_ext_trans_mountpoint($mpoint);
}
my $mdev = $href->{'LVDEV'};
my $mopt = "";
......@@ -1140,7 +1202,7 @@ sub os_check_storage_slice($$)
# see if there is a logical volume with appropriate name
# else if BSID==ANY:
# see if there is a logical volume with appropriate name
# if there is a mountpoint, see if it exists in /etc/fstab
# if there is a mountpoint, see if it exists in $FSTAB
#
if ($href->{'CLASS'} eq "local") {
my $lv = $href->{'VOLNAME'};
......@@ -1231,10 +1293,15 @@ sub os_check_storage_slice($$)
#
my $mpoint = $href->{'MOUNTPOINT'};
if ($mpoint) {
if (ISFORDOCKERVM()) {
$mpoint = $href->{'MOUNTPOINT'} =
_docker_get_ext_trans_mountpoint($mpoint);
}
my $line = `$MOUNT | grep '^/dev/$rdev on '`;
if (!$line) {
#
# See if the mount exists in /etc/fstab.
# See if the mount exists in $FSTAB.
#
# XXX Right now if it does not, it might be because we
# removed it prior to creating an image. So we make some
......@@ -1245,7 +1312,7 @@ sub os_check_storage_slice($$)
# and not only the fstab line but the mountpoint might be
# missing. We attempt to repair this case as well.
#
$line = `grep '^/dev/$dev\[\[:space:\]\]' /etc/fstab`;
$line = `grep '^/dev/$dev\[\[:space:\]\]' $FSTAB`;
if (!$line) {
warn(" $lv: mount of /dev/$dev missing from fstab; sanity checking and re-adding...\n");
my $fstype = get_fstype($href, "/dev/$dev");
......@@ -1269,8 +1336,8 @@ sub os_check_storage_slice($$)
return -1;
}
if (!open(FD, ">>/etc/fstab")) {
warn("*** $lv: could not add mount to /etc/fstab\n");
if (!open(FD, ">>$FSTAB")) {
warn("*** $lv: could not add mount to $FSTAB\n");
return -1;
}
print FD "# /dev/$dev added by $BINDIR/rc/rc.storage\n";
......@@ -1445,20 +1512,25 @@ sub os_create_storage($$)
# Mount the filesystem
#
my $mpoint = $href->{'MOUNTPOINT'};
if (defined($mpoint) && ISFORDOCKERVM()) {
$mpoint = $href->{'MOUNTPOINT'} =
_docker_get_ext_trans_mountpoint($mpoint);
}
if (! -d "$mpoint" && mysystem("$MKDIR -p $mpoint $redir")) {
warn("*** $lv: could not create mountpoint '$mpoint'$logmsg\n");
return 0;
}
#
# XXX because mounts in /etc/fstab happen before iSCSI and possibly
# XXX because mounts in $FSTAB happen before iSCSI and possibly
# even the network are setup, we don't put our mounts there as we
# do for local blockstores. Instead, the check_storage call will
# take care of these mounts.
#
if (!($href->{'CLASS'} eq "SAN" && $href->{'PROTO'} eq "iSCSI")) {
if (!open(FD, ">>/etc/fstab")) {
warn("*** $lv: could not add mount to /etc/fstab\n");
if (!open(FD, ">>$FSTAB")) {
warn("*** $lv: could not add mount to $FSTAB\n");
return 0;
}
print FD "# $mdev added by $BINDIR/rc/rc.storage\n";
......@@ -1565,7 +1637,7 @@ sub os_create_storage_slice($$$)
# create an LVM PV/VG from all available space (part 4 on sysvol,
# extra hard drives), create LV with appropriate name from VG.
# if there is a mountpoint:
# create a filesystem on device, mount it, add to /etc/fstab
# create a filesystem on device, mount it, add to $FSTAB
#
if ($href->{'CLASS'} eq "local") {
my $lv = $href->{'VOLNAME'};
......@@ -1809,6 +1881,10 @@ sub os_remove_storage_element($$$)
#
if (exists($href->{'MOUNTPOINT'})) {
my $mpoint = $href->{'MOUNTPOINT'};
if (ISFORDOCKERVM()) {
$mpoint = $href->{'MOUNTPOINT'} =
_docker_get_ext_trans_mountpoint($mpoint);
}
if (mysystem("$UMOUNT $mpoint")) {
warn("*** $bsid: could not unmount $mpoint\n");
......@@ -1909,6 +1985,10 @@ sub os_remove_storage_slice($$$)
#
if (exists($href->{'MOUNTPOINT'})) {
my $mpoint = $href->{'MOUNTPOINT'};
if (ISFORDOCKERVM()) {
$mpoint = $href->{'MOUNTPOINT'} =
_docker_get_ext_trans_mountpoint($mpoint);
}
if (mysystem("$UMOUNT $mpoint")) {
warn("*** $lv: could not unmount $mpoint\n");
......@@ -1922,8 +2002,8 @@ sub os_remove_storage_slice($$$)
if ($teardown) {
my $tdev = "/dev/$dev";
$tdev =~ s/\//\\\//g;
if (mysystem("sed -E -i -e '/^(# )?$tdev/d' /etc/fstab")) {
warn("*** $lv: could not remove mount from /etc/fstab\n");
if (mysystem("sed -E -i -e '/^(# )?$tdev/d' $FSTAB")) {
warn("*** $lv: could not remove mount from $FSTAB\n");
}
}
}
......
......@@ -4829,7 +4829,8 @@ COMMAND_PROTOTYPE(dostorageconfig)
}
mysql_free_result(res);
res = mydb_query("select rb.bsidx, r.vname, rb.vname, rb.size "
res = mydb_query("select rb.bsidx, r.vname, rb.vname, rb.size, vl.ip, "
" vl.mask "
"from reserved_blockstores as rb "
" left join reserved as r "
" on r.node_id = rb.vnode_id and "
......@@ -4839,7 +4840,7 @@ COMMAND_PROTOTYPE(dostorageconfig)
" r.exptidx = vl.exptidx "
"where vl.vname in (%s) "
" and vl.pid='%s' and vl.eid='%s'",
4, buf, reqp->pid, reqp->eid);
6, buf, reqp->pid, reqp->eid);
if (!res) {
error("STORAGECONFIG: %s: DB Error getting connected "
......@@ -4858,8 +4859,9 @@ COMMAND_PROTOTYPE(dostorageconfig)
volsize = atoi(row[3]);
OUTPUT(buf, sizeof(buf),
"CMD=ELEMENT IDX=%d HOSTID=%s VOLNAME=%s VOLSIZE=%d",
cmdidx++, hostid, vname, volsize);
"CMD=ELEMENT IDX=%d HOSTID=%s VOLNAME=%s VOLSIZE=%d"
" HOSTIP=%s HOSTMASK=%s",
cmdidx++, hostid, vname, volsize, row[4], row[5]);
sendstoreconf(sock, tcp, reqp, buf, vname, 1, NULL);
}
mysql_free_result(res);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment