Commit 2ead2a68 authored by Mike Hibler's avatar Mike Hibler

Inching toward recursive use, ala what is needed for elabinelab or subbosses.

Add the ability of the master server to have a "parent" from which it can
download an image if it doesn't have it or if the image is out of date.
Had to add some more goo to the GET reply, notably a hash so that we can
check for up-to-dateness.

The actual part where we upcall to the parent isn't done yet, that is why
this is "inching toward" and not "leaping and bounding toward"...

Also redid the child process management to not use SIGCHLD, no need for that.
parent db6c03a8
......@@ -354,13 +354,31 @@ main(int argc, char **argv)
ClientLogInit();
#ifdef MASTER_SERVER
if (imageid && !ClientNetFindServer(&serverip, imageid, askonly, 5)) {
fprintf(stderr, "Could not get download info for '%s'\n",
imageid);
exit(1);
if (imageid) {
GetReply reply;
int method = askonly ? MS_METHOD_ANY : MS_METHOD_MULTICAST;
int timo = 5; /* XXX */
if (!ClientNetFindServer(ntohl(serverip.s_addr), portnum,
imageid, method, askonly, timo,
&reply))
fatal("Could not get download info for '%s'", imageid);
if (askonly) {
PrintGetInfo(imageid, &reply);
exit(0);
}
if (reply.error)
fatal("%s: server returned error: %s",
imageid, GetMSError(reply.error));
log("%s: address: %s:%d",
imageid, inet_ntoa(mcastaddr), portnum);
mcastaddr.s_addr = htonl(reply.addr);
portnum = reply.port;
}
if (askonly)
exit(0);
#endif
ClientNetInit();
......
......@@ -5,6 +5,7 @@
#include <stdlib.h>
#include <signal.h>
#include <assert.h>
#include "decls.h"
#include "configdefs.h"
#include "log.h"
......@@ -81,6 +82,27 @@ config_get_host_authinfo(struct in_addr *in, char *imageid,
return myconfig->config_get_host_authinfo(in, imageid, get, put);
}
void
config_dump_host_authinfo(struct config_host_authinfo *ai)
{
char *none = "<NONE>";
int i;
if (ai == NULL)
return;
fprintf(stderr, "HOST authinfo %p:\n", ai);
fprintf(stderr, " hostid: %s\n", ai->hostid ? ai->hostid : none);
if (ai->numimages > 0) {
fprintf(stderr, " %d image(s):\n", ai->numimages);
for (i = 0; i < ai->numimages; i++)
fprintf(stderr, " [%d]: imageid='%s', path='%s'\n",
i, ai->imageinfo[i].imageid,
ai->imageinfo[i].path);
}
fprintf(stderr, " extra: %p\n", ai->extra);
}
void
config_free_host_authinfo(struct config_host_authinfo *ai)
{
......@@ -100,14 +122,14 @@ config_auth_by_IP(struct in_addr *host, char *imageid,
struct config_host_authinfo *ai;
if (config_get_host_authinfo(host, imageid, &ai, 0))
return CONFIG_ERR_HA_FAILED;
return MS_ERROR_FAILED;
if (ai->hostid == NULL) {
config_free_host_authinfo(ai);
return CONFIG_ERR_HA_NOHOST;
return MS_ERROR_NOHOST;
}
if (ai->numimages == 0) {
config_free_host_authinfo(ai);
return CONFIG_ERR_HA_NOACCESS;
return MS_ERROR_NOACCESS;
}
if (aip)
*aip = ai;
......@@ -123,25 +145,6 @@ config_get_server_address(struct config_host_authinfo *ai, int methods,
addr, port, method);
}
char *
config_perror(int code)
{
switch (code) {
case CONFIG_ERR_HA_FAILED:
return "host authentication failed";
case CONFIG_ERR_HA_NOHOST:
return "unknown host";
case CONFIG_ERR_HA_NOIMAGE:
return "unknown image";
case CONFIG_ERR_HA_NOACCESS:
return "permission denied";
case CONFIG_ERR_HA_NOMETHOD:
return "invalid method";
default:
return "unknown error";
}
}
void
config_dump(FILE *fd)
{
......
......@@ -6,6 +6,7 @@
#ifdef USE_EMULAB_CONFIG
#include <sys/param.h>
#include <sys/stat.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
......@@ -39,8 +40,10 @@ static char *GROUPSDIR = GROUPSROOT_DIR;
static char *USERSDIR = USERSROOT_DIR;
static char *SCRATCHDIR = SCRATCHROOT_DIR;
#ifndef ELABINELAB
/* XXX should be autoconfiged as part of Emulab build */
static char *IMAGEDIR = "/usr/testbed/images";
#endif
/* Emit aliases when dumping the config info; makes it smaller */
static int dump_doaliases = 1;
......@@ -204,6 +207,7 @@ emulab_free_host_authinfo(struct config_host_authinfo *ai)
for (i = 0; i < ai->numimages; i++) {
FREE(ai->imageinfo[i].imageid);
FREE(ai->imageinfo[i].path);
FREE(ai->imageinfo[i].sig);
FREE(ai->imageinfo[i].get_options);
FREE(ai->imageinfo[i].put_options);
FREE(ai->imageinfo[i].extra);
......@@ -325,6 +329,8 @@ allow_stddirs(char *imageid,
char *fpath, *shdir, *pdir, *gdir, *scdir, *udir;
int doput = 0, doget = 0;
struct emulab_ha_extra_info *ei;
struct config_imageinfo *ci;
struct stat sb;
if (get == NULL && put == NULL)
return;
......@@ -359,7 +365,6 @@ allow_stddirs(char *imageid,
int ni, i;
size_t ns;
char *dirs[8];
struct config_imageinfo *ci;
/*
* Right now, only allow PUT to scratchdir if it exists.
......@@ -376,7 +381,13 @@ allow_stddirs(char *imageid,
ci = &put->imageinfo[i];
ci->imageid = NULL;
ci->path = mystrdup(dirs[i - put->numimages]);
ci->flags = CONFIG_IMAGE_ISDIR;
ci->flags = CONFIG_PATH_ISDIR;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
ci->get_options = NULL;
ci->get_methods = 0;
ci->put_options = NULL;
......@@ -403,7 +414,13 @@ allow_stddirs(char *imageid,
ci = &get->imageinfo[i];
ci->imageid = NULL;
ci->path = mystrdup(dirs[i - get->numimages]);
ci->flags = CONFIG_IMAGE_ISDIR;
ci->flags = CONFIG_PATH_ISDIR;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
set_get_options(get, i);
set_get_methods(get, i);
ci->put_options = NULL;
......@@ -441,26 +458,40 @@ allow_stddirs(char *imageid,
put->imageinfo = mymalloc(sizeof(struct config_imageinfo));
put->numimages = 1;
put->imageinfo[0].imageid = mystrdup(imageid);
put->imageinfo[0].path = mystrdup(fpath);
put->imageinfo[0].flags = CONFIG_IMAGE_ISFILE;
put->imageinfo[0].get_options = NULL;
put->imageinfo[0].get_methods = 0;
put->imageinfo[0].put_options = NULL;
put->imageinfo[0].extra = NULL;
ci = &put->imageinfo[0];
ci->imageid = mystrdup(imageid);
ci->path = mystrdup(fpath);
ci->flags = CONFIG_PATH_ISFILE;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
ci->get_options = NULL;
ci->get_methods = 0;
ci->put_options = NULL;
ci->extra = NULL;
}
if (doget) {
assert(get->imageinfo == NULL);
get->imageinfo = mymalloc(sizeof(struct config_imageinfo));
get->numimages = 1;
get->imageinfo[0].imageid = mystrdup(imageid);
get->imageinfo[0].path = mystrdup(fpath);
get->imageinfo[0].flags = CONFIG_IMAGE_ISFILE;
ci = &get->imageinfo[0];
ci->imageid = mystrdup(imageid);
ci->path = mystrdup(fpath);
ci->flags = CONFIG_PATH_ISFILE;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
set_get_options(get, 0);
set_get_methods(get, 0);
get->imageinfo[0].put_options = NULL;
get->imageinfo[0].extra = NULL;
ci->put_options = NULL;
ci->extra = NULL;
}
done:
......@@ -643,6 +674,8 @@ emulab_get_host_authinfo(struct in_addr *in, char *imageid,
put->numimages = 0;
for (i = 0; i < nrows; i++) {
struct emulab_ii_extra_info *ii;
struct config_imageinfo *ci;
struct stat sb;
char *imageid;
row = mysql_fetch_row(res);
......@@ -657,16 +690,22 @@ emulab_get_host_authinfo(struct in_addr *in, char *imageid,
strcpy(imageid, row[0]);
strcat(imageid, "/");
strcat(imageid, row[2]);
put->imageinfo[put->numimages].imageid = imageid;
put->imageinfo[put->numimages].path = mystrdup(row[3]);
put->imageinfo[put->numimages].flags =
CONFIG_IMAGE_ISFILE;
put->imageinfo[put->numimages].get_methods = 0;
put->imageinfo[put->numimages].get_options = NULL;
put->imageinfo[put->numimages].put_options = NULL;
ci = &put->imageinfo[put->numimages];
ci->imageid = imageid;
ci->path = mystrdup(row[3]);
ci->flags = CONFIG_PATH_ISFILE;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
ci->get_methods = 0;
ci->get_options = NULL;
ci->put_options = NULL;
ii = mymalloc(sizeof *ii);
ii->DB_imageid = atoi(row[4]);
put->imageinfo[put->numimages].extra = ii;
ci->extra = ii;
put->numimages++;
}
mysql_free_result(res);
......@@ -703,6 +742,8 @@ emulab_get_host_authinfo(struct in_addr *in, char *imageid,
get->numimages = 0;
for (i = 0; i < nrows; i++) {
struct emulab_ii_extra_info *ii;
struct config_imageinfo *ci;
struct stat sb;
char *imageid;
row = mysql_fetch_row(res);
......@@ -717,16 +758,22 @@ emulab_get_host_authinfo(struct in_addr *in, char *imageid,
strcpy(imageid, row[0]);
strcat(imageid, "/");
strcat(imageid, row[2]);
get->imageinfo[get->numimages].imageid = imageid;
get->imageinfo[get->numimages].path = mystrdup(row[3]);
get->imageinfo[get->numimages].flags =
CONFIG_IMAGE_ISFILE;
ci = &get->imageinfo[get->numimages];
ci->imageid = imageid;
ci->path = mystrdup(row[3]);
ci->flags = CONFIG_PATH_ISFILE;
if (stat(ci->path, &sb) == 0) {
ci->sig = mymalloc(sizeof(time_t));
*(time_t *)ci->sig = sb.st_mtime;
ci->flags |= CONFIG_SIG_ISMTIME;
} else
ci->sig = NULL;
set_get_methods(get, get->numimages);
set_get_options(get, get->numimages);
get->imageinfo[get->numimages].put_options = NULL;
ci->put_options = NULL;
ii = mymalloc(sizeof *ii);
ii->DB_imageid = atoi(row[4]);
get->imageinfo[get->numimages].extra = ii;
ci->extra = ii;
get->numimages++;
}
mysql_free_result(res);
......@@ -735,7 +782,8 @@ emulab_get_host_authinfo(struct in_addr *in, char *imageid,
/*
* Finally add on the standard directories that a node can access.
*/
allow_stddirs(NULL, get, put);
if (imageid == NULL)
allow_stddirs(imageid, get, put);
done:
free(node);
......@@ -775,7 +823,7 @@ dump_host_authinfo(FILE *fd, char *node, char *cmd,
*/
else {
for (i = 0; i < ai->numimages; i++)
if (ai->imageinfo[i].flags == CONFIG_IMAGE_ISFILE)
if (ai->imageinfo[i].flags == CONFIG_PATH_ISFILE)
fprintf(fd, "%s ", ai->imageinfo[i].imageid);
}
......@@ -783,7 +831,7 @@ dump_host_authinfo(FILE *fd, char *node, char *cmd,
* And dump any directories that can be accessed
*/
for (i = 0; i < ai->numimages; i++)
if (ai->imageinfo[i].flags == CONFIG_IMAGE_ISDIR)
if (ai->imageinfo[i].flags == CONFIG_PATH_ISDIR)
fprintf(fd, "%s/* ", ai->imageinfo[i].path);
fprintf(fd, "\n");
......
......@@ -8,6 +8,7 @@
struct config_imageinfo {
char *imageid; /* unique name of image */
char *path; /* path where image is stored */
void *sig; /* signature of image */
int flags; /* */
char *get_options; /* options for GET operation */
int get_methods; /* allowed GET transfer mechanisms */
......@@ -16,10 +17,13 @@ struct config_imageinfo {
};
/* flags */
#define CONFIG_IMAGE_ISFILE 0x1 /* path is an image file */
#define CONFIG_IMAGE_ISDIR 0x2 /* path is a directory */
#define CONFIG_IMAGE_ISGLOB 0x4 /* path is a file glob */
#define CONFIG_IMAGE_ISRE 0x8 /* path is a perl RE */
#define CONFIG_PATH_ISFILE 0x1 /* path is an image file */
#define CONFIG_PATH_ISDIR 0x2 /* path is a directory */
#define CONFIG_PATH_ISGLOB 0x4 /* path is a file glob */
#define CONFIG_PATH_ISRE 0x8 /* path is a perl RE */
#define CONFIG_SIG_ISMTIME 0x10 /* sig is path mtime */
#define CONFIG_SIG_ISMD5 0x20 /* sig is MD5 hash of path */
#define CONFIG_SIG_ISSHA1 0x40 /* sig is SHA1 hash of path */
/* methods */
#define CONFIG_IMAGE_UNKNOWN 0x0
......@@ -59,18 +63,12 @@ extern int config_read(void);
extern int config_get_host_authinfo(struct in_addr *, char *,
struct config_host_authinfo **,
struct config_host_authinfo **);
extern void config_dump_host_authinfo(struct config_host_authinfo *);
extern void config_free_host_authinfo(struct config_host_authinfo *);
extern int config_auth_by_IP(struct in_addr *, char *,
struct config_host_authinfo **);
extern int config_get_server_address(struct config_host_authinfo *, int,
in_addr_t *, in_port_t *, int *);
extern char * config_perror(int);
extern void * config_save(void);
extern int config_restore(void *);
extern void config_dump(FILE *);
#define CONFIG_ERR_HA_FAILED 1 /* internal host auth error */
#define CONFIG_ERR_HA_NOHOST 2 /* no such host */
#define CONFIG_ERR_HA_NOIMAGE 3 /* no such image */
#define CONFIG_ERR_HA_NOACCESS 4 /* access not allowed for host */
#define CONFIG_ERR_HA_NOMETHOD 5 /* not avail to host via method */
......@@ -322,27 +322,35 @@ typedef struct {
/* imageid length: large enough to hold an ascii encoded SHA 1024 hash */
#define MS_MAXIDLEN 256
/* ditto for signature */
#define MS_MAXSIGLEN 256
/*
* Master server messages.
* These are sent via unicast TCP.
*/
typedef struct {
int32_t type;
uint8_t methods;
uint8_t status;
uint16_t idlen;
uint8_t imageid[MS_MAXIDLEN];
} __attribute__((__packed__)) GetRequest;
typedef struct {
uint8_t method;
uint8_t isrunning;
uint16_t error;
in_addr_t addr;
in_port_t port;
uint16_t sigtype;
uint8_t signature[MS_MAXSIGLEN];
} __attribute__((__packed__)) GetReply;
typedef struct {
int32_t type;
union {
struct {
uint8_t methods;
uint8_t status;
uint16_t idlen;
char imageid[MS_MAXIDLEN];
} __attribute__((__packed__)) getrequest;
struct {
uint8_t method;
uint8_t isrunning;
uint16_t error;
in_addr_t addr;
in_port_t port;
} __attribute__((__packed__)) getreply;
GetRequest getrequest;
GetReply getreply;
} body;
} MasterMsg_t;
......@@ -356,6 +364,19 @@ typedef struct {
#define MS_METHOD_MULTICAST 2
#define MS_METHOD_BROADCAST 4
#define MS_METHOD_ANY 7
#define MS_SIGTYPE_NONE 0
#define MS_SIGTYPE_MTIME 1
#define MS_SIGTYPE_MD5 2
#define MS_SIGTYPE_SHA1 3
#define MS_ERROR_FAILED 1 /* internal host auth error */
#define MS_ERROR_NOHOST 2 /* no such host */
#define MS_ERROR_NOIMAGE 3 /* no such image */
#define MS_ERROR_NOACCESS 4 /* access not allowed for host */
#define MS_ERROR_NOMETHOD 5 /* not avail to host via method */
#define MS_ERROR_INVALID 6 /* invalid argument */
#define MS_ERROR_TRYAGAIN 7 /* try again later */
#endif
/*
......@@ -372,7 +393,8 @@ void PacketReply(Packet_t *p);
int PacketValid(Packet_t *p, int nchunks);
void dump_network(void);
#ifdef MASTER_SERVER
int ClientNetFindServer(struct in_addr *, char *, int, int);
int ClientNetFindServer(in_addr_t, in_port_t, char *, int, int, int,
GetReply *);
int MsgSend(int, MasterMsg_t *, size_t, int);
int MsgReceive(int, MasterMsg_t *, size_t, int);
#endif
......
This diff is collapsed.
......@@ -580,27 +580,32 @@ MsgReceive(int msock, MasterMsg_t *msg, size_t size, int timo)
/*
* Contact the master server to discover download information for imageid.
* 'serverip' and 'portnum' refer to the master server at this point,
* it in turn will return the addr/port for the actual download.
* 'sip' and 'sport' are the addr/port of the master server, 'method'
* specifies the desired download method, 'askonly' is set to just ask
* for information about the image (without starting a server), 'timeout'
* is how long to wait for a response.
*
* On success, return non-zero with 'reply' filled in with the server's
* response IN HOST ORDER. On failure returns zero.
*/
int
ClientNetFindServer(struct in_addr *sip, char *imageid,
int statusonly, int timeout)
ClientNetFindServer(in_addr_t sip, in_port_t sport, char *imageid,
int method, int askonly, int timeout, GetReply *reply)
{
struct sockaddr_in name;
MasterMsg_t msg;
int msock, len, err;
int msock, len;
if ((msock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
perror("Could not allocate socket for master server");
return 0;
}
if (portnum == 0)
portnum = MS_PORTNUM;
if (sport == 0)
sport = MS_PORTNUM;
name.sin_family = AF_INET;
name.sin_addr = *sip;
name.sin_port = htons(portnum);
name.sin_addr.s_addr = htonl(sip);
name.sin_port = htons(sport);
if (connect(msock, (struct sockaddr *)&name, sizeof(name)) < 0) {
perror("Connecting to master server");
close(msock);
......@@ -609,17 +614,17 @@ ClientNetFindServer(struct in_addr *sip, char *imageid,
memset(&msg, 0, sizeof msg);
msg.type = htonl(MS_MSGTYPE_GETREQUEST);
if (statusonly) {
if (askonly) {
msg.body.getrequest.status = 1;
msg.body.getrequest.methods = MS_METHOD_ANY;
} else {
msg.body.getrequest.methods = MS_METHOD_MULTICAST;
msg.body.getrequest.methods = method;
}
len = strlen(imageid);
if (len > MS_MAXIDLEN)
len = MS_MAXIDLEN;
msg.body.getrequest.idlen = htons(len);
strncpy(msg.body.getrequest.imageid, imageid, MS_MAXIDLEN);
strncpy((char *)msg.body.getrequest.imageid, imageid, MS_MAXIDLEN);
len = sizeof msg.type + sizeof msg.body.getrequest;
if (!MsgSend(msock, &msg, len, timeout)) {
......@@ -641,33 +646,16 @@ ClientNetFindServer(struct in_addr *sip, char *imageid,
}
close(msock);
err = ntohs(msg.body.getreply.error);
if (!err) {
mcastaddr.s_addr = msg.body.getreply.addr;
portnum = (int)ntohs(msg.body.getreply.port);
}
if (statusonly) {
if (err)
fprintf(stderr, "%s: server denied access (%d)\n",
imageid, err);
else if (msg.body.getreply.isrunning)
fprintf(stderr, "%s: server running %s:%d\n",
imageid, inet_ntoa(mcastaddr), portnum);
else
fprintf(stderr, "%s: access allowed, methods=0x%x\n",
imageid, msg.body.getreply.method);
return 1;
}
if (err)
fprintf(stderr, "%s: server returned error %d\n",
imageid, err);
else
fprintf(stderr, "%s: address: %s:%d\n",
imageid, inet_ntoa(mcastaddr), portnum);
/*
* Convert the reply info to host order
*/
*reply = msg.body.getreply;
reply->error = ntohs(reply->error);
reply->addr = ntohl(reply->addr);
reply->port = ntohs(reply->port);
reply->sigtype = ntohs(reply->sigtype);
return (err == 0);
return 1;
}
#endif
......@@ -594,3 +594,96 @@ ClientStatsDump(unsigned int id, ClientStats_t *stats)
}
}
#endif
#ifdef MASTER_SERVER
#include "configdefs.h"
char *
GetMSError(int error)
{
char *err;
switch (error) {
case MS_ERROR_FAILED:
err = "server authentication error";
break;
case MS_ERROR_NOHOST:
err = "unknown host";
break;
case MS_ERROR_NOIMAGE:
err = "unknown image";
break;
case MS_ERROR_NOACCESS:
err = "access not allowed";
break;
case MS_ERROR_NOMETHOD:
err = "not available via specified method";
break;
case MS_ERROR_INVALID:
err = "invalid argument";
break;
default:
err = "unknown error";
break;
}
return err;
}
char *
GetMSMethods(int methods)
{
static char mbuf[256];
mbuf[0] = '\0';
if (methods & MS_METHOD_UNICAST) {
if (mbuf[0] != '\0')
strcat(mbuf, "/");
strcat(mbuf, "unicast");
}
if (methods & MS_METHOD_MULTICAST) {
if (mbuf[0] != '\0')
strcat(mbuf, "/");
strcat(mbuf, "multicast");
}
if (methods & MS_METHOD_BROADCAST) {
if (mbuf[0] != '\0')
strcat(mbuf, "/");
strcat(mbuf, "broadcast");
}
if (mbuf[0] == '\0')
strcat(mbuf, "UNKNOWN");
return mbuf;
}
void
PrintGetInfo(char *imageid, GetReply *reply)
{
if (reply->error) {
log("%s: server denied access: %s",
imageid, GetMSError(reply->error));
return;
}
if (reply->isrunning) {
struct in_addr in;
in.s_addr = htonl(reply->addr);
log("%s: access allowed, server running at %s:%d using %s",
imageid, inet_ntoa(in), reply->port,
GetMSMethods(reply->method));
} else
log("%s: access allowed, available methods=%s",
imageid, GetMSMethods(reply->method));
switch (reply->sigtype) {
case MS_SIGTYPE_MTIME:
{
time_t mt = ntohl(*(time_t *)reply->signature);
log(" modtime=%s", ctime(&mt));
}
default:
break;
}
}
#endif
......@@ -71,6 +71,11 @@ int BlockMapFirst(BlockMap_t *blockmap);
int BlockMapApply(BlockMap_t *blockmap, int chunk,
int (*func)(int, int, int, void *), void *farg);
void ClientStatsDump(unsigned int id, ClientStats_t *stats);
#ifdef MASTER_SERVER
char *GetMSError(int error);
char *GetMSMethods(int methods);
void PrintGetInfo(char *imageid, GetReply *reply);
#endif
/* Compat */
#define CHUNKSIZE ChunkSize(-1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment