disk-agent.cc 44.1 KB
Newer Older
1
2
/*
 * EMULAB-COPYRIGHT
3
 * Copyright (c) 2010-2012 University of Utah and the Flux Group.
4
5
6
 * All rights reserved.
 */

7
/* This program implements the Disk agent for Emulab.
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 * It listens to objtype "disk" and creates/modifies/ device mapper(DM)
 * disks. Also, we can inject errors of various types on these DM disks.
 */

#include <cassert>
#include <cmath>
#include <ctime>
#include <string>
#include <map>
#include <list>
#include <vector>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <fstream>
23
24
25
26
27
#include <paths.h>
#include <fcntl.h>
#include <dirent.h>
#include <sys/types.h>
#include <sys/resource.h>
28
#include <cerrno>
29
#include "log.h"
30
31
#include <cstdio>
#include <algorithm>
32
#include <dirent.h>
33
34
#include <ctype.h>
extern "C" {
35
	#include "libdevmapper.h"
Mike Hibler's avatar
Mike Hibler committed
36
	#include "be_user.h" 
37
38
}

39
40
41
42
#ifdef HAVE_ELVIN
#include <elvin/elvin.h>
#endif

43
44
45
46
47
48
49
50
51
52
53
54
55
#include <sys/time.h>

using namespace std;

// For getopt
#include <unistd.h>

#include "event.h"
#include "tbdefs.h"
#define LINE_SIZE 4096
#define MAX_BUFFER 4096
#define err(msg, x...) fprintf(stderr, msg "\n", ##x)

56
57
58
/* Hard coding the number of disks we allow for now to keep things simpler */
#define MAX_DISKS 10

59
/* A flag to init vols only when required */
60
61
int done_init_vols = 0;

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
/**
 * Structure used to track individual agents.
 */
struct diskinfo {
    char        name[TBDB_FLEN_EVOBJNAME];

    char           *initial_cmdline;
    char           *cmdline;
	char 		   *type;
	char 		   *initial_type;
	char		   *mountpoint;
	char		   *initial_mountpoint;
	char		   *parameters;
	char		   *initial_parameters;	
    int     pid;
77
	unsigned long   token;
78
79
80
81
82
83
84
85
    struct diskinfo *next;
};

/**
 * Refers to the head of the agent list.
 */
static struct diskinfo *diskinfos;

86
87
88
89
90
91
92
/* Maximum allowed disks */
static int numdisks;

/* Maximum allowed disk partitions */
static int volindex = 0;

static char *LOGDIR = "/local/logs";
93

94
95
96
97
98
99
100
101
102
103
104
105
106
107
char * _table = NULL;

enum { EVENT_BUFFER_SIZE = 5000 };

namespace g
{
  std::string experimentName;
  bool debug = false;
}

vector<string> device_params;

void readArgs(int argc, char * argv[]);
void usage(char * name);
108
109
110
111
112
113
114
115
116
117
118
119
120

/**
 * Parse the configuration file containing the list of agents and their initial
 * settings.
 *
 * @param filename The name of the config file.
 * @return Zero on success, -1 otherwise.
 */
static int  parse_configfile(char *filename);
static void set_disk(struct diskinfo *dinfo, char *args);



121
122
123
124
125
126
127
128
129
130
131
// Reads the map file, initializes the pipe and pipeVault data
// structure, and sets up the two subscription strings for events.
void writePidFile(string const & pidFile);
void initEvents(string const & server, string const & port,
                string const & keyFile, string const & subscription,
                string const & group);
void subscribe(event_handle_t handle, address_tuple_t eventTuple,
               string const & subscription, string const & group);
void callback(event_handle_t handle,
              event_notification_t notification, void *data);

132
133
/**
 * Handler for the TIME start event.  This callback will stop all running
134
 * disks, reset the disk agent configuration to the original version specified in
135
136
137
138
139
140
141
142
143
144
145
 * the config file, and delete all the files in the log directory.
 *
 * @param handle The connection to the event system.
 * @param notification The start event.
 * @param data NULL
 */
static void start_callback(event_handle_t handle,
                   event_notification_t notification,
                   void *data);


146
//DM Device routines
147
148
149
int create_dm_device(struct diskinfo *dinfo, char *);
int run_dm_device(struct diskinfo *dinfo, char *);
int modify_dm_device(struct diskinfo *dinfo, char *);
150
151
152
153
154
155
156
157
158
int resume_dm_device(char *);
static int _device_info(char *);
static int _parse_line(struct dm_task *, char *, const char *,int);
static int _parse_file(struct dm_task *, const char *);
static void _display_info_long(struct dm_task *,  struct dm_info *);
int _get_device_params(const char *);
string exec_output(string);
string itos(int );

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
/* Initialize the partitions that will be used by disk-agent.
 * LVM is being used to make logical volumes to ease the pain
 * instead of having to deal with physical volumes.
 * We allow 10 disks as of now. Maybe we should change this in
 * the future.
 */
void init_volumes(void)
{
        string cmd;

        /* Since creating partitions on real disk is painful, we
         * make use of lvm and give logical volumes to host disks.
         * mkextrafs script which will create 10 logical volumes.
         */
        cout << "Creating the disk partition ..." << endl;
174
        cmd = "/usr/testbed/bin/mkextrafs -f -q -l -m vol1,vol2,vol3,vol4,vol5,vol6,vol7,vol8,vol9,vol10";
175
176
177
178
179
180
181
182
183
        int i = system(const_cast<char *>(cmd.c_str()));
        if(i)
        {
                cerr << "Failed to init logical vols" <<endl;
                exit(1);
        }

}

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208

/**
 * Dump the diskinfos list to standard out.
 */
static void
dump_diskinfos(void)
{
    struct diskinfo *pi;

    for (pi = diskinfos; pi != NULL; pi = pi->next) {
        printf(
               "  Name: %s\n"
               "  type: %s\n"
               "  mountpoint: %s\n"
               "  parameters: %s\n"
               "  command: %s\n",
               pi->name ? pi->name : "(not set)",
               pi->type ? pi->type : "(not set)",
               pi->mountpoint ? pi->mountpoint : "(not set)",
               pi->parameters ? pi->parameters : "(not set)",
               pi->cmdline ? pi->cmdline : "(not set)"
               );
    }
}

209
/*
210
 * Open a new log file for a disk agent and update the symlink used to refer
211
212
 * to the latest invocation of the agent.
 *
213
 * @param dinfo The dinfo we are opening the logs for.
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
 * @param type The log type: "out" for standard out and "err" for standard
 * error.
 * @return The new file descriptor or -1 if there was a failure.
 */
static int
open_logfile(struct diskinfo *dinfo, const char *type)
{
    char buf[BUFSIZ], buf2[BUFSIZ];
    int error = 0, retval;

    assert(dinfo != NULL);
    assert(type != NULL);
    assert((strcmp(type, "out") == 0) || (strcmp(type, "err") == 0));

    /*
     * Construct the name of the log file, which is made up of the program
     * agent name, the event token, and the type (e.g. out, err).
     */
    snprintf(buf, sizeof(buf),
         "%s/%s.%s.%lu",
         LOGDIR, dinfo->name, type, dinfo->token);
    if ((retval = open(buf, O_WRONLY|O_CREAT|O_TRUNC, 0640)) < 0) {
        errorc("could not create log file %s\n", buf);
    }
    else {
        /*
         * We've successfully created the file, now create the
         * symlinks to that refer to the last run and a tagged run.
         */
        snprintf(buf, sizeof(buf),
             "./%s.%s.%lu",
             dinfo->name, type, dinfo->token);
        snprintf(buf2, sizeof(buf2),
             "%s/%s.%s",
             LOGDIR, dinfo->name, type);
        if ((unlink(buf2) < 0) && (errno != ENOENT)) {
            error = 1;
            errorc("could not unlink old last run link %s\n",
                   buf2);
        }
        else if (symlink(buf, buf2) < 0) {
            error = 1;
            errorc("could not symlink last run %s\n", buf);
        }
        #if 0
        if (dinfo->tag != NULL) {
            snprintf(buf2, sizeof(buf2),
                 "%s/%s.%s.%s",
                 LOGDIR, dinfo->name, dinfo->tag, type);
            if ((unlink(buf2) < 0) && (errno != ENOENT)) {
                error = 1;
                errorc("could not unlink old tag link %s\n",
                       buf2);
            }
            else if (symlink(buf, buf2) < 0) {
                error = 1;
                errorc("could not symlink tagged run %s\n",
                       buf);
            }
        }
        #endif

    }

    if (error) {
        close(retval);
        retval = -1;
    }

    return retval;
}

/*
 * Returns the disk agent structure given the disk name
 */
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
static struct diskinfo *
find_agent(char *name)
{
    struct diskinfo *dinfo, *retval = NULL;

    dinfo = diskinfos;
    while (dinfo) {
        if (! strcmp(dinfo->name, name)) {
            retval = dinfo;
            break;
        }
        dinfo = dinfo->next;
    }

    return retval;
}
305
306
307

int main(int argc, char * argv[])
{
308
  cout << "Beginning program" << endl;
309
310
311
312
  readArgs(argc, argv);
  return 0;
}

313
314
315
/*
 * Parse the command line arguments and initialize the event system
 */
316
317
void readArgs(int argc, char * argv[])
{
318
  cout << "Processing arguments" << endl;
319
320
  string server;
  string port;
321
  char *logfile = NULL;
322
  char *pidfile = NULL;
323
  char *configfile;
324
  string keyFile;
325
  string tokenfile;
326
  string subscription;
327
  string vnode;
328
  string group;
Mike Hibler's avatar
Mike Hibler committed
329
  string user;
330
331
  char buf[MAX_BUFFER];
  FILE *fp;
332
333
334
335
336

  // Prevent getopt from printing an error message.
  opterr = 0;

  /* get params from the optstring */
Mike Hibler's avatar
Mike Hibler committed
337
  char const * argstring = "hds:p:l:u:i:e:c:k:o:g:v:t:U:";
338
339
340
341
342
  int option = getopt(argc, argv, argstring);
  while (option != -1)
  {
    switch (option)
    {
343
344
345
	case 'h':
	  usage(argv[0]);
	  break;
346
347
348
349
350
351
352
353
354
355
    case 'd':
      g::debug = true;
      pubsub_debug = 1;
      break;
    case 's':
      server = optarg;
      break;
    case 'p':
      port = optarg;
      break;
356
357
358
359
360
361
362
	case 'l':
	  logfile = optarg;
	  break;
	case 'c':
	  configfile = optarg;
	  break;
    case 'e':
363
364
365
366
367
368
369
370
      g::experimentName = optarg;
      break;
    case 'k':
      keyFile = optarg;
      break;
    case 'u':
      subscription = optarg;
      break;
371
372
373
374
375
376
377
378
379
380
381
382
	case 'o':
	  LOGDIR = optarg;
	  break;
	case 'i':
	  pidfile = optarg;
	  break;
	case 't':
	  tokenfile = optarg;
	  break;
	case 'v':
	  vnode = optarg;
	  break;
383
384
385
    case 'g':
      group = optarg;
      break;
Mike Hibler's avatar
Mike Hibler committed
386
387
388
    case 'U':
      user = optarg;
      break;
389
390
391
392
393
394
395
396
    default:
      usage(argv[0]);
      break;
    }
    option = getopt(argc, argv, argstring);
  }

  /*Check if all params are specified, otherwise, print usage and exit*/
Mike Hibler's avatar
Mike Hibler committed
397
  if(server == "" || user == "" || g::experimentName == "")
398
399
      usage(argv[0]);

400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
  /*
   * Write out a pidfile if root.
   */
  if (!getuid()) {
		if (pidfile)
            strcpy(buf, pidfile);
        else
            sprintf(buf, "%s/diskagent.pid", _PATH_VARRUN);
        fp = fopen(buf, "w");
        if (fp != NULL) {
            fprintf(fp, "%d\n", getpid());
            (void) fclose(fp);
        }
  }

Mike Hibler's avatar
Mike Hibler committed
415
416
  be_user(const_cast<char *>(user.c_str()));

417
  if(g::debug)
418
419
420
421
422
423
	loginit(0, logfile);
  else {
	if(logfile)
		loginit(0, logfile);
	else
		loginit(1, "disk-agent");
424
  }
425
426
  //if(subscription == "")
	//subscription = "DISK";
427
428
	
  if (parse_configfile(configfile) != 0)
429
430
  {
      cerr << "Error while parsing config file" <<endl;
431
      exit(1);
432
  }
433
434


435
436
437
438
439
  initEvents(server, port, keyFile, subscription, group);
}

void usage(char * name)
{
Mike Hibler's avatar
Mike Hibler committed
440
  cerr << "Usage: " << name << " -e proj/exp -s server -U user [-h][-d] [-p port] "
441
       << "[-l logfile] [-c config file] [-i pidFile] [-k keyFile] [-u subscription] [-g group]" << endl;
442
443
444
  exit(-1);
}

445
446
447
/*
 * Initialize the event system and listen for notifications
 */
448
449
450
451
void initEvents(string const & server, string const & port,
                string const & keyFile, string const & subscription,
                string const & group)
{
452
  cout << "Initializing event system" << endl;
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
  string serverString = "elvin://" + server;
  event_handle_t handle;
  if (port != "")
  {
    serverString += ":" + port;
  }
  cerr << "Server string: " << serverString << endl;
  if (keyFile != "")
  {
    handle = event_register_withkeyfile(const_cast<char *>(serverString.c_str()), 0,
                                        const_cast<char *>(keyFile.c_str()));
  }
  else
  {
    handle = event_register_withkeyfile(const_cast<char *>(serverString.c_str()), 0, NULL);
  }
  if (handle == NULL)
  {
    cerr << "Could not register with event system" << endl;
    exit(1);
  }

  address_tuple_t eventTuple = address_tuple_alloc();

  subscribe(handle, eventTuple, subscription, group);

  address_tuple_free(eventTuple);

481
482
483
484
485
486
487
488
489

  /*
   * Begin the event loop, waiting to receive event notifications:
   */
 
  while(1) {
	if(event_main(handle) == 0)
		cerr << "Event main stopped!" << endl;
  }
490
491
}

492
493
494
/*
 * Subscribes to the events we want from the event system.
 */
495
496
497
void subscribe(event_handle_t handle, address_tuple_t eventTuple,
               string const & subscription, string const & group)
{
498
499
500
501
  char agentlist[MAX_BUFFER];
  bzero(agentlist, sizeof(agentlist));
  struct diskinfo *dinfo;

502
503
504
505
506
  string name = subscription;
  if (group != "")
  {
    name += "," + group;
  }
507
508
509
510
511
512
513
514
515
516
517
518
519
  /*
   * Cons up the agentlist for subscription below.
   */
  dinfo = diskinfos;
  while (dinfo) {

  	if (strlen(agentlist))
        strcat(agentlist, ",");
    	strcat(agentlist, dinfo->name);

        dinfo = dinfo->next;
  }

520
  eventTuple->objname = agentlist;
521
  eventTuple->objtype = TBDB_OBJECTTYPE_DISK;
522
523
524
525
526
  eventTuple->eventtype = 
				TBDB_EVENTTYPE_START ","
				TBDB_EVENTTYPE_RUN ","
				TBDB_EVENTTYPE_CREATE ","
				TBDB_EVENTTYPE_MODIFY;
527
528
529
530
531
532
533
  eventTuple->expt = const_cast<char *>(g::experimentName.c_str());
  eventTuple->host = ADDRESSTUPLE_ANY;
  eventTuple->site = ADDRESSTUPLE_ANY;
  eventTuple->group = ADDRESSTUPLE_ANY;
  if (event_subscribe(handle, callback, eventTuple, NULL) == NULL)
  {
    cerr << "Could not subscribe to " << eventTuple->eventtype << " event" << endl;
534

535
  }
536
537
538
539
540
541
542
543
544
545
546
547
  
    eventTuple->objtype   = TBDB_OBJECTTYPE_TIME;
    eventTuple->objname   = ADDRESSTUPLE_ANY;
    eventTuple->eventtype = TBDB_EVENTTYPE_START;

    /*
     * Subscribe to the TIME start event we specified above.
     */
    if (! event_subscribe(handle, start_callback, eventTuple, NULL)) {
        cerr << "could not subscribe to event" << endl;
    }

548
549
}

550
551
552
553
554
/*
 * Event handler:
 * Receives notification from the event system.
 * Calls appropriate event handler.
 */
555
556
557
558
559
560
561
void callback(event_handle_t handle,
              event_notification_t notification,
              void * data)
{
  char name[EVENT_BUFFER_SIZE];
  char type[EVENT_BUFFER_SIZE];
  char args[EVENT_BUFFER_SIZE];
562
563
  unsigned long   token = ~0;
  int pid, in_fd, out_fd = -1, err_fd = -1;
564
  
565
566
  struct diskinfo *dinfo;

567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
  struct timeval basicTime;
  gettimeofday(&basicTime, NULL);
  map<string, int> eventtype;

  eventtype["CREATE"]  = 0;
  eventtype["MODIFY"]  = 1;
  eventtype["STOP"]    = 2;
  eventtype["RUN"]     = 3;
  eventtype["START"]   = 3;

  double floatTime = basicTime.tv_sec + basicTime.tv_usec/1000000.0;
  ostringstream timeStream;
  timeStream << setprecision(3) << setiosflags(ios::fixed | ios::showpoint);
  timeStream << floatTime;
  string timestamp = timeStream.str();


  if (event_notification_get_string(handle, notification, const_cast<char *>("OBJNAME"), name, EVENT_BUFFER_SIZE) == 0)
  {
    cerr << timestamp << ": ERROR: Could not get the object name" << endl;
    return;
  }

590

591
592
593
594
595
596
  if (event_notification_get_string(handle, notification, const_cast<char *>("EVENTTYPE"), type, EVENT_BUFFER_SIZE) == 0)
  {
    cerr << timestamp << ": ERROR: Could not get the event type" << endl;
    return;
  }
  string event = type;
597
	 
598
599
600
601
602
603
604
605
  event_notification_get_string(handle, notification,const_cast<char *>("ARGS"), args, EVENT_BUFFER_SIZE); 
  
 
  /* Find the agent and */
  dinfo = find_agent(name);
  if (!dinfo) {
      cout << "Invalid disk agent: "<< name <<endl;
      return;
606
607
  }

608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
  event_notification_get_int32(handle, notification,
                     "TOKEN", (int32_t *)&token);

    /*
     * The command is going to be run via the shell.
     * We do not know anything about the command line, so we reinit
     * the log file so that any output goes into a different file. If
     * the user command line includes a redirection, that will get the
     * output since the sh will set up the descriptors appropriately.
     */
    if (((out_fd = open_logfile(dinfo, "out")) < 0) ||
        ((err_fd = open_logfile(dinfo, "err")) < 0)) {
        if (out_fd >= 0)
            close(out_fd);
        if (err_fd >= 0)
            close(err_fd);

        cerr << "could not setup log files" <<endl;
        exit(1);
    }

    if (out_fd != STDOUT_FILENO) {
        dup2(out_fd, STDOUT_FILENO);
        close(out_fd);
    }
    if (err_fd != STDERR_FILENO) {
        dup2(err_fd, STDERR_FILENO);
        close(err_fd);
    }

    close(STDIN_FILENO);
    if ((in_fd = open(_PATH_DEVNULL, O_RDONLY)) >= 0) {
        if (in_fd != STDIN_FILENO) {
            dup2(in_fd, STDIN_FILENO);
            close(in_fd);
        }
    }

646
647
648
649
  /* Call the event handler routine based on the event */
  switch(eventtype[event])
  {
	case 0:
650
651
652
653
654
655
656
657
		/* Avoid creating lvm partitions until needed
  		   done_init_flags is being used for this purpose
		 */
		if(!done_init_vols) {
			init_volumes();
			done_init_vols=1;
		}

658
		/* Event is to create a dm disk */	
659
		if(!create_dm_device(dinfo, args))
660
			cerr << "DM failed in create_dm_device()" << endl;
661
662
663
		event="";
		break;
  	case 1:
664
665
666
667
668
669
670
671
        /* Avoid creating lvm partitions until needed
           done_init_flags is being used for this purpose
         */
        if(!done_init_vols) {
            init_volumes();
            done_init_vols=1;
        }

672
		/* Event is to modify the dm disk */
673
		if(!modify_dm_device(dinfo, args))
674
			cerr << "DM failed in modify_dm_device()" << endl;
675
676
677
678
679
		event="";
	 	break;	
	case 2:
		break;
	case 3:
680
681
682
683
684
685
686
687
        /* Avoid creating lvm partitions until needed
           done_init_flags is being used for this purpose
         */
        if(!done_init_vols) {
            init_volumes();
            done_init_vols=1;
        }

688
		if(!run_dm_device(dinfo, args))
689
			 cerr << "DM failed in run_dm_device()" << endl;
690
691
692
		event="";
		break;
	default:
693
		cerr << "Don't recognize the event type" << endl;
694
695
696
697
  }
 
}

698
699
700
701
702
703
/*
 * Handles case where user specifies a disk type and a mountpoint.
 * This routine creates a new disk if it does not exist and mounts it.
 * If the disk already exists then it will simply change the parameters
 * specified by the user.
 */
704
int run_dm_device(struct diskinfo *dinfo, char *args)
705
706
707
{
	struct dm_task *dmt;

708
	set_disk(dinfo, args);
Yathindra Naik's avatar
Yathindra Naik committed
709
710
711
712
713

	/* Check if the required parameters for this event is supplied 
	 */
	if(!dinfo->type || !dinfo->mountpoint) 
	{
Yathindra Naik's avatar
Yathindra Naik committed
714
		cerr << "Disk type or mountpoint not specified" <<endl;
Yathindra Naik's avatar
Yathindra Naik committed
715
716
717
		return 0;
	}
		
718
    /* DEBUG */
Yathindra Naik's avatar
Yathindra Naik committed
719
    cout <<"Event START/RUN"<<endl;
720
    dump_diskinfos();
721

722
723

	if (_device_info(dinfo->name)) {
724
725
726
727
728
729
730
731
732
733
734
		/* DM device exists so we'll reload it with params supplied */
		uint64_t start=0, length=0;
	    char *target_type=NULL, *params=NULL;
		
		string params_str="", diskname="";
		stringstream split;

	 	if (!(dmt = dm_task_create(DM_DEVICE_RELOAD))) {
            cout << "in dm task create"<<endl;
            return 0;
        }		
735
		if ( !dm_task_set_name(dmt, dinfo->name)){
736
737
738
739
			dm_task_destroy(dmt);
            return 0;
		}
	
740
		if (!(_get_device_params(dinfo->name))) {
741
742
743
			dm_task_destroy(dmt);
            return 0;
		}
744
		cout << device_params[0] << device_params[1] << device_params[2] << device_params[3] << device_params[4] << endl;
745
746
747
	
		start 		    = strtoul (device_params[0].c_str(),NULL,0);
		length 		    = strtoul (device_params[1].c_str(),NULL,0);
748
		target_type 	= dinfo->type;
749
750
751

		split		   << device_params[3];

752
		/* Get the diskname from dm parameters */
753
		getline(split,diskname,' ');
Yathindra Naik's avatar
Yathindra Naik committed
754
755
756
		
		if(dinfo->parameters == NULL)
			dinfo->parameters = " ";
757
758
759
760
761
762
763
764
	
		/* For error DM type, we might not the disk name.
		 * So we store the major:minor number in device_params[4].
		 * Useful for converting error disk back to linear.
		 */  
           	
		 if(diskname == "")
			diskname = device_params[4];
Yathindra Naik's avatar
Yathindra Naik committed
765

766
 		params_str 	= diskname + " " + "0 "+ dinfo->parameters;
767
768
769
		cout << "diskname after "<<params_str<<endl;
		params 		= const_cast<char *>(params_str.c_str());

770
		cout <<"start: "<<start<<" "<<"len: "<<length<<" "<<"target_type: "<<target_type<<"parameters: "<<params<<endl;	
771
772
		
		if (!dm_task_add_target(dmt, start, length, target_type, params)) {
773
774
				dm_task_destroy(dmt);
                return 0;
775
776
777
		}
		/* Now that we have the dm ready; Run it. */
		if (!dm_task_run(dmt)){
778
779
				dm_task_destroy(dmt);
                return 0;
780
		}
781
782
783
784
785
786
		/* After running dm_task_run, dm_devices would
			not appear in /dev/mapper. Need to call these
			routines to fix that.
		*/
		dm_udev_wait(0);
		dm_task_update_nodes();
787
788

		/* Resume this dm device for changes to take effect. */
789
		resume_dm_device(dinfo->name);
790
791
792
793
794
795
796
797
798
799
800
		dm_task_destroy(dmt);
		return 1;

	}	
 	else {
		char *ttype=NULL, *ptr=NULL;
		unsigned long long start, size;
		string str,cmd;

		/* DM device does not exist. So we'll create it. */
		if(!(dmt = dm_task_create(DM_DEVICE_CREATE))){
801
802
803
        	cout << "in dm task create"<<endl;
	        return 0;
       	}
804
805

		/* Set properties on the new dm device */
806
		string prefix = "/dev/mapper/";
Yathindra Naik's avatar
Yathindra Naik committed
807
808
809
810
811
812
813
		
		if(dinfo->name == NULL)
		{
			cerr << "Disk name not specified!" <<endl;
			return 0;
		}

814
815
		string dm_disk_name = prefix + dinfo->name;
		if (!dm_task_set_name(dmt, dinfo->name)) {
816
817
818
819
			cout << "in task set name"<<endl;
			dm_task_destroy(dmt);
			return 0;
		}
820
821
822
823
824
825
826

		/* 
		 * Need to find out which logical volume is free;
		 * We can check this based on the open count of
  		 * logical volumes. Simple scan through the list 
         * logical volumes that we have created looking for
		 * the first free volume.
827
		 */
828
829
		volindex  = 1;
		int count = 0;
830
		do
Yathindra Naik's avatar
Yathindra Naik committed
831
		{
832
				++count;
833
				cmd = "lvdisplay emulab/vol"+itos(volindex)+" | grep open | cut -d ' ' -f 21-21";
834
835
836
				string open_count_str = exec_output(cmd);
				int open_count = atoi(const_cast<char *>(open_count_str.c_str()));     
				if(open_count > 0)
837
					++volindex;
838
				else
839
         	    	break;
840
841
842
843
844
845
		}while(count <= 10);

		if(count == 10)
		{
				cerr << "No logical volumes free to map disks!" <<endl;
				return 0;
Yathindra Naik's avatar
Yathindra Naik committed
846
		}
847

848
		cmd = "mkfs.ext3 /dev/emulab/vol"+itos(volindex);
849
850
851
852
853
854
855
856
857
858
859
		int i = system(const_cast<char *>(cmd.c_str()));
		if(i)
		{
				if(volindex > 1)
					--volindex;
				cerr << "mkfs failed for /dev/emulab/vol"+volindex <<endl ;
				dm_task_destroy(dmt);
				return 0;
		}
		string disk = "/dev/emulab/vol"+itos(volindex);

860
861
862
		/* So we have the new partition. Find out the
		 * size of partition to create the new dm disk.
		 */
863
		cmd = "blockdev --getsz "+disk;
864
865
866
		string str_size = exec_output(cmd);
		if (str_size == "") {
			dm_task_destroy(dmt);
867
            return 0;
868
869
		}
		
870
		cout << "Mapping the virtual disk " << dinfo->name << "on "<<disk<< endl; 
871
872
		/*
		 * Hardcoding all the values.
873
874
875
876
877
		 * Users not to worry about the geometry of the disk such as the start
		 * sector, end sector, size etc
		 */
		start = 0;
		size = strtoul(const_cast<char *>(str_size.c_str()), NULL, 0);
878
879
880
		ttype = dinfo->type;
		if (dinfo->parameters != NULL) {
			str = dinfo->parameters;
881
882
883
884
			str.erase(std::remove(str.begin(), str.end(), '\n'), str.end());
			string params = disk + " 0 " + str;
			params.erase(std::remove(params.begin(), params.end(), '\n'), params.end());
			ptr = const_cast<char *>(params.c_str());
885
			cout <<"PARAMETERS: "<<params<<endl;
886
887
888
889
890
891
		}
	
		else {
			string params = disk+" 0";
			params.erase(std::remove(params.begin(), params.end(), '\n'), params.end());
			ptr = const_cast<char *>(params.c_str());
892
			cout <<"PARAMETERS: "<<params<<endl;
893
894
895
896
897
898
899
900
901
902
903
904
905
906
		}

		cout <<start<<" "<<size<<" "<<ttype<<" "<<ptr<<endl;
		if (!dm_task_add_target(dmt, start, size, ttype, ptr)) {
			dm_task_destroy(dmt);
			return 0;
		}
		/* Now that we have the dm ready; Run it. */
		if (!dm_task_run(dmt)){
			cout <<"in task run"<<endl;
			dm_task_destroy(dmt);
			return 0;
		}

907
908
		dm_udev_wait(0);
        dm_task_update_nodes();
909

910
911
912
913
914
915
916
917
918
		dm_disk_name.erase(std::remove(dm_disk_name.begin(), dm_disk_name.end(), ' '), dm_disk_name.end());

		_device_info(const_cast<char *>(dm_disk_name.c_str()));
		if(dinfo->mountpoint == NULL)
		{
				cerr << "Mountpoint not specified!" << endl;
				dm_task_destroy(dmt);
				return 0;
		}
919
		str = dinfo->mountpoint;
920
		cmd = "mkdir -p "+str;
921
922
923
924
925
926
927
		i = system(const_cast<char *>(cmd.c_str()));
		if(i)
		{
				cerr << "Could not create mountpoint" <<endl;
				dm_task_destroy(dmt);
				return 0;
		}
928

929
		cmd = "mount -t ext3 "+dm_disk_name+" "+str;
930
931
932
933
934
935
936
937
938
		i = system(const_cast<char *>(cmd.c_str()));
		if(i)
		{
				--volindex;
				cerr << "mount failed " <<endl;
				dm_task_destroy(dmt);
				return 0;
		}
		cout << dinfo->name << " is mounted on " <<str <<endl;
939
940
941
		dm_task_destroy(dmt);
		return 1;
	}
942
	return 1;
943
944
945
946
947
948
949
950
}



/* This routine will create a device mapper(DM) device.
* It takes the arguments through the buffer and
* name specifies the name of the DM device.
*/
951
int create_dm_device(struct diskinfo *dinfo, char *args)
952
953
954
955
956
{
	struct dm_task *dmt;
	string str="";
	int r=0;

957
	set_disk(dinfo, args);
Yathindra Naik's avatar
Yathindra Naik committed
958

959
960
961
    /* DEBUG */
    cout <<"Event CREATE"<<endl;
    dump_diskinfos();
962

963
    if(dinfo->cmdline == NULL) {
964
        cerr << "Cmdline is empty!" << endl;
965
966
        return 0;
    }
967
968
969
970
971
972
973

	/* Create a new dm device */
	if(!(dmt = dm_task_create(DM_DEVICE_CREATE))){
		cout << "in dm task create"<<endl;
		return 0;
	}

974
975
976
    /* Set properties on the new dm device */
	if (!dm_task_set_name(dmt, dinfo->name))
        goto out;
977

978
979
980
981
	/* Tokenize the command line */ 
	if(!_parse_line(dmt, dinfo->cmdline, "", 0)) {
		cout<<"in parse_line"<<endl;
		goto out;	
982
983
984
985
	}
	
	
	/* Now that we have the dm ready; Run it. */
986
    if (!dm_task_run(dmt)){
987
988
989
		cout <<"in task run"<<endl;
		goto out;
	}
990
991
992
993
	
	dm_udev_wait(0);
    dm_task_update_nodes();        

994
	if (!_device_info(dinfo->name)) {
995
996
997
998
999
		goto out;
	}

	r=1;
	out:
1000
       dm_task_destroy(dmt);
1001
1002
1003
1004
1005
1006
1007
	   return r; 
}

/* This routine modifies the properties of a DM device.
 * The new argumenets are specified through the buffer.
 * name specifies the DM deive name.
 */
1008
int modify_dm_device(struct diskinfo *dinfo, char *args)
1009
{
1010
	struct dm_task *dmt;
1011
1012
	int r=0;

1013
	set_disk(dinfo, args);
Yathindra Naik's avatar
Yathindra Naik committed
1014
1015


1016
1017
1018
    /* DEBUG */
    cout <<"Event MODIFY"<<endl;
    dump_diskinfos();
1019
1020


Yathindra Naik's avatar
Yathindra Naik committed
1021
1022
	if(dinfo->cmdline == NULL) {
		cout << "Cmdline is empty!" << endl;
1023
1024
		return 0;
	}
1025

1026
1027
1028
    /* Create a new dm device */
    if(!(dmt = dm_task_create(DM_DEVICE_RELOAD)))
        return 0;
1029

1030
1031
1032
    /* Set properties on the new dm device */
    if (!dm_task_set_name(dmt, dinfo->name))
        goto out;
1033

1034
1035
1036
1037
	/* Tokenize the cmdline */
    if(!_parse_line(dmt, dinfo->cmdline, "", 0))
        goto out;
    
1038

1039
1040
1041
    /* Now that we have the dm ready; Run it. */
    if (!dm_task_run(dmt))
        goto out;
1042

1043
1044
    dm_udev_wait(0);
    dm_task_update_nodes();
1045

1046
1047
1048
    if (!_device_info(dinfo->name)) 
        goto out;
	
1049
1050
 
	/* Resume this dm device for changes to take effect. */
1051
1052
1053
1054
1055
1056
	resume_dm_device(dinfo->name);
        
	r=1;
    out:
       dm_task_destroy(dmt);
       return r;
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
}


/* This routine MUST be called whenever we modify properties
 * of a DM device. Once this routine returns the modified
 * properties of the DM device is made alive.
 */
int resume_dm_device(char *name)
{
	/* dmt stores all information related to the device mapper */
	struct dm_task *dmt;
	int r=0;

	/* This creates a dm device to be RESUMED */
1071
1072
    if (!(dmt = dm_task_create(DM_DEVICE_RESUME)))
    	return 0;
1073
1074

	/* This sets up the name of dm device */
1075
1076
    if (!dm_task_set_name(dmt, name))
    	goto out;
1077

1078
	#if 0
1079
1080
1081
	/* This adds the dm device node */
        if (!dm_task_set_add_node(dmt, DM_ADD_NODE_ON_RESUME))
                goto out;
1082
	#endif
1083
1084
1085
1086
1087
1088
    /* Now that we have the dm device. Run it to make it alive. */
    if (!dm_task_run(dmt))
		goto out;

    dm_udev_wait(0);
    dm_task_update_nodes();
1089

1090
1091
1092
1093
1094
	/* DM disk is created; Query it. */
	r=1;
    out:
        dm_task_destroy(dmt);
        return r;
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
}

/*This routine basically sets the properties specified in the buffer
 *on the device mapper disk.
 */
static int _parse_line(struct dm_task *dmt, char *buffer, const char *file,
                       int line)
{
        char ttype[LINE_SIZE], *ptr;
        unsigned long long start, size;
        int n;

        /*trim trailing space */
        for (ptr = buffer + strlen(buffer) - 1; ptr >= buffer; ptr--)
                if (!isspace((int) *ptr))
                        break;
        ptr++;
        *ptr = '\0';

        /* trim leading space */
        for (ptr = buffer; *ptr && isspace((int) *ptr); ptr++)
                ;

        if (!*ptr || *ptr == '#')
                return 1;

        if (sscanf(ptr, "%llu %llu %s %n",
                   &start, &size, ttype, &n) < 3) {
1123
                printf("Invalid format on line %d of table %s", line, file);
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
                return 0;
        }

        ptr += n;

        if (!dm_task_add_target(dmt, start, size, ttype, ptr))
                return 0;

        return 1;
}

/*This routine reads the file line by line and passes the line
 *to _parse_line routine to set the properties for the device
 *mapper disk.
 */
static int _parse_file(struct dm_task *dmt, const char *file)
{
        char *buffer = NULL;
        size_t buffer_size = 0;
        FILE *fp;
        int r = 0, line = 0;

        /* one-line table on cmdline */
//        if (_table)
//                return _parse_line(dmt, _table, "", ++line);

        /* OK for empty stdin */
        if (file) {
                if (!(fp = fopen(file, "r"))) {
                        err("Couldn't open '%s' for reading", file);
                        return 0;
                }
        } else
                fp = stdin;

#ifndef HAVE_GETLINE
        buffer_size = LINE_SIZE;
        if (!(buffer = (char *)dm_malloc(buffer_size))) {
                err("Failed to malloc line buffer.");
                return 0;
        }

        while (fgets(buffer, (int) buffer_size, fp))
#else
        while (getline(&buffer, &buffer_size, fp) > 0)
#endif
                if (!_parse_line(dmt, buffer, file ? : "on stdin", ++line))
                        goto out;

        r = 1;

      out:
        memset(buffer, 0, buffer_size);
#ifndef HAVE_GETLINE
        dm_free(buffer);
#else
        free(buffer);
#endif
        if (file && fclose(fp))
                fprintf(stderr, "%s: fclose failed: %s", file, strerror(errno));

        return r;
}

1188
1189
1190
1191
1192
/*
 * Helper routine which executes a shell command
 * and returns the output. This comes in handy for
 * knowing the status of dm devices for example.
 */
1193
1194
1195
1196
1197
1198
1199
1200
1201
string exec_output(string cmd)
{
	// setup
	string data;
	FILE *stream;
	char buffer[MAX_BUFFER];

	// do it
	if (!(stream = popen(cmd.c_str(), "r"))) {
1202
		cerr << "Exec failed" << endl;
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
		data="";
		goto out;
	}
	while ( fgets(buffer, MAX_BUFFER, stream) != NULL )
		data.append(buffer);
	
	out:
	pclose(stream);
	return data;
}

1214
/* Prints the status of DM device */
1215
1216
1217
1218
1219
1220
1221
1222
static int _device_info(char *name)
{
	struct dm_task *dmt;
	struct dm_info info;       

	if (!(dmt = dm_task_create(DM_DEVICE_INFO)))
                return 0;

1223
1224
1225
	/* Set properties on the new dm device */
    if (!dm_task_set_name(dmt, name))
    	goto out;
1226
1227


1228
1229
	if (!dm_task_run(dmt))
        goto out;
1230

1231
1232
	dm_udev_wait(0);
    dm_task_update_nodes();
1233

1234
1235
    if (!dm_task_get_info(dmt, &info))
        goto out;
1236
1237
1238
1239
1240
1241
1242
1243
1244
	
	if (info.exists)
		_display_info_long(dmt, &info);

	out:
		dm_task_destroy(dmt);
        	return info.exists ? 1 : 0;
}

1245
/* Prints a detailed status of DM device */
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
static void _display_info_long(struct dm_task *dmt, struct dm_info *info)
{
        const char *uuid;
        uint32_t read_ahead;

        if (!info->exists) {
                printf("Device does not exist.\n");
                return;
        }

        printf("Name:              %s\n", dm_task_get_name(dmt));

        printf("State:             %s%s\n",
               info->suspended ? "SUSPENDED" : "ACTIVE",
               info->read_only ? " (READ-ONLY)" : "");

        /* FIXME Old value is being printed when it's being changed. */
        if (dm_task_get_read_ahead(dmt, &read_ahead))
                printf("Read Ahead:        %lu\n", read_ahead);

        if (!info->live_table && !info->inactive_table)
                printf("Tables present:    None\n");
        else
                printf("Tables present:    %s%s%s\n",
                       info->live_table ? "LIVE" : "",
                       info->live_table && info->inactive_table ? " & " : "",
                       info->inactive_table ? "INACTIVE" : "");
        if (dm_task_get_read_ahead(dmt, &read_ahead))
                printf("Read Ahead:        %lu\n", read_ahead);

        if (!info->live_table && !info->inactive_table)
                printf("Tables present:    None\n");
        else
                printf("Tables present:    %s%s%s\n",
                       info->live_table ? "LIVE" : "",
                       info->live_table && info->inactive_table ? " & " : "",
                       info->inactive_table ? "INACTIVE" : "");

        if (info->open_count != -1)
                printf("Open count:        %d\n", info->open_count);

        printf("Event number:      %lu\n", info->event_nr);
        printf("Major, minor:      %d, %d\n", info->major, info->minor);

        if (info->target_count != -1)
                printf("Number of targets: %d\n", info->target_count);

        if ((uuid = dm_task_get_uuid(dmt)) && *uuid)
                printf("UUID: %s\n", uuid);

        printf("\n");
}

1299
1300
1301
1302
/*
 * Extract the properties of existing DM device.
 * Useful when altering some properties of DM device.
 */
1303
1304
1305
1306
int _get_device_params(const char *name)
{
        struct dm_info info;
        struct dm_task *dmt;
1307
1308
		uint64_t start, length;
		char *target_type, *params;
1309
		string diskname;
1310

1311
1312
1313
		void *next = NULL;
		int r=0;
		unsigned long long size;
1314
		int minor_number;
1315
	
1316
1317
1318
		device_params.clear();
		if (!(dmt = dm_task_create(DM_DEVICE_TABLE)))
		return 0;
1319

1320
1321
		if (!dm_task_set_name(dmt,name))
				goto out;
1322

1323
1324
		if (!dm_task_run(dmt))
				goto out;
1325

1326
1327
1328
	    dm_udev_wait(0);
    	dm_task_update_nodes();

1329
		if (!dm_task_get_info(dmt, &info) || !info.exists)
1330
1331
                goto out;

1332
1333
		/* This is a hack for now to make things work with lvm vols
		   We create 10 lvm vols and the major:minor number decided
1334
		   by device mapper rolls back, so subtracting it by 10.
1335
1336
1337
1338
	       If we increase the lvm vols then we have to change this.
		*/

		minor_number = info.minor - 10;
1339
1340
		diskname = itos(info.major)+":"+itos(info.minor);
		
1341
1342
1343
1344
1345
        do {
                next = dm_get_next_target(dmt, next, &start, &length,
                                          &target_type, &params);
                size += length;
		
1346
1347
1348
1349
1350
				device_params.push_back(itos(start));
				device_params.push_back(itos(length));
				device_params.push_back(target_type);
				device_params.push_back(params);
				device_params.push_back(diskname);	
1351
1352
1353

        } while (next);

1354
1355
1356
		r=1;
		out:
		dm_task_destroy(dmt);
1357
1358
1359
        return r;
}

1360
1361
/* convert int to string */
string itos(int i)	
1362
1363
1364
1365
1366
{
	stringstream s;
	s << i;
	return s.str();
}
1367

1368
1369
1370
1371
/* 
 * Parse the config file which lists the initial 
 * disk configuration of disk agents.
 */
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
static int
parse_configfile(char *filename)
{
    FILE    *fp;
    char    buf[BUFSIZ];
	struct diskinfo *dinfo;

    assert(filename != NULL);
    assert(strlen(filename) > 0);

    if ((fp = fopen(filename, "r")) == NULL) {
1383
        cerr << "could not open configfile "<< filename <<endl;
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
        return -1;
    }

    while (fgets(buf, sizeof(buf), fp)) {
        int cc = strlen(buf);
        if (buf[cc-1] == '\n')
            buf[cc-1] = '\0';

		if(!strncmp(buf, "DISK", 4)) {
			char *value;
			int rc;
		
			dinfo = (struct diskinfo *) calloc(1, sizeof(*dinfo));
		
			if(!dinfo) {
1399
				cerr << "parse_configfile: out of memory" <<endl;
1400
1401
1402
1403
				goto bad;
			}

            if ((rc = event_arg_get(buf, "DISKNAME", &value)) <= 0) {
1404
                cerr << "parse_configfile: bad agent name" << endl;
1405
1406
1407
                goto bad;
            }
            else if (rc >= sizeof(dinfo->name)) {
1408
                cerr << "parse_configfile: agent name is too long" << endl; 
1409
1410
1411
1412
1413
                goto bad;
            }
            strncpy(dinfo->name, value, rc);
            dinfo->name[rc] = '\0';

Yathindra Naik's avatar
Yathindra Naik committed
1414
1415
            if ((rc = event_arg_get(buf,
                        "COMMAND",
1416
                        &dinfo->cmdline)) >= 0) {
Yathindra Naik's avatar
Yathindra Naik committed
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
                dinfo->cmdline[strlen(dinfo->cmdline) - 1] =
                    '\0'; // remove trailing single quote
                /* Prepend exec to replace the 'csh' */
                asprintf(&dinfo->cmdline,
                     "%s",
                     dinfo->cmdline);
            }
            dinfo->initial_cmdline = dinfo->cmdline;


            if ((rc = event_arg_get(buf,
                         "PARAMETERS",
1429
                         &dinfo->parameters)) >= 0) {
1430
1431
1432
1433
1434

				/* Since event_arg_get simply returns the entire string,
				   we need to prune out foreign chars */
				char *ptr = strstr(dinfo->parameters, "COMMAND=");
				if(*ptr != NULL)
1435
						*(ptr-1) = '\0';
1436

1437
1438
1439
				dinfo->parameters[strlen(dinfo->parameters) - 1] =
						'\0';

Yathindra Naik's avatar
Yathindra Naik committed
1440
1441
1442
1443
1444
1445
				asprintf(&dinfo->parameters,
                     "%s",
                     dinfo->parameters);
            }
            dinfo->initial_parameters = dinfo->parameters;

1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
            if ((rc = event_arg_dup(buf, "DISKTYPE", &dinfo->type)) == 0) {
            	free(dinfo->type);
				dinfo->type = NULL;			    
            }
			dinfo->initial_type = dinfo->type;			

            if ((rc = event_arg_dup(buf, "MOUNTPOINT", &dinfo->mountpoint)) == 0) {
                free(dinfo->mountpoint);
                dinfo->mountpoint = NULL;