libplab.py.in 45.1 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2
3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5
6
7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9
10
11
12
13
14
15
16
17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18
19
20
"""

import sys
21
22
23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
   
Kirk Webb committed
25
import traceback
Kirk Webb's avatar
   
Kirk Webb committed
26
import socket
Kirk Webb's avatar
Kirk Webb committed
27
28
29
import httplib
import xml.parsers.expat
import re
Kirk Webb's avatar
   
Kirk Webb committed
30
import calendar
Kirk Webb's avatar
   
Kirk Webb committed
31

Kirk Webb's avatar
   
Kirk Webb committed
32
from popen2 import Popen4
Kirk Webb's avatar
   
Kirk Webb committed
33
from warnings import warn
34

35
36
37
38
39
40
41
42
43
44
45
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
Kirk Webb's avatar
   
Kirk Webb committed
46
from mod_PLCNM import mod_PLCNM
47
48

agents = {'PLC'    : mod_PLC,
Kirk Webb's avatar
   
Kirk Webb committed
49
50
          'dslice' : mod_dslice,
          'PLCNM'  : mod_PLCNM}
51

Kirk Webb's avatar
Kirk Webb committed
52
53
54
55
56
57
#
# output control vars
#
verbose = 0
debug = 0

58
59
60
#
# Constants
#
Kirk Webb's avatar
   
Kirk Webb committed
61
DEF_AGENT = "PLCNM";
62

63
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
Kirk Webb's avatar
   
Kirk Webb committed
64
65
66

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
67
NODEPROBEINT  = 30
68

69
70
71
TBOPS = "@TBOPSEMAIL@".replace("\\","")
MAILTAG = "@THISHOMEBASE@"

72
RESERVED_PID = "emulab-ops"
73
RESERVED_EID = "hwdown"       # start life in hwdown
74
75
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
76

Kirk Webb's avatar
   
Kirk Webb committed
77
78
79
80
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18", )
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
81
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
82
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
Kirk Webb's avatar
   
Kirk Webb committed
83

84
ALLOWED_NODES = ()
Kirk Webb's avatar
   
Kirk Webb committed
85
86
87
88
89
#ALLOWED_NODES = ("128.112.139.80", "128.112.139.72", "169.229.50.4")
IGNORED_NODES = ("10283",)

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
90

Kirk Webb's avatar
   
Kirk Webb committed
91
PLABNODE = "@prefix@/sbin/plabnode"
92
SSH = "@prefix@/bin/sshtb"
93
94
NAMED_SETUP = "@prefix@/sbin/named_setup"

Kirk Webb's avatar
Kirk Webb committed
95
96
97
98
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
99

100
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
101
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
Kirk Webb's avatar
Kirk Webb committed
102
103
104
105
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
Kirk Webb's avatar
   
Kirk Webb committed
106
107
108
109
110
PLAB_SVC_SLICENAME = "utah_svc_slice"
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"

PLABEXPIREWARN = 2*24*60*60   # two days advance warning.
111

112
113
114
115
116
#
# var to track failed renewals
#
failedrenew = []

117
118
119
120
121
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

Kirk Webb's avatar
   
Kirk Webb committed
122
123
124
125
126
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

127

128
129
130
#
# Plab abstraction
#
Kirk Webb's avatar
Kirk Webb committed
131

Kirk Webb's avatar
Kirk Webb committed
132
133
134
#
# Class responsible for parsing planetlab sites file
#
Kirk Webb's avatar
Kirk Webb committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
            pass
        
        elif name == "HOST":
Kirk Webb's avatar
   
Kirk Webb committed
173
174
175
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
Kirk Webb's avatar
Kirk Webb committed
176
177
            self.__hosts.append({'HNAME' : attrs['NAME'],
                                 'IP'    : attrs['IP'],
Kirk Webb's avatar
   
Kirk Webb committed
178
179
                                 'PLABID': attrs['NODE_ID'],
                                 'MAC'   : attrs['MAC'],
Kirk Webb's avatar
Kirk Webb committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
                                 'SITE'  : self.__sitename})
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
            pass
        return

        
197
class Plab:
Kirk Webb's avatar
Kirk Webb committed
198
    def __init__(self, agent=None):
199
200
201
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
Kirk Webb's avatar
Kirk Webb committed
202
        if debug:
203
204
205
            print "Using module: %s" % self.agent.modname
            pass
        pass
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

Kirk Webb's avatar
Kirk Webb committed
223
    def updateNodeEntries(self, ignorenew = False):
224
        """
Kirk Webb's avatar
Kirk Webb committed
225
        Finds out which Plab nodes are available, and
226
227
228
229
230
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
Kirk Webb's avatar
Kirk Webb committed
231
        discovered.
232
233
234
235
236
237

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
238
239
240
        
        print "Getting available Plab nodes ..."

241
        avail = []
242
        try:
Kirk Webb's avatar
Kirk Webb committed
243
244
245
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
246
        # XXX: rewrite to use more elegant exception info gathering.
247
248
        except:
            extype, exval, extrace = sys.exc_info()
249
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
250
            if debug:
251
252
253
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
Kirk Webb's avatar
   
Kirk Webb committed
254

Kirk Webb's avatar
Kirk Webb committed
255
        if debug:
256
257
            print "Got advertisement list:"
            print avail
Kirk Webb's avatar
Kirk Webb committed
258
            pass
Kirk Webb's avatar
   
Kirk Webb committed
259

Kirk Webb's avatar
   
Kirk Webb committed
260
261
262
263
264
265
266
267
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
        if len(ALLOWED_NODES) or len(IGNORED_NODES):
            allowed = []
Kirk Webb's avatar
Kirk Webb committed
268
            for nodeent in avail:
Kirk Webb's avatar
   
Kirk Webb committed
269
270
271
272
273
274
275
276
277
                if nodeent['PLABID'] in IGNORED_NODES:
                    continue
                elif len(ALLOWED_NODES):
                    if nodeent['IP'] in ALLOWED_NODES:
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
278
279
                    pass
                pass
Kirk Webb's avatar
   
Kirk Webb committed
280
281
282
283
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
Kirk Webb's avatar
Kirk Webb committed
284
            pass
285

Kirk Webb's avatar
   
Kirk Webb committed
286
287
288
289
290
291
292
293
294
295
296
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

297
        # Get node info we already have.
298
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
299
        if debug:
300
301
            print "Got known pnodes:"
            print known
Kirk Webb's avatar
Kirk Webb committed
302
            pass
303

Kirk Webb's avatar
Kirk Webb committed
304
        # Create list of nodes to add or update
Kirk Webb's avatar
   
Kirk Webb committed
305
306
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
Kirk Webb's avatar
Kirk Webb committed
307
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
308
309
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
310
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
Kirk Webb's avatar
   
Kirk Webb committed
311
312
313
            res = self.__matchPlabNode(nodeent, known)
            if not res:
                toadd.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
314
                pass
Kirk Webb's avatar
   
Kirk Webb committed
315
316
            elif len(res[1]):
                toupdate.append(res)
Kirk Webb's avatar
Kirk Webb committed
317
                pass
Kirk Webb's avatar
Kirk Webb committed
318

Kirk Webb's avatar
   
Kirk Webb committed
319
320
        # Process the list of nodes to add
        addstr = ""
321
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
322
            # Are we ignoring new entries?
323
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
324
                if verbose:
325
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
Kirk Webb's avatar
Kirk Webb committed
326
327
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
328
            # If not ignoring, do the addition/update.
329
            else:
Kirk Webb's avatar
   
Kirk Webb committed
330
331
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
332
                    # Get the linktype here so we can report it in email.
Kirk Webb's avatar
Kirk Webb committed
333
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
334
                    if debug:
Kirk Webb's avatar
Kirk Webb committed
335
336
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
337
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
338
339
340
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
Kirk Webb's avatar
Kirk Webb committed
341
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
342
                              (nodeent['PLABID'],
Kirk Webb's avatar
Kirk Webb committed
343
344
345
346
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
Kirk Webb's avatar
   
Kirk Webb committed
347
                    addstr += nodestr
Kirk Webb's avatar
Kirk Webb committed
348
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
349
350
                pass
            pass
351

Kirk Webb's avatar
   
Kirk Webb committed
352
353
354
355
356
357
358
359
360
361
362
363
        # Process node attribute updates.
        updstr = ""
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
            for updent in toupdate:
                self.__updateNode(updent)
                # Add a line for the add/update message.
                nodestr = updent[0] + "\n"
                for attr,val in updent[1].items():
                    nodestr += "\t%s:\t%s => %s\n" % (attr,val[0],val[1])
                    pass
                updstr += nodestr + "\n"
Kirk Webb's avatar
Kirk Webb committed
364
365
                pass
            pass
Kirk Webb's avatar
   
Kirk Webb committed
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382

        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
Kirk Webb's avatar
Kirk Webb committed
383
        return
384

Kirk Webb's avatar
   
Kirk Webb committed
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
            if len(same):
                return (nid, diff)
            pass
        return ()

415
416
417
418
419
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
Kirk Webb's avatar
   
Kirk Webb committed
420
421
422
        res = DBQueryFatal("select node_id,plab_id,hostname,IP,mac"
                           " from plab_mapping")
        
423
        ret = {}
Kirk Webb's avatar
   
Kirk Webb committed
424
425
426
427
428
        for nodeid, plabid, hostname, ip, mac in res:
            ret[nodeid] = {'PLABID'  : plabid,
                           'HNAME'   : hostname,
                           'IP'      : ip,
                           'MAC'     : mac}
Kirk Webb's avatar
Kirk Webb committed
429
            pass
Kirk Webb's avatar
   
Kirk Webb committed
430
431
432
433
434
435
436
437
438
439
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
440
        return ret
Kirk Webb's avatar
   
Kirk Webb committed
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        uniqattrs = ['PLABID', 'HNAME', 'IP', 'MAC']
        attrs = {}
        dups = {}
        
        for ent in nodelist:
            for attr in uniqattrs:
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
Kirk Webb's avatar
   
Kirk Webb committed
466
        
Kirk Webb's avatar
Kirk Webb committed
467
    def __findLinkType(self, nodeent):
468
469
470
471
472
473
474
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

Kirk Webb's avatar
Kirk Webb committed
475
        This can't detect DSL links..
476
        """
477
        # Is host international (or flux/emulab local)?
478
        from socket import gethostbyaddr, getfqdn, herror
Kirk Webb's avatar
Kirk Webb committed
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
501
        # Is host on I2?
Kirk Webb's avatar
Kirk Webb committed
502
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
503
504
505
506
507
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
508
509
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
510

511
512
513
514
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
                break
        else:
Kirk Webb's avatar
Kirk Webb committed
515
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
516

517
        # Must be plain 'ole Internet
Kirk Webb's avatar
Kirk Webb committed
518
519
        nodeent['LINKTYPE'] = "pcplabinet"
        return
520

Kirk Webb's avatar
Kirk Webb committed
521
    def __addNode(self, nodeent, update = False):
522
523
524
525
526
527
528
        """
        getFree helper function.  Adds a new Plab pnode and associated
        vnodes to the DB.  linktype should be one of (inet2, inet, intl,
        dsl).

        XXX This duplicates a lot of the functionality of newwanode.
        Note that, very unlike newwanode, the node is initially up,
529
530
531
        since it had to be up to be added in the first place.  This also
        adds some additional fields that newwanode doesn't, and takes
        advantage of the fact that the Plab nodes may be added in bulk.
532
        """
Kirk Webb's avatar
   
Kirk Webb committed
533
534
        # block out common termination signals while adding a node
        osigs = disable_sigs(TERMSIGS)
535
        defosid, controliface = self.__getNodetypeInfo()
Kirk Webb's avatar
   
Kirk Webb committed
536
537
538
        nidnum, priority = self.__nextFreeNodeid()
        nodeid = "%s%d" % (NODEPREFIX, nidnum)
        vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
Kirk Webb's avatar
Kirk Webb committed
539
        hostonly = nodeent['HNAME'].replace(".", "-")
540
        
Kirk Webb's avatar
Kirk Webb committed
541
542
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)
543

Kirk Webb's avatar
   
Kirk Webb committed
544
        try:
Kirk Webb's avatar
Kirk Webb committed
545
            DBQueryFatal("replace into nodes"
546
                         " (node_id, type, phys_nodeid, role, priority,"
Kirk Webb's avatar
   
Kirk Webb committed
547
                         "  op_mode, def_boot_osid,"
548
                         "  allocstate, allocstate_timestamp,"
Kirk Webb's avatar
   
Kirk Webb committed
549
                         "  eventstate, state_timestamp)"
550
                         " values (%s, %s, %s, %s, %s,"
Kirk Webb's avatar
   
Kirk Webb committed
551
552
553
554
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
555
                          'FREE_CLEAN',
Kirk Webb's avatar
   
Kirk Webb committed
556
                          'ISUP'))
557

558
559
560
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
Kirk Webb's avatar
Kirk Webb committed
561
                         (nodeid))
562

Kirk Webb's avatar
Kirk Webb committed
563
            DBQueryFatal("replace into widearea_nodeinfo"
564
565
                         " (node_id, contact_uid, hostname, site)"
                         " values (%s, %s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
566
                         (nodeid, 'bnc', nodeent['HNAME'], nodeent['SITE']))
567

Kirk Webb's avatar
Kirk Webb committed
568
            DBQueryFatal("replace into interfaces"
Kirk Webb's avatar
   
Kirk Webb committed
569
570
571
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
572
573
                         (nodeid, 0, 1, nodeent['IP'], 'fxp',
                          controliface, 'ctrl'))
Kirk Webb's avatar
   
Kirk Webb committed
574

Kirk Webb's avatar
Kirk Webb committed
575
            DBQueryFatal("replace into reserved"
Kirk Webb's avatar
   
Kirk Webb committed
576
577
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
578
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))
Kirk Webb's avatar
   
Kirk Webb committed
579

Kirk Webb's avatar
Kirk Webb committed
580
            DBQueryFatal("replace into node_auxtypes"
Kirk Webb's avatar
   
Kirk Webb committed
581
582
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
583
                         (nodeid, nodeent['LINKTYPE'], 1))
Kirk Webb's avatar
   
Kirk Webb committed
584
            
Kirk Webb's avatar
Kirk Webb committed
585
            DBQueryFatal("replace into node_auxtypes"
586
587
588
589
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
Kirk Webb's avatar
Kirk Webb committed
590
            DBQueryFatal("replace into node_status"
591
592
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
593
                         (nodeid, 'down'))
Kirk Webb's avatar
   
Kirk Webb committed
594

Kirk Webb's avatar
   
Kirk Webb committed
595
596
597
598
599
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
600
601

            # Create a single reserved plab vnode for the managment sliver.
Kirk Webb's avatar
   
Kirk Webb committed
602
603
604
605
606
607
608
609
            # XXX I left it as "20" cause of all the existing ones.
            # XXXX I set it to 1 due to the above comment (correct?)
            #      since we are re-creating anyway.            
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
610
611
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
Kirk Webb's avatar
Kirk Webb committed
612
                pass
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
Kirk Webb's avatar
Kirk Webb committed
636
            # Put the last vnode created into the special monitoring expt.
Kirk Webb's avatar
   
Kirk Webb committed
637
638
639
640
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
Kirk Webb's avatar
Kirk Webb committed
641
642
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
643
644
645
646
647
648
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
649
                     "\n Please clean up!\n" % tbmsg, TBOPS)
Kirk Webb's avatar
   
Kirk Webb committed
650
651
            enable_sigs(osigs)
            raise
652

Kirk Webb's avatar
   
Kirk Webb committed
653
654
        # last but not least, unblock signals
        enable_sigs(osigs)
Kirk Webb's avatar
Kirk Webb committed
655
        return
656

Kirk Webb's avatar
   
Kirk Webb committed
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
    def __updateNode(self, updent):
        """
        Updates changed node attributes in the mapping table.
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}
        nodeid, chattrs = updent
        
        if len(chattrs) > 2:
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
                     "%s has been moved to hwdown." % (chattrs, nodeid)
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
            SENDMAIL(TBOPS,
                     "More than 2 plab node attrs have changed on %s" % nodeid,
                     errmsg,
                     TBOPS)
            raise RuntimeError, errmsg # XXX: maybe don't raise an exception.
        
        updstr = ",".join(map(lambda x: "%s='%s'" % (attrmap[x[0]],x[1][1]),
                              chattrs.items()))
        DBQueryFatal("update plab_mapping set " + updstr + " where node_id=%s",
                     (nodeid))
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

691
692
    def __getNodetypeInfo(self):
        """
693
694
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
695
        it doesn't change.
696
697
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
698
            if debug:
699
                print "Getting node type info"
Kirk Webb's avatar
Kirk Webb committed
700
                pass
701
            res = DBQueryFatal("select osid, control_iface"
702
                               " from node_types"
703
704
705
                               " where type = 'pcplabphys'")
            assert (len(res) == 1), "Failed to get node type info"
            (self.__getNodetypeInfoCache, ) = res
Kirk Webb's avatar
Kirk Webb committed
706
707
            pass
        
708
709
710
711
712
713
714
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
715
        if debug:
716
717
718
719
720
721
722
723
724
725
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
Kirk Webb's avatar
Kirk Webb committed
726
            pass
727
728
        finally:
            DBQueryFatal("unlock tables")
Kirk Webb's avatar
Kirk Webb committed
729
730
            pass
        
731
732
733
        return nodeid, priority

    def renew(self):
734
        """
Kirk Webb's avatar
   
Kirk Webb committed
735
736
737
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
738
        """
Kirk Webb's avatar
   
Kirk Webb committed
739

740
        now = int(time.mktime(time.gmtime()))
Kirk Webb's avatar
   
Kirk Webb committed
741
        res = DBQueryFatal("select pid, eid from plab_slices");
Kirk Webb's avatar
Kirk Webb committed
742
        
743
        loadedSlices = {}
Kirk Webb's avatar
Kirk Webb committed
744
        global failedrenew # XXX
745
746
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
747
748
        ret = 0

Kirk Webb's avatar
   
Kirk Webb committed
749
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
750

Kirk Webb's avatar
   
Kirk Webb committed
751
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
752

753
754
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
755
                pass
756
757
758
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
759
                pass
760
            
Kirk Webb's avatar
   
Kirk Webb committed
761
762
763
764
765
766
            res = slice.renew()
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
767
768
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
769
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
770
                if (slice.leaseend - now) < PLABEXPIREWARN:
771
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
772
773
                    pass
                pass
774
775
776
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
Kirk Webb's avatar
   
Kirk Webb committed
777
                    pass
Kirk Webb's avatar
Kirk Webb committed
778
                    
779
780
        if newfail:
            failedrenew += newfail
Kirk Webb's avatar
Kirk Webb committed
781
            failstr = ""
782
            for n in newfail:
Kirk Webb's avatar
   
Kirk Webb committed
783
784
                failstr += "%s/%s (expires: %s UTC)\n" % \
                           (n[:2] + (time.ctime(n[2]),))
Kirk Webb's avatar
Kirk Webb committed
785
                pass
Kirk Webb's avatar
   
Kirk Webb committed
786
            
787
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
Kirk Webb's avatar
   
Kirk Webb committed
788
                     "Failed to renew the following leases:\n%s" %
789
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
790
            pass
Kirk Webb's avatar
Kirk Webb committed
791

792
793
794
        if failsoon:
            failstr = ""
            for n in failsoon:
Kirk Webb's avatar
   
Kirk Webb committed
795
796
797
798
799
800
                failstr += "%s/%s: (expires: %s UTC)\n" % \
                           (n[:2] + (time.ctime(n[2]),))
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
801
            pass
Kirk Webb's avatar
   
Kirk Webb committed
802
        
Kirk Webb's avatar
Kirk Webb committed
803
804
805
        return
    
    pass # end class Plab
806

807
808
809
810
811
812

#
# Slice abstraction
#

class Slice:
Kirk Webb's avatar
   
Kirk Webb committed
813
814

    def __init__(self, plab, pid, eid, slicename = None):
815
816
        self.plab = plab
        self.pid, self.eid = pid, eid
817
        self.slicemeta = None
Kirk Webb's avatar
   
Kirk Webb committed
818
819
        self.slicename = slicename
        return
820
821
822
823
824
825
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
Kirk Webb's avatar
   
Kirk Webb committed
826
827
828
829
830
831
832

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
            self.slicename = PLAB_SVC_SLICENAME
            adminbit = 1
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
833
834
835
836
837
838
839
840
841
842
        if not self.slicename:
            res = DBQueryFatal("select idx from experiments "
                               "where pid=%s "
                               "and eid=%s",
                               (self.pid, self.eid))
            if not len(res):
                raise RuntimeError, "Didn't get any results while looking for idx"
            eindex = res[0][0]
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass
Kirk Webb's avatar
   
Kirk Webb committed
843

844
        print "Creating Plab slice %s." % self.slicename
Kirk Webb's avatar
   
Kirk Webb committed
845

846
        try:
Kirk Webb's avatar
Kirk Webb committed
847
848
849
850
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)

            DBQueryFatal("insert into plab_slices"
Kirk Webb's avatar
   
Kirk Webb committed
851
852
                         " (pid, eid, slicename, slicemeta, leaseend, admin)"
                         " values (%s, %s, %s, %s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
853
                         (self.pid, self.eid, self.slicename,
Kirk Webb's avatar
   
Kirk Webb committed
854
855
856
857
                          self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          adminbit))
858
859
            pass
        except:
Kirk Webb's avatar
Kirk Webb committed
860
            self.plab.agent.deleteSlice(self)
861
862
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
863
864
            raise

865
866
        return res

867
868
869
870
    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.
871
872
873

        XXX This should probably be made lazy, since not all operations
        really need it
874
        """
Kirk Webb's avatar
Kirk Webb committed
875
        if verbose:
876
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
Kirk Webb's avatar
   
Kirk Webb committed
877
878
        res = DBQueryFatal("select slicename, slicemeta, leaseend "
                           " from plab_slices "
879
880
881
882
883
884
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
Kirk Webb's avatar
   
Kirk Webb committed
885
        ((self.slicename, self.slicemeta, self.leaseend), ) = res
Kirk Webb's avatar
Kirk Webb committed
886
        self.leaseend = calendar.timegm(time.strptime(str(self.leaseend),
Kirk Webb's avatar
   
Kirk Webb committed
887
888
                                                      "%Y-%m-%d %H:%M:%S"))
        return
889

Kirk Webb's avatar
Kirk Webb committed
890
891
    def renew(self):
        """
Kirk Webb's avatar
   
Kirk Webb committed
892
893
        Renews slice lease.  We want this to be the maximum allowed by law..
        Store the expiration time in UTC.
Kirk Webb's avatar
Kirk Webb committed
894
895
896
897
        """
        print "Renewing lease for slice %s" % self.slicename

        try:
Kirk Webb's avatar
   
Kirk Webb committed
898
899
900
901
902
903
904
905
            ret = self.plab.agent.renewSlice(self)
            DBQueryFatal("update plab_slices "
                         " set slicemeta=%s, leaseend=%s "
                         " where slicename=%s",
                         (self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          self.slicename))
Kirk Webb's avatar
Kirk Webb committed
906
907
            pass
        except:
Kirk Webb's avatar
   
Kirk Webb committed
908
            print "slice.renew: Slice renewal failed:"
Kirk Webb's avatar
Kirk Webb committed
909
910
911
912
            traceback.print_exc()
            ret = 0
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
913
        return ret
Kirk Webb's avatar
Kirk Webb committed
914

915
916
917
918
919
920
921
    def destroy(self):
        """
        Frees all nodes in this slice and destroys the slice.  Note
        that this will really pound the DB if there are many nodes left
        in the slice, but those should be removed by Emulab before the
        slice is destroyed.
        """
922
        print "Destroying Plab slice %s." % self.slicename
923
924
925
926
927
928
929
930
        res = DBQueryFatal("select node_id from plab_slice_nodes"
                           " where slicename = %s",
                           (self.slicename))
        print "\tRemoving any remaining nodes in slice.."
        for (nodeid,) in res:
            node = self.loadNode(nodeid)
            node.free()
            del node  # Encourage the GC'er
931

Kirk Webb's avatar
   
Kirk Webb committed
932
        osigs = disable_sigs(TERMSIGS)
933

934
        try:
Kirk Webb's avatar
Kirk Webb committed
935
            self.plab.agent.deleteSlice(self)
936
937
938
939
940
941
            pass
        except:
            print "Failed to delete slice!"
            traceback.print_exc()
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
942
        try:
943
            print "\tRemoving slice DB entry."
Kirk Webb's avatar
   
Kirk Webb committed
944
945
946
947
948
949
            DBQueryFatal("delete from plab_slices where slicename = %s",
                         (self.slicename,))
        except:
            print "Error deleting slice from DB!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error deleting slice from DB",
950
                     "Slice deletion error:\n\n%s" % tbstr, TBOPS)
Kirk Webb's avatar
   
Kirk Webb committed
951
952
953
954
            enable_sigs(osigs)
            raise
        
        enable_sigs(osigs)
955

956
    def createNode(self, nodeid, force=False):
957
958
959
        """
        Node factory function
        """
960
961
962
963
964
965
966
967
        # XXX: KRW - The following is a hack to help me with testing.
        if not nodeid.startswith("plab"):
            IP = socket.gethostbyname(nodeid)
            qres = DBQueryFatal("select n.node_id from nodes as n left join "
                                "interfaces as i on n.node_id = i.node_id "
                                "where i.IP = %s", (IP,))
            assert (len(qres) > 0), "Node does not exist in DB: %s" % nodeid
            nodeid = qres[0][0] + "-20"
968
        node = Node(self, nodeid)
969
        node._create(force)
970
971
972
973
974
975
976
977
978
979
        return node

    def loadNode(self, nodeid):
        """
        Node factory function
        """
        node = Node(self, nodeid)
        node._load()
        return node

Kirk Webb's avatar
   
Kirk Webb committed
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
    def updateSliceMeta(self):
        """
        Grab current slice metadata from Planetlab and store in db
        """
        try:
            self.slicemeta = self.plab.agent.getSliceMeta(self)
            DBQueryFatal("update plab_slices set "
                         "slicemeta=%s where slicename=%s",
                         (self.slicemeta, self.slicename))
            pass
        except:
            print "Error updating slice metadata!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error updating slice metadata",
                     "Slice metadata update error:\n\n%s" % tbstr, TBOPS)
            raise

    def getSliceNodes(self):
        """
        Return a tuple containing the nodes that belong to this slice
        """
For faster browsing, not all history is shown. View entire blame