Commit acbf5767 authored by David Johnson's avatar David Johnson

Reorg top level; add more resource info; add timestamp/runtime metadata.

Now the top-level keys are: 'META' (metadata about the collection run,
so that whoever pulls this file back to boss doesn't have to check its
ctime/mtime to know how stale the data is -- times in GMT); 'info',
which has keys like 'images', 'vms', 'networks', 'subnets', 'ports',
'routers', and a UUID->dict where the dict has a 'name' field (HRN),
'status' (if the resource has status; all do); and 'deleted' (True
or False).  Then the periods (which were previously top-level keys)
are now keys in the 'periods' top-level dict.
parent f77aac78
......@@ -147,7 +147,7 @@ def get_hypervisor_hostname(client,resource):
r_hostnames[hh] = hostname
LOG.debug("resource: " + pp.pformat(resource))
#LOG.debug("resource: " + pp.pformat(resource))
hh = resource.metadata['host']
if not hh in r_hostnames.keys():
LOG.error("hostname hash %s doesn't map to a known hypervisor hostname!" % (hh,))
......@@ -171,7 +171,10 @@ def fetchall(client):
tt = time.gmtime()
ct = time.mktime(tt)
cts = time.strftime('%Y-%m-%dT%H:%M:%S',tt)
datadict = {}
periods = {}
info = {}
#datadict = {}
# Ok, collect all the statistics, grouped by VM, for the period. We
......@@ -180,7 +183,7 @@ def fetchall(client):
#qm = QueryManager()
q = client.query_samples.query(limit=1)
for period in PERIODS:
datadict[period] = {}
periods[period] = {}
vm_dict = dict() #count=vm_0,list=[])
cpu_util_dict = dict()
......@@ -225,13 +228,15 @@ def fetchall(client):
hostname = get_hypervisor_hostname(client,resource)"%s for %s on %s = %f"
% (meter,rid,hostname,stat.avg))"%s for %s on %s = %f (resource=%s)"
% (meter,rid,hostname,stat.avg,pp.pformat(resource)))
if not hostname in vm_dict:
vm_dict[hostname] = {}
if not vmrid in vm_dict[hostname]:
vm_dict[hostname][vmrid] = resource.metadata['display_name']
vm_dict[hostname][vmrid] = dict(name=resource.metadata['display_name'],
if not hostname in mdict:
mdict[hostname] = dict(total=0.0,vms={})
......@@ -243,16 +248,24 @@ def fetchall(client):
mdict[hostname]['vms'][vmrid] += stat.avg
datadict[period][meter] = mdict
periods[period][meter] = mdict
datadict[period]['vm_info'] = vm_dict
info['vms'] = vm_dict
# Now also query the API delta meters:
meters = [ 'network.create','network.update','network.delete',
# NB: very important that the .delete meters come first, for
# each resource type. Why? Because we only put the resource
# details into the info dict one time (because we don't know
# how to merge details for a given resource if we see it again
# later and it differs) -- and sometimes we know if a resource
# is deleted based on if the delete method has been called for
# it (i.e. for network resources); for other resources like
# images, there's a deleted bit in the metadata we can just read.
meters = [ 'network.delete','network.create','network.update',
'image.upload','image.update' ]
rdicts = dict()
for meter in meters:
......@@ -278,7 +291,24 @@ def fetchall(client):"%s for %s on %s = %f (%s)"
% (meter,rid,hostname,stat.sum,pp.pformat(resource)))
if rplural and not rid in rdicts[rplural]:
rdicts[rplural][rid] = resource.metadata['name']
deleted = False
if meter.endswith('.delete') \
or ('deleted' in resource.metadata \
and resource.metadata['deleted'] \
in ['True','true',True]):
deleted = True
rdicts[rplural][rid] = dict(name=resource.metadata['name'],
status = None
if 'state' in resource.metadata:
status = resource.metadata['state']
elif 'status' in resource.metadata:
status = resource.metadata['status']
if not status is None:
rdicts[rplural][rid]['status'] = str(status).lower()
if rplural:
rname = rplural
......@@ -291,14 +321,44 @@ def fetchall(client):
mdict[hostname]['total'] += stat.sum
mdict[hostname][rname][rid] = stat.sum
datadict[period][meter] = mdict
periods[period][meter] = mdict
for (k,v) in rdicts.iteritems():
datadict[period][k] = v
for (res,infodict) in rdicts.iteritems():
# If we haven't seen this resource before, slap all
# the info we've found for all those resource ids
# into our info dict for this resource type. Else,
# carefully merge -- if we've already collected info
# for a specific resource, don't overwrite that.
# The theory for the Else case is that newer info
# is better, if the older info differs (which I can't
# think right now it would... should be the same for
# all periods).
if not res in info:
info[res] = infodict
for (resid,resinfodict) in infodict.iteritems():
if not resid in info[res]:
info[res][resid] = resinfodict
ett = time.gmtime()
ect = time.mktime(ett)
ects = time.strftime('%Y-%m-%dT%H:%M:%S',ett)
gmoffset = time.timezone
daylight = False
if time.daylight:
gmoffset = time.altzone
daylight = True
metadata = dict(start=cts,start_timestamp=ct,
return datadict
return dict(periods=periods,info=info,META=metadata)
def preload_resources(client):
global resources
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment