Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add "since" field to dynamodb data store #22

Merged
merged 1 commit into from
May 28, 2015
Merged

add "since" field to dynamodb data store #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ To do
Changes
*******

- Add 'since' field to DynamoDB data store
(https://github.com/zc/cimaa/issues/18).

0.5.3 (2015-03-10)
==================

Expand Down
25 changes: 17 additions & 8 deletions src/zc/cimaa/dynamodb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
'updated',
data_type=boto.dynamodb2.types.NUMBER),
],
includes=['name', 'updated'],
includes=['name', 'since', 'updated'],
)
],
),
Expand All @@ -39,6 +39,7 @@
class DB:

def __init__(self, config, tables=tuple(schemas)):
# {agent: {name: fault_data}}
self.last_faults = {}
conn, prefix = connect(config)
for name in schemas:
Expand All @@ -56,7 +57,7 @@ def faults():
return [_fault_data(item)
for item in self.faults.query_2(agent__eq=agent)]

self.last_faults[agent] = set(fault['name'] for fault in faults)
self.last_faults[agent] = {fault['name']: fault for fault in faults}
return faults

def set_faults(self, agent, faults):
Expand All @@ -69,23 +70,30 @@ def set_faults(self, agent, faults):
def write_faults():
self._set_faults(agent, faults, old_faults)

self.last_faults[agent] = set(fault['name'] for fault in faults)
self.last_faults[agent] = {fault['name']: fault for fault in faults}

def _set_faults(self, agent, faults, old_faults):
now = int(time.time())
with self.faults.batch_write() as batch:
#print batch.__class__
# Heartbeat
batch.put_item(dict(
agent='_',
name=agent,
updated=int(time.time()),
updated=now,
))

for fault in faults:
data = fault.copy()
data['agent'] = agent
name = fault['name']
if name in old_faults:
if 'since' not in old_faults[name]:
old_faults[name]['since'] = now
data['since'] = old_faults[name]['since']
del old_faults[name]
else:
data['since'] = now
batch.put_item(data, overwrite=True)
old_faults.discard(data['name'])
for name in old_faults:
batch.delete_item(agent=agent, name=name)

Expand Down Expand Up @@ -171,7 +179,7 @@ def decorator(function):
return decorator


def _convert_tstamp(data, name):
def _convert_timestamp(data, name):
if name in data:
try:
data[name] = int(data[name])
Expand All @@ -184,7 +192,8 @@ def _fault_data(item):
# dynamodb doesn't populate keys with empty strings
if u'message' not in data:
data[u'message'] = u''
_convert_tstamp(data, u"updated")
_convert_timestamp(data, u"since")
_convert_timestamp(data, u"updated")
if u'severity' in data:
# Ints, not Decimals:
data[u'severity'] = int(data[u'severity'])
Expand Down
27 changes: 18 additions & 9 deletions src/zc/cimaa/dynamodb.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,20 +93,23 @@ And perform some operations:
{u'agent': u'agent',
u'message': u'',
u'name': u'blank',
u'severity': 50},
u'severity': 50,
u'since': T},
{u'agent': u'agent',
u'message': u'f2 is bad',
u'name': u'f2',
u'severity': 40},
u'severity': 40,
u'since': T},
{u'agent': u'agent',
u'message': u'f3 is bad',
u'name': u'f3',
u'severity': 50,
u'since': T,
u'triggered': u'y'}],...

Notice that the faults data includes data for an agent '_'. This is
Notice that the faults data includes data for an agent '_'. This is
heartbeat data that tells us when the agent last ran. We can use this
to find old agents:
to find agents that no longer report:

>>> db.old_agents(900) # agents that haven't run in 15 minutes
[]
Expand All @@ -118,15 +121,18 @@ to find old agents:
[{u'agent': u'agent',
u'message': u'',
u'name': u'blank',
u'severity': 50},
u'severity': 50,
u'since': T},
{u'agent': u'agent',
u'message': u'f2 is bad',
u'name': u'f2',
u'severity': 40},
u'severity': 40,
u'since': T},
{u'agent': u'agent',
u'message': u'f3 is bad',
u'name': u'f3',
u'severity': 50,
u'since': T,
u'triggered': u'y'}]
>>> db.set_faults('agent', [])

Expand All @@ -149,7 +155,8 @@ has to ensure that it gets restored to avoid KeyErrors::
[{u'agent': u'agent',
u'message': u'',
u'name': u'blank',
u'severity': 50}]
u'severity': 50,
u'since': T}]

The remove_agent method is used to remove an agent from the database
completely; both faults and the heartbeat record are removed, while
Expand All @@ -169,11 +176,13 @@ records for other agents are not touched:
{u'agent': u'tnega',
u'message': u'f1 is bad',
u'name': u'f1',
u'severity': 40},
u'severity': 40,
u'since': T},
{u'agent': u'tnega',
u'message': u'f2 is bad',
u'name': u'f2',
u'severity': 40}],
u'severity': 40,
u'since': T}],
'squelches': []}


Expand Down