Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement #18 Program option #68

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions conftest/README
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
To test
- copy crashin fakemail to /R/bin
- copy crashmail.conf testcrash.conf testcrash2.conf to /etc/supervisor/conf.d

monitor log files in /L/
- You'll get an 'Ignoring hostname: testcrash' message every 10s in /L/crashmail.log (testcrash)

- You'll get 2 'unexpected exit, mailing' message every 30s in /L/crashmail.log (notifycrash:testcrash2, notifycrash:testcrash3)
and the message in /L/fakemail.log
7 changes: 7 additions & 0 deletions conftest/crashin
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash
# test program: non 0 exit code after timeout
timeout="$1"
echo $(date) "Crashing in $timeout seconds"
sleep "$timeout"
echo $(date) "CRASHING NOW"
exit 123
10 changes: 10 additions & 0 deletions conftest/crashmail.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# config to use the fakemail mail sender
# and monitor only testcrash2 (crashes every 30s)
[eventlistener:crashmail]
command =
/usr/local/bin/crashmail
-p notifycrash:*
-o hostname -m [email protected]
-s '/R/bin/fakemail -t -i -f [email protected]'
events=PROCESS_STATE_EXITED
stderr_logfile=/L/crashmail.log
6 changes: 6 additions & 0 deletions conftest/fakemail
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash
# don't send any mail: only write the args and stdin to the logfile
logfile=/L/fakemail.log
echo $(date) "$*" >> "$logfile"
cat >> "$logfile"
echo >> "$logfile"
9 changes: 9 additions & 0 deletions conftest/testcrash.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# test program: crashes every 10 seconds
[program:testcrash]
command=/R/bin/crashin 10

autostart=true
autorestart=true

stdout_logfile=/L/testcrash.log
redirect_stderr=true
21 changes: 21 additions & 0 deletions conftest/testcrash2.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[group:notifycrash]
programs=testcrash2,testcrash3

# test program: crashes every 30 seconds
[program:testcrash2]
command=/R/bin/crashin 30

autostart=true
autorestart=true

stdout_logfile=/L/testcrash2.log
redirect_stderr=true

[program:testcrash3]
command=/R/bin/crashin 33

autostart=true
autorestart=true

stdout_logfile=/L/testcrash3.log
redirect_stderr=true
34 changes: 28 additions & 6 deletions superlance/crashmail.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

-p -- specify a supervisor process_name. Send mail when this process
transitions to the EXITED state unexpectedly. If this process is
part of a group, it can be specified using the
part of a group, it must be specified using the
'process_name:group_name' syntax.

-a -- Send mail when any child of the supervisord transitions
Expand Down Expand Up @@ -66,6 +66,7 @@
"""

import os
import re
import sys

from supervisor import childutils
Expand Down Expand Up @@ -124,10 +125,25 @@ def runforever(self, test=False):
if self.optionalheader:
subject = self.optionalheader + ':' + subject

self.stderr.write('unexpected exit, mailing\n')
self.stderr.flush()

self.mail(self.email, subject, msg)
ident = pheaders['processname']
if pheaders['groupname'] != ident:
ident = pheaders['groupname'] + ":" + ident

if self.any or \
not self.programs or \
any(prog.match(ident) for prog in self.programs):

self.stderr.write('\nunexpected exit, mailing\n')
self.stderr.flush()

self.mail(self.email, subject, msg)

else:

self.stderr.write('\nignoring %s\n' % subject)
self.stderr.flush()


childutils.listener.ok(self.stdout)
if test:
Expand All @@ -140,7 +156,7 @@ def mail(self, email, subject, msg):
body += msg
with os.popen(self.sendmail, 'w') as m:
m.write(body)
self.stderr.write('Mailed:\n\n%s' % body)
self.stderr.write('Mailed:\n\n%s\n' % body)
self.mailed = body


Expand All @@ -167,13 +183,19 @@ def main(argv=sys.argv):
email = None
optionalheader = None

progGroupRE = re.compile(r"(\w+):\*")

for option, value in opts:

if option in ('-h', '--help'):
usage()

if option in ('-p', '--program'):
programs.append(value)
pg = progGroupRE.match(value)
if pg:
programs.append(re.compile(pg.group(1)+":.*"))
else:
programs.append(re.compile(re.escape(value)))

if option in ('-a', '--any'):
any = True
Expand Down
54 changes: 44 additions & 10 deletions superlance/tests/crashmail_test.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import unittest
from superlance.compat import StringIO
import re

class CrashMailTests(unittest.TestCase):
def _getTargetClass(self):
Expand Down Expand Up @@ -29,7 +30,7 @@ def _makeOnePopulated(self, programs, any, response=None):
return prog

def test_runforever_not_process_state_exited(self):
programs = {'foo':0, 'bar':0, 'baz_01':0 }
programs = []
any = None
prog = self._makeOnePopulated(programs, any)
prog.stdin.write('eventname:PROCESS_STATE len:0\n')
Expand All @@ -38,7 +39,7 @@ def test_runforever_not_process_state_exited(self):
self.assertEqual(prog.stderr.getvalue(), 'non-exited event\n')

def test_runforever_expected_exit(self):
programs = ['foo']
programs = [re.compile('foo')]
any = None
prog = self._makeOnePopulated(programs, any)
payload=('expected:1 processname:foo groupname:bar '
Expand All @@ -51,7 +52,7 @@ def test_runforever_expected_exit(self):
self.assertEqual(prog.stderr.getvalue(), 'expected exit\n')

def test_runforever_unexpected_exit(self):
programs = ['foo']
programs = [re.compile('bar:foo')]
any = None
prog = self._makeOnePopulated(programs, any)
payload=('expected:0 processname:foo groupname:bar '
Expand All @@ -63,20 +64,53 @@ def test_runforever_unexpected_exit(self):
prog.runforever(test=True)
output = prog.stderr.getvalue()
lines = output.split('\n')
self.assertEqual(lines[0], 'unexpected exit, mailing')
self.assertEqual(lines[1], 'Mailed:')
self.assertEqual(lines[2], '')
self.assertEqual(lines[3], 'To: [email protected]')
self.assertTrue('Subject: [foo]: foo crashed at' in lines[4])
self.assertEqual(lines[5], '')

self.assertEqual(lines[1], 'unexpected exit, mailing')
self.assertEqual(lines[2], 'Mailed:')
self.assertEqual(lines[3], '')
self.assertEqual(lines[4], 'To: [email protected]')
self.assertTrue('Subject: [foo]: foo crashed at' in lines[5])
self.assertEqual(lines[6], '')
self.assertTrue(
'Process foo in group bar exited unexpectedly' in lines[6])
'Process foo in group bar exited unexpectedly' in lines[7])
import os
f = open(os.path.join(self.tempdir, 'email.log'), 'r')
mail = f.read()
f.close()
self.assertTrue(
'Process foo in group bar exited unexpectedly' in mail)

def test_runforever_unexpected_exit_group(self):
programs = [re.compile('bar:*')]
any = None
prog = self._makeOnePopulated(programs, any)
payload=('expected:0 processname:foo groupname:bar '
'from_state:RUNNING pid:1')
prog.stdin.write(
'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload))
prog.stdin.write(payload)
prog.stdin.seek(0)
prog.runforever(test=True)
output = prog.stderr.getvalue()
lines = output.split('\n')

self.assertEqual(lines[1], 'unexpected exit, mailing')

def test_runforever_unexpected_exit_ignored(self):
programs = [re.compile('notfoo')]
any = None
prog = self._makeOnePopulated(programs, any)
payload=('expected:0 processname:foo groupname:bar '
'from_state:RUNNING pid:1')
prog.stdin.write(
'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload))
prog.stdin.write(payload)
prog.stdin.seek(0)
prog.runforever(test=True)
output = prog.stderr.getvalue()
lines = output.split('\n')

self.assertTrue('ignoring [foo]: foo crashed ' in lines[1])

if __name__ == '__main__':
unittest.main()