Status eines remote Servers mittels ssh auslesen und kritische Zustände alarmieren

Mit Anregungen von SshSysMon von Christopher LaPointe und der Frage von sisamiwe im KNX-User-Forum

Python Lib paramiko für die SSH-Kommunikation installieren:

sudo apt-get install build-essential libssl-dev libffi-dev python-dev
pip install paramiko

item.conf:

    [server]

    [[smarthome_local]]
      ssh_server = 'smarthome localhost'
      ssh_host = 'localhost'
      ssh_user = 'smarthome'
      ssh_passwd = 'smarthome'
      [[[loadavg]]]
        type = list
        ssh_cmd_pull = 'cat /proc/loadavg'
        [[[[avg1min]]]]
          type = num
          eval = sh...()[0]
          eval_trigger = ..
        [[[[avg5min]]]]
          type = num
          eval = sh...()[1]
          eval_trigger = ..
        [[[[avg15min]]]]
            type = num
            eval = sh...()[2]
            eval_trigger = ..
        [[[[alarm1]]]]
          name = 'Load Avg 5min > 1.0!'
          type = bool
          eval = sh...avg5min() > 1.0
          eval_trigger = ..avg5min
        [[[[alarm2]]]]
          name = 'Load Avg 5min > 2.0!'
          type = bool
          eval = sh...avg5min() > 2.0
          eval_trigger = ..avg5min

      [[[uptime]]]
        type = list
        ssh_cmd_pull = 'cat /proc/uptime'
        [[[[s]]]]
          type = num
          eval = sh...()[0]
          eval_trigger = ..
        [[[[days]]]]
          type = num
          eval =  int(sh...s() / 60 / 60 / 24)
          eval_trigger = ..s
        [[[[hours]]]]
          type = num
          eval =  int((sh...s() - sh...days() * 24 * 60 * 60) / 60 / 60)
          eval_trigger = ..s
        [[[[mins]]]]
          type = num
          eval =  int((int(sh...s()) - int(sh...days()) * 24 * 60 * 60 - int(sh...hours()) * 60 * 60 ) / 60)
          eval_trigger = ..s

        [[[[pretty]]]]
          name = days_hours_min_secs
          type = str
          eval = "{0} days - {1} hours - {2} minutes".format(sh...days(), sh...hours(), sh...mins())
          eval_trigger = ..s
        [[[[alarm2]]]]
          name = 'Server reboot last 5min!!!'
          type = bool
          eval = sh...s() < 300
          eval_trigger = ..s

      [[[disk]]]
              name = 'root'
              type = list
              ssh_cmd_pull = 'df / -BM | grep / | tr -s " "'
              [[[[used]]]]
                  type = num
                  eval = sh...()[1][:-1]
                  eval_trigger = ..
              [[[[avail]]]]
                  type = num
                  eval = sh...()[2][:-1]
                  eval_trigger = ..
              [[[[total]]]]
                  type = num
                  eval = sum
                  eval_trigger = ..used | ..avail

      [[[mem]]]
              name = 'Mem'
              type = list
              ssh_cmd_pull = 'free -m | grep Mem |tr -s " "'
              [[[[total]]]]
                  type = num
                  eval = sh...()[1]
                  eval_trigger = ..
              [[[[used]]]]
                  type = num
                  eval = sh...()[2]
                  eval_trigger = ..
              [[[[free]]]]
                  type = num
                  eval = sh...()[3]
                  eval_trigger = ..
              [[[[shared]]]]
                  type = num
                  eval = sh...()[4]
                  eval_trigger = ..
              [[[[buffers]]]]
                  type = num
                  eval = sh...()[5]
                  eval_trigger = ..
              [[[[cached]]]]
                  type = num
                  eval = sh...()[6]
                  eval_trigger = ..

      [[[swap]]]
              name = 'Swap'
              type = list
              ssh_cmd_pull = 'free -m | grep Swap |tr -s " "'
              [[[[total]]]]
                  type = num
                  eval = sh...()[1]
                  eval_trigger = ..
              [[[[used]]]]
                  type = num
                  eval = sh...()[2]
                  eval_trigger = ..
              [[[[free]]]]
                  type = num
                  eval = sh...()[3]
                  eval_trigger = ..

etc/logic.conf

[sshserver]
    filename = sshserver.py
    cycle = 300
    crontab = init

logics/sshserver.conf

    #!/usr/bin/env python
    #
    #
    logger.info("Logik SSH_Server : by :" + trigger['by'] )
    #logger.info("Logik SSH_Server : source :" + trigger['source'] )
    #logger.info("Logik SSH_Server : dest :" + trigger['dest'] )
    #logger.info("Logik SSH_Server : value :" + trigger['value'] )

    DEFAULT_KEY_PATH = "~/.ssh/id_rsa"

    import paramiko

    ssh_hosts = {}
    for item in sh.find_items('ssh_server'):    # findet alle Items die ein Attribut 'ssh_host' besitzen
    	host = item.conf['ssh_host']
    	user = item.conf['ssh_user']
    	passwd = item.conf['ssh_passwd']
    	#key = item.conf['ssh_key']
    	#key = ''
    	#if not passwd or key:
    	#		rsakey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(key or DEFAULT_KEY_PATH))
    	#else:
    	#		rsakey = None

    	#ssh_hosts[host] = {'user' : user, 'passwd': passwd, 'key': rsakey}
    	ssh_hosts[host] = {'user' : user, 'passwd': passwd}


    ssh = {}
    for host in ssh_hosts.keys():  # connect to all hosts
    	ssh[host] = paramiko.SSHClient()
    	ssh[host].set_missing_host_key_policy( paramiko.AutoAddPolicy())
    	ssh[host].connect(host, username=ssh_hosts[host]['user'], password=ssh_hosts[host]['passwd'],) #key=rsakey)

    # cat /proc/uptime
    # 1516600.94 2900685.66
    # cat /proc/loadavg
    # 0.10 0.16 0.17 1/167 8686

    if trigger['by'] == 'Scheduler':   # scheduler / cycle ??

    	for item in sh.find_items('ssh_cmd_pull'):    # findet alle Items die ein Attribut 'ssh_cmd' besitzen und führe cmd auf remote host aus
    		cmd = item.conf['ssh_cmd_pull']
    		if isinstance(cmd, list):
    			cmd = "'"+' | '.join(cmd)+"'"
    		host = item.return_parent().conf['ssh_host']
    		stdin, stdout, stderr = ssh[host].exec_command(cmd)
    		res = stdout.readlines()
    		logger.info("Logik SSH_Server : {0} : {1}".format(cmd, res) )
    		item(res[0].split(' '))
    elif trigger['by'] == 'fdhahsdf':  # watchitem ??
    	item = sh.return_item(trigger['source'])
    	cmd = item.conf['ssh_cmd_push']
    	host = item.return_parent().conf['ssh_host']
    	stdin, stdout, stderr = ssh[host].exec_command(cmd)
    	stdout.readlines()
    	item(0)

Dashboard im Backend:

      <!DOCTYPE html>
      {% extends "base.html" %}
      {% import "navbar.html" as nav with context %}
      {% block navbar %}
      	{{ nav }}
      {% endblock navbar %}

      {% block styles %}
      <!-- Bootstrap and font awesome -->
      <meta http-equiv="X-UA-Compatible" content="IE=edge">
      <meta name="viewport" content="width=device-width, initial-scale=1">
      <link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css">
      <link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.5.0/css/font-awesome.min.css">
      <style type="text/css">
      .inspector {
        height: 350px;
      }
      pre.cmdbox {
        max-height: 120px;
        overflow: auto;
        word-wrap: normal;
        white-space: pre;
      }
      </style>
      {% endblock styles %}
      {% block scripts %}
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.2/jquery.min.js"></script>
      <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>

      <!-- Chartjs -->
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/1.1.0/Chart.min.js"></script>

      <script type="text/javascript">
      Chart.defaults.global.animation =  'true';
      </script>
      {% endblock scripts %}



      {% block content %}
      <div class="table-responsive">
          <h1>{{ server.conf['ssh_server'] }} </h1>
      		<table class="table table-striped table-hover">
      			<thead>
      				<tr>
      					<th width="200px">{{ _('Status') }}</th>
      					<th width="200px">{{ _('Memory') }}</th>
      					<th width="200px">{{ _('Disk Space') }}</th>
                <th width="200px">{{ _('CPU Load') }}</th>
      				</tr>
      			</thead>
            <tbody>
              <tr>
                <td><!-- Uptime -->
                  <h3>Uptime:</h3>
                  <strong> {{ server.uptime.pretty() }}</strong>
                </td>

                <td rowspan="2"> <!-- MEMORY -->
                    <div class="col-xs-6">
                      <h3>Mem</h3>
                      <canvas id="memgraph-{{ server.id() }}" width="125" height="125"></canvas>
                      <table class="table table-condensed">
                        <tr><td>Free</td><td>{{ server.mem.free() }}</td></tr>
                        <tr><td>Used</td><td>{{ server.mem.used() }}</td></tr>
                        <tr><td>Total</td><td>{{ server.mem.total() }}</td></tr>
                        <tr><td>Cached</td><td>{{ server.mem.cached() }}</td></tr>
                      </table>
                    </div>
                    <div class="col-xs-6">
                      <h3>Swap</h3>
                      <canvas id="swapgraph-{{ server.id() }}" width="125" height="125"></canvas>
                      <table class="table table-condensed">
                        <tr><td>Free</td><td>{{ server.swap.free() }}</td></tr>
                        <tr><td>Used</td><td>{{ server.swap.used() }}</td></tr>
                        <tr><td>Total</td><td>{{ server.swap.total() }}</td></tr>
                      </table>
                    </div>
                    <script type="text/javascript">
                    (function(){
                      var data = [
                        {
                          value: ({{ server.mem.used() - server.mem.cached()  }})|0,
                          color: "#FF4136",
                          label: "Used (MB)"
                        },
                        {
                          value: ({{ server.mem.cached() }})|0,
                          color: "#0074D9",
                          label: "Cached (MB)"
                        },
                        {
                          value: {{ server.mem.free() }}|0,
                          color: "#DDDDDD",
                          label: "Free (MB)"
                        }
                      ];

                      var ctx = document.getElementById("memgraph-{{ server.id() }}").getContext("2d");
                      new Chart(ctx).Doughnut(data);
                    })();

                    (function(){
                      var data = [
                        {
                          value: ({{ server.swap.used() }})|0,
                          color: "#FF851B",
                          label: "Used (MB)"
                        },
                        {
                          value: {{ server.swap.free() }}|0,
                          color: "#DDDDDD",
                          label: "Free (MB)"
                        }
                      ];

                      var ctx = document.getElementById("swapgraph-{{server.id()}}").getContext("2d");
                      new Chart(ctx).Doughnut(data);
                    })();
                    </script>
                </td>

                <td rowspan="2"> <!-- Disk space -->
                  <h3>{{ server.disk._name }}</h3>
                    <canvas id="disk-{{server.id()}}-{{'disk1'}}" width="150" height="150"></canvas>
                    <script type="text/javascript">
                    (function(){
                      var data = [
                        {
                          value: {{ server.disk.used() / 1024 }}|0,
                          color: "#001f3f",
                          label: "Used (GB)"
                        },
                        {
                          value: {{ server.disk.avail() / 1024 }}|0,
                          color: "#DDDDDD",
                          label: "Free(GB)"
                        }
                      ];

                      var ctx = document.getElementById("disk-{{server.id()}}-{{'disk1'}}").getContext("2d");
                      new Chart(ctx).Pie(data);
                    })();
                    </script>
                    <table class="table table-striped table-condensed">
                      <tr>
                        <td>Used</td>
                        <td>{{ '{:6.2f} GB'.format(server.disk.used() / 1024) }}</td>
                      </tr>
                      <tr>
                        <td>Free</td>
                        <td>{{ '{:6.2f} GB'.format(server.disk.avail() / 1024) }}</td>
                      </tr>
                      <tr>
                        <td>Total</td>
                        <td>{{ '{:6.2f} GB'.format(server.disk.total() / 1024) }}</td>
                      </tr>
                    </table>
                </td>

                <td rowspan="2">
                  <!-- Load Average -->
                    <canvas id="loadavg-{{server.id()}}" width="300" height="250"></canvas>
                    <script type="text/javascript">
                    (function(){
                      var data = {
                        labels: ["1m", "5m", "15m"],
                        datasets: [
                          {
                            label: "Load",
                            fillColor: "rgba(151,187,205,0.5)",
                                  strokeColor: "rgba(151,187,205,0.8)",
                                  highlightFill: "rgba(151,187,205,0.75)",
                                  highlightStroke: "rgba(151,187,205,1)",
                            data: [{{ server.loadavg.avg1min() }}, {{ server.loadavg.avg5min() }}, {{ server.loadavg.avg15min()  }}]
                          }
                        ]
                      };

                      var ctx = document.getElementById("loadavg-{{server.id()}}").getContext("2d");
                      new Chart(ctx).Bar(data);
                    })();
                    </script>
                </td>

              </tr>
            </tr>
            <tr>
                <td> <!-- Alarms -->
                  <div>
                    <h3>Alarms</h3>
                  <table class="table table-condensed table-striped">
                    {% for alarm2 in sh.match_items(server.id()+'.*.alarm2') %}
                    <tr>
                      <td width="180px">{{ alarm2._name }}</td>
                      <td>
                        {% if alarm2() %}
                        <span class="glyphicon glyphicon-remove" style="color: red"></span>
                        {% else %}
                        <span class="glyphicon glyphicon-ok" style="color: green"></span>
                        {% endif %}
                      </td>
                    </tr>
                    {% endfor %}
                    {% for alarm1 in sh.match_items(server.id()+'.*.alarm1') %}
                    <tr>
                      <td width="180px">{{ alarm1._name }}</td>
                      <td>
                        {% if alarm1() %}
                        <span class="glyphicon glyphicon-remove" style="color: orange"></span>
                        {% else %}
                        <span class="glyphicon glyphicon-ok" style="color: green"></span>
                        {% endif %}
                      </td>
                    </tr>
                    {% endfor %}
                  </table>
                </div>
            </td>

            </tbody>
          </table>
        </div>
      {% endblock %}

Folgende Seite im BackendCore.py hinzufügen:

@cherrypy.expose
def dashboard_html(self, sshserver='smarthome localhost'):
    tmpl = self.env.get_template('dashboard.html')
    server = self._sh.match_items('*.'+sshserver)[0]
    return tmpl.render( sh = self._sh, server=server)

Template navbar.html:

....
			<!-- S Y S T E M  -->
		<li class="nav-item dropdown">
			<a class="nav-link dropdown-toggle" id="supportedContentDropdown"
				 data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">{{ _('System', 'menu') }}<span class="caret"></span></a>
				<ul class="dropdown-menu" aria-labelledby="supportedContentDropdown">
					<li><a class="dropdown-item" href="system.html">{{ _('Systeminfo', 'menu') }}</a></li>
					<li role="separator" class="divider"></li>
					<li><a class="dropdown-item" href="/dashboard.html?sshserver=wallmeier_info">{{ _('Server Dashboard wallmeier.info', 'menu') }}</a></li>
					<li><a class="dropdown-item" href="/dashboard.html?sshserver=smarthome_local">{{ _('Server Dashboard smarthome_local', 'menu') }}</a></li>
				</ul>
		</li>