Skip to content

Commit

Permalink
Merge pull request #5 from nices96/master
Browse files Browse the repository at this point in the history 
Add Elaspsed Time, GC Time and Thread Count thresholds for create alerts.
  • Loading branch information
@nices96
nices96 authored Nov 11, 2016
2 parents babef81 + 7f74faa commit 5fcbbd7
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 3 deletions.
10 changes: 10 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,19 @@
- 신규 Agent 연결
- Agent의 연결 해제
- Agent의 재접속
- 응답시간의 임계치 초과
- GC Time의 임계치 초과
- Thread 갯수의 임계치 초과

### Properties (스카우터 서버 설치 경로 하위의 conf/scouter.conf)
* **_ext\_plugin\_telegram\_send\_alert_** : Telegram 메시지 발송 여부 (true / false) - 기본 값은 false
* **_ext\_plugin\_telegram\_debug_** : 로깅 여부 - 기본 값은 false
* **_ext\_plugin\_telegram\_level_** : 수신 레벨(0 : INFO, 1 : WARN, 2 : ERROR, 3 : FATAL) - 기본 값은 0
* **_ext\_plugin\_telegram\_bot\_token_** : Telegram Bot Token
* **_ext\_plugin\_telegram\_chat\_id_** : chat_id(Integer) 또는 채널 이름(String)
* **_ext\_plugin\_elapsed\_time_threshold_** : 응답시간의 임계치 (ms) - 기본 값은 0으로, 0일때 응답시간의 임계치 초과 여부를 확인하지 않는다.
* **_ext\_plugin\_gc\_time_threshold_** : GC Time의 임계치 (ms) - 기본 값은 0으로, 0일때 GC Time의 임계치 초과 여부를 확인하지 않는다.
* **_ext\_plugin\_thread\_count_threshold_** : Thread Count의 임계치 - 기본 값은 0으로, 0일때 Thread Count의 임계치 초과 여부를 확인하지 않는다.

* Example
```
Expand All @@ -25,6 +31,10 @@ ext_plugin_telegram_debug=true
ext_plugin_telegram_level=0
ext_plugin_telegram_bot_token=185780011:AAGVaPyWCoZ8y1mHZEK1jFmbLwpcjlsJoJY
ext_plugin_telegram_chat_id=@ScouterDemoChannel
ext_plugin_elapsed_time_threshold=5000
ext_plugin_gc_time_threshold=5000
ext_plugin_thread_count_threshold=300
```

### Dependencies
Expand Down
145 changes: 142 additions & 3 deletions src/scouter/plugin/server/alert/telegram/TelegramPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@
*/
package scouter.plugin.server.alert.telegram;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
Expand All @@ -28,13 +35,25 @@
import com.google.gson.Gson;

import scouter.lang.AlertLevel;
import scouter.lang.TextTypes;
import scouter.lang.TimeTypeEnum;
import scouter.lang.counters.CounterConstants;
import scouter.lang.pack.AlertPack;
import scouter.lang.pack.MapPack;
import scouter.lang.pack.ObjectPack;
import scouter.lang.pack.PerfCounterPack;
import scouter.lang.pack.XLogPack;
import scouter.lang.plugin.PluginConstants;
import scouter.lang.plugin.annotation.ServerPlugin;
import scouter.net.RequestCmd;
import scouter.server.Configure;
import scouter.server.CounterManager;
import scouter.server.Logger;
import scouter.server.core.AgentManager;
import scouter.server.db.TextRD;
import scouter.server.netio.AgentCall;
import scouter.util.DateUtil;
import scouter.util.HashUtil;

/**
* Scouter server plugin to send alert via telegram
Expand All @@ -45,6 +64,48 @@ public class TelegramPlugin {

// Get singleton Configure instance from server
final Configure conf = Configure.getInstance();

private static AtomicInteger ai = new AtomicInteger(0);
private static List<Integer> javaeeObjHashList = new ArrayList<Integer>();

public TelegramPlugin() {
if (ai.incrementAndGet() == 1) {
ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);

// thread count check
executor.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
for (int objHash : javaeeObjHashList) {
if (AgentManager.isActive(objHash)) {
ObjectPack objectPack = AgentManager.getAgent(objHash);
MapPack mapPack = new MapPack();
mapPack.put("objHash", objHash);

mapPack = AgentCall.call(objectPack, RequestCmd.OBJECT_THREAD_LIST, mapPack);

int threadCountThreshold = conf.getInt("ext_plugin_thread_count_threshold", 0);
int threadCount = mapPack.getList("name").size();

if (threadCountThreshold != 0 && threadCount > threadCountThreshold) {
AlertPack ap = new AlertPack();

ap.level = AlertLevel.WARN;
ap.objHash = objHash;
ap.title = "Thread count exceed a threahold.";
ap.message = objectPack.objName + "'s Thread count(" + threadCount + ") exceed a threshold.";
ap.time = System.currentTimeMillis();
ap.objType = objectPack.objType;

alert(ap);
}
}
}
}
},
0, 5, TimeUnit.SECONDS);
}
}

@ServerPlugin(PluginConstants.PLUGIN_SERVER_ALERT)
public void alert(final AlertPack pack) {
Expand Down Expand Up @@ -126,7 +187,7 @@ public void run() {
}
}

@ServerPlugin(PluginConstants.PLUGIN_SERVER_OBJECT)
@ServerPlugin(PluginConstants.PLUGIN_SERVER_OBJECT)
public void object(ObjectPack pack) {
if (pack.version != null && pack.version.length() > 0) {
AlertPack ap = null;
Expand All @@ -140,7 +201,12 @@ public void object(ObjectPack pack) {
ap.title = "An object has been activated.";
ap.message = pack.objName + " is connected.";
ap.time = System.currentTimeMillis();
ap.objType = "scouter";

if (AgentManager.getAgent(pack.objHash) != null) {
ap.objType = AgentManager.getAgent(pack.objHash).objType;
} else {
ap.objType = "scouter";
}

alert(ap);
} else if (op.alive == false) {
Expand All @@ -151,13 +217,86 @@ public void object(ObjectPack pack) {
ap.title = "An object has been activated.";
ap.message = pack.objName + " is reconnected.";
ap.time = System.currentTimeMillis();
ap.objType = "scouter";
ap.objType = AgentManager.getAgent(pack.objHash).objType;

alert(ap);
}
// inactive state can be handled in alert() method.
}
}

@ServerPlugin(PluginConstants.PLUGIN_SERVER_XLOG)
public void xlog(XLogPack pack) {
try {
int elapsedThreshold = conf.getInt("ext_plugin_elapsed_time_threshold", 0);

if (elapsedThreshold != 0 && pack.elapsed > elapsedThreshold) {
String serviceName = TextRD.getString(DateUtil.yyyymmdd(pack.endTime), TextTypes.SERVICE, pack.service);

AlertPack ap = new AlertPack();

ap.level = AlertLevel.WARN;
ap.objHash = pack.objHash;
ap.title = "Elapsed time exceed a threahold.";
ap.message = "[" + AgentManager.getAgentName(pack.objHash) + "] "
+ pack.service + "(" + serviceName + ") "
+ "elapsed time(" + pack.elapsed + " ms) exceed a threshold.";
ap.time = System.currentTimeMillis();
ap.objType = AgentManager.getAgent(pack.objHash).objType;

alert(ap);
}

} catch (Exception e) {
Logger.printStackTrace(e);
}
}

@ServerPlugin(PluginConstants.PLUGIN_SERVER_COUNTER)
public void counter(PerfCounterPack pack) {
String objName = pack.objName;
int objHash = HashUtil.hash(objName);
String objType = null;
String objFamily = null;

if (AgentManager.getAgent(objHash) != null) {
objType = AgentManager.getAgent(objHash).objType;
}

if (objType != null) {
objFamily = CounterManager.getInstance().getCounterEngine().getObjectType(objType).getFamily().getName();
}

try {
// in case of objFamily is javaee
if (CounterConstants.FAMILY_JAVAEE.equals(objFamily)) {
// save javaee type's objHash
if (!javaeeObjHashList.contains(objHash)) {
javaeeObjHashList.add(objHash);
}

if (pack.timetype == TimeTypeEnum.REALTIME) {
long gcTimeThreshold = conf.getLong("ext_plugin_gc_time_threshold", 0);
long gcTime = pack.data.getLong(CounterConstants.JAVA_GC_TIME);

if (gcTimeThreshold != 0 && gcTime > gcTimeThreshold) {
AlertPack ap = new AlertPack();

ap.level = AlertLevel.WARN;
ap.objHash = objHash;
ap.title = "GC time exceed a threahold.";
ap.message = objName + "'s GC time(" + gcTime + " ms) exceed a threshold.";
ap.time = System.currentTimeMillis();
ap.objType = objType;

alert(ap);
}
}
}
} catch (Exception e) {
Logger.printStackTrace(e);
}
}

private void println(Object o) {
if (conf.getBoolean("ext_plugin_telegram_debug", false)) {
Expand Down

0 comments on commit 5fcbbd7

Please sign in to comment.