diff --git a/README.md b/README.md index 234e362..fe70b4c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ - 신규 Agent 연결 - Agent의 연결 해제 - Agent의 재접속 + - 응답시간의 임계치 초과 + - GC Time의 임계치 초과 + - Thread 갯수의 임계치 초과 ### Properties (스카우터 서버 설치 경로 하위의 conf/scouter.conf) * **_ext\_plugin\_telegram\_send\_alert_** : Telegram 메시지 발송 여부 (true / false) - 기본 값은 false @@ -16,6 +19,9 @@ * **_ext\_plugin\_telegram\_level_** : 수신 레벨(0 : INFO, 1 : WARN, 2 : ERROR, 3 : FATAL) - 기본 값은 0 * **_ext\_plugin\_telegram\_bot\_token_** : Telegram Bot Token * **_ext\_plugin\_telegram\_chat\_id_** : chat_id(Integer) 또는 채널 이름(String) +* **_ext\_plugin\_elapsed\_time_threshold_** : 응답시간의 임계치 (ms) - 기본 값은 0으로, 0일때 응답시간의 임계치 초과 여부를 확인하지 않는다. +* **_ext\_plugin\_gc\_time_threshold_** : GC Time의 임계치 (ms) - 기본 값은 0으로, 0일때 GC Time의 임계치 초과 여부를 확인하지 않는다. +* **_ext\_plugin\_thread\_count_threshold_** : Thread Count의 임계치 - 기본 값은 0으로, 0일때 Thread Count의 임계치 초과 여부를 확인하지 않는다. * Example ``` @@ -25,6 +31,10 @@ ext_plugin_telegram_debug=true ext_plugin_telegram_level=0 ext_plugin_telegram_bot_token=185780011:AAGVaPyWCoZ8y1mHZEK1jFmbLwpcjlsJoJY ext_plugin_telegram_chat_id=@ScouterDemoChannel + +ext_plugin_elapsed_time_threshold=5000 +ext_plugin_gc_time_threshold=5000 +ext_plugin_thread_count_threshold=300 ``` ### Dependencies diff --git a/src/scouter/plugin/server/alert/telegram/TelegramPlugin.java b/src/scouter/plugin/server/alert/telegram/TelegramPlugin.java index 96edd65..facd12b 100644 --- a/src/scouter/plugin/server/alert/telegram/TelegramPlugin.java +++ b/src/scouter/plugin/server/alert/telegram/TelegramPlugin.java @@ -17,6 +17,13 @@ */ package scouter.plugin.server.alert.telegram; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpPost; @@ -28,13 +35,25 @@ import com.google.gson.Gson; import scouter.lang.AlertLevel; +import scouter.lang.TextTypes; +import scouter.lang.TimeTypeEnum; +import scouter.lang.counters.CounterConstants; import scouter.lang.pack.AlertPack; +import scouter.lang.pack.MapPack; import scouter.lang.pack.ObjectPack; +import scouter.lang.pack.PerfCounterPack; +import scouter.lang.pack.XLogPack; import scouter.lang.plugin.PluginConstants; import scouter.lang.plugin.annotation.ServerPlugin; +import scouter.net.RequestCmd; import scouter.server.Configure; +import scouter.server.CounterManager; import scouter.server.Logger; import scouter.server.core.AgentManager; +import scouter.server.db.TextRD; +import scouter.server.netio.AgentCall; +import scouter.util.DateUtil; +import scouter.util.HashUtil; /** * Scouter server plugin to send alert via telegram @@ -45,6 +64,48 @@ public class TelegramPlugin { // Get singleton Configure instance from server final Configure conf = Configure.getInstance(); + + private static AtomicInteger ai = new AtomicInteger(0); + private static List javaeeObjHashList = new ArrayList(); + + public TelegramPlugin() { + if (ai.incrementAndGet() == 1) { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + // thread count check + executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + for (int objHash : javaeeObjHashList) { + if (AgentManager.isActive(objHash)) { + ObjectPack objectPack = AgentManager.getAgent(objHash); + MapPack mapPack = new MapPack(); + mapPack.put("objHash", objHash); + + mapPack = AgentCall.call(objectPack, RequestCmd.OBJECT_THREAD_LIST, mapPack); + + int threadCountThreshold = conf.getInt("ext_plugin_thread_count_threshold", 0); + int threadCount = mapPack.getList("name").size(); + + if (threadCountThreshold != 0 && threadCount > threadCountThreshold) { + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = objHash; + ap.title = "Thread count exceed a threahold."; + ap.message = objectPack.objName + "'s Thread count(" + threadCount + ") exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = objectPack.objType; + + alert(ap); + } + } + } + } + }, + 0, 5, TimeUnit.SECONDS); + } + } @ServerPlugin(PluginConstants.PLUGIN_SERVER_ALERT) public void alert(final AlertPack pack) { @@ -126,7 +187,7 @@ public void run() { } } - @ServerPlugin(PluginConstants.PLUGIN_SERVER_OBJECT) + @ServerPlugin(PluginConstants.PLUGIN_SERVER_OBJECT) public void object(ObjectPack pack) { if (pack.version != null && pack.version.length() > 0) { AlertPack ap = null; @@ -140,7 +201,12 @@ public void object(ObjectPack pack) { ap.title = "An object has been activated."; ap.message = pack.objName + " is connected."; ap.time = System.currentTimeMillis(); - ap.objType = "scouter"; + + if (AgentManager.getAgent(pack.objHash) != null) { + ap.objType = AgentManager.getAgent(pack.objHash).objType; + } else { + ap.objType = "scouter"; + } alert(ap); } else if (op.alive == false) { @@ -151,13 +217,86 @@ public void object(ObjectPack pack) { ap.title = "An object has been activated."; ap.message = pack.objName + " is reconnected."; ap.time = System.currentTimeMillis(); - ap.objType = "scouter"; + ap.objType = AgentManager.getAgent(pack.objHash).objType; alert(ap); } // inactive state can be handled in alert() method. } } + + @ServerPlugin(PluginConstants.PLUGIN_SERVER_XLOG) + public void xlog(XLogPack pack) { + try { + int elapsedThreshold = conf.getInt("ext_plugin_elapsed_time_threshold", 0); + + if (elapsedThreshold != 0 && pack.elapsed > elapsedThreshold) { + String serviceName = TextRD.getString(DateUtil.yyyymmdd(pack.endTime), TextTypes.SERVICE, pack.service); + + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = pack.objHash; + ap.title = "Elapsed time exceed a threahold."; + ap.message = "[" + AgentManager.getAgentName(pack.objHash) + "] " + + pack.service + "(" + serviceName + ") " + + "elapsed time(" + pack.elapsed + " ms) exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = AgentManager.getAgent(pack.objHash).objType; + + alert(ap); + } + + } catch (Exception e) { + Logger.printStackTrace(e); + } + } + + @ServerPlugin(PluginConstants.PLUGIN_SERVER_COUNTER) + public void counter(PerfCounterPack pack) { + String objName = pack.objName; + int objHash = HashUtil.hash(objName); + String objType = null; + String objFamily = null; + + if (AgentManager.getAgent(objHash) != null) { + objType = AgentManager.getAgent(objHash).objType; + } + + if (objType != null) { + objFamily = CounterManager.getInstance().getCounterEngine().getObjectType(objType).getFamily().getName(); + } + + try { + // in case of objFamily is javaee + if (CounterConstants.FAMILY_JAVAEE.equals(objFamily)) { + // save javaee type's objHash + if (!javaeeObjHashList.contains(objHash)) { + javaeeObjHashList.add(objHash); + } + + if (pack.timetype == TimeTypeEnum.REALTIME) { + long gcTimeThreshold = conf.getLong("ext_plugin_gc_time_threshold", 0); + long gcTime = pack.data.getLong(CounterConstants.JAVA_GC_TIME); + + if (gcTimeThreshold != 0 && gcTime > gcTimeThreshold) { + AlertPack ap = new AlertPack(); + + ap.level = AlertLevel.WARN; + ap.objHash = objHash; + ap.title = "GC time exceed a threahold."; + ap.message = objName + "'s GC time(" + gcTime + " ms) exceed a threshold."; + ap.time = System.currentTimeMillis(); + ap.objType = objType; + + alert(ap); + } + } + } + } catch (Exception e) { + Logger.printStackTrace(e); + } + } private void println(Object o) { if (conf.getBoolean("ext_plugin_telegram_debug", false)) {