Created
August 16, 2018 19:14
-
-
Save jasontedor/17ffd2e5e7b3732eb22801cb2ac076b6 to your computer and use it in GitHub Desktop.
failed watch
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "took" : 1, | |
| "timed_out" : false, | |
| "_shards" : { | |
| "total" : 1, | |
| "successful" : 1, | |
| "skipped" : 0, | |
| "failed" : 0 | |
| }, | |
| "hits" : { | |
| "total" : 30, | |
| "max_score" : null, | |
| "hits" : [ | |
| { | |
| "_index" : ".watcher-history-9-2018.08.16", | |
| "_type" : "doc", | |
| "_id" : "JYPUvc75TMCSkohHCS5KyQ_elasticsearch_nodes_7425d6b6-ee99-4be3-b35c-31f4dfb69e51-2018-08-16T19:13:41.446Z", | |
| "_score" : null, | |
| "_source" : { | |
| "watch_id" : "JYPUvc75TMCSkohHCS5KyQ_elasticsearch_nodes", | |
| "node" : "JBVOs_BeTciRXxwbtM1jHA", | |
| "state" : "executed", | |
| "status" : { | |
| "state" : { | |
| "active" : true, | |
| "timestamp" : "2018-08-16T18:42:07.262Z" | |
| }, | |
| "last_checked" : "2018-08-16T19:13:41.446Z", | |
| "last_met_condition" : "2018-08-16T19:13:41.446Z", | |
| "actions" : { | |
| "send_email_to_admin" : { | |
| "ack" : { | |
| "timestamp" : "2018-08-16T18:42:07.262Z", | |
| "state" : "awaits_successful_execution" | |
| } | |
| }, | |
| "add_to_alerts_index" : { | |
| "ack" : { | |
| "timestamp" : "2018-08-16T18:42:07.262Z", | |
| "state" : "awaits_successful_execution" | |
| } | |
| } | |
| }, | |
| "execution_state" : "executed", | |
| "version" : -1 | |
| }, | |
| "trigger_event" : { | |
| "type" : "schedule", | |
| "triggered_time" : "2018-08-16T19:13:41.446Z", | |
| "schedule" : { | |
| "scheduled_time" : "2018-08-16T19:13:40.959Z" | |
| } | |
| }, | |
| "input" : { | |
| "chain" : { | |
| "inputs" : [ | |
| { | |
| "check" : { | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-es-*" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 2, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ], | |
| "_source" : [ | |
| "cluster_state.nodes_hash", | |
| "cluster_state.nodes.*.name", | |
| "cluster_state.nodes.*.ephemeral_id" | |
| ], | |
| "collapse" : { | |
| "field" : "cluster_state.nodes_hash" | |
| }, | |
| "query" : { | |
| "bool" : { | |
| "filter" : [ | |
| { | |
| "term" : { | |
| "cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ" | |
| } | |
| }, | |
| { | |
| "term" : { | |
| "type" : "cluster_stats" | |
| } | |
| }, | |
| { | |
| "range" : { | |
| "timestamp" : { | |
| "gte" : "now-2d" | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "alert" : { | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-alerts-6" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 1, | |
| "_source" : [ | |
| "nodes.hash" | |
| ], | |
| "query" : { | |
| "bool" : { | |
| "filter" : [ | |
| { | |
| "term" : { | |
| "metadata.cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ" | |
| } | |
| }, | |
| { | |
| "term" : { | |
| "metadata.watch" : "elasticsearch_nodes" | |
| } | |
| } | |
| ] | |
| } | |
| }, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "kibana_settings" : { | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-kibana-*" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 1, | |
| "query" : { | |
| "bool" : { | |
| "filter" : { | |
| "term" : { | |
| "type" : "kibana_settings" | |
| } | |
| } | |
| } | |
| }, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| }, | |
| "condition" : { | |
| "script" : { | |
| "source" : "ctx.payload.check.hits.hits.length > 1 && (ctx.payload.alert.hits.total == 0 || ctx.payload.alert.hits.hits[0]._source.nodes.hash != ctx.payload.check.hits.hits[0]._source.cluster_state.nodes_hash)", | |
| "lang" : "painless" | |
| } | |
| }, | |
| "metadata" : { | |
| "name" : "X-Pack Monitoring: Nodes Changed (JYPUvc75TMCSkohHCS5KyQ)", | |
| "xpack" : { | |
| "severity" : 1999, | |
| "cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ", | |
| "version_created" : 6040099, | |
| "watch" : "elasticsearch_nodes", | |
| "link" : "elasticsearch/nodes", | |
| "alert_index" : ".monitoring-alerts-6", | |
| "type" : "monitoring" | |
| } | |
| }, | |
| "result" : { | |
| "execution_time" : "2018-08-16T19:13:41.446Z", | |
| "execution_duration" : 15, | |
| "input" : { | |
| "type" : "chain", | |
| "status" : "success", | |
| "payload" : { | |
| "alert" : { | |
| "_shards" : { | |
| "total" : 0, | |
| "failed" : 0, | |
| "successful" : 0, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ ], | |
| "total" : 0, | |
| "max_score" : 0.0 | |
| }, | |
| "took" : 0, | |
| "timed_out" : false | |
| }, | |
| "kibana_settings" : { | |
| "_shards" : { | |
| "total" : 1, | |
| "failed" : 0, | |
| "successful" : 1, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ | |
| { | |
| "_index" : ".monitoring-kibana-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "interval_ms" : 10000, | |
| "cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ", | |
| "source_node" : { | |
| "transport_address" : "172.21.0.2:9300", | |
| "ip" : "172.21.0.2", | |
| "host" : "172.21.0.2", | |
| "name" : "JBVOs_B", | |
| "uuid" : "JBVOs_BeTciRXxwbtM1jHA", | |
| "timestamp" : "2018-08-16T19:13:39.490Z" | |
| }, | |
| "kibana_settings" : { | |
| "kibana" : { | |
| "transport_address" : "0:5601", | |
| "name" : "kibana.example.org", | |
| "host" : "0", | |
| "index" : ".kibana", | |
| "uuid" : "26ba524c-4be0-4b7d-bc57-3f07c2f229a9", | |
| "version" : "6.4.0", | |
| "snapshot" : true, | |
| "status" : "green" | |
| } | |
| }, | |
| "type" : "kibana_settings", | |
| "timestamp" : "2018-08-16T19:13:39.490Z" | |
| }, | |
| "_id" : "xdQoRGUB6aPRDY57WDOl", | |
| "sort" : [ | |
| 1534446819490 | |
| ], | |
| "_score" : null | |
| } | |
| ], | |
| "total" : 179, | |
| "max_score" : null | |
| }, | |
| "took" : 1, | |
| "timed_out" : false | |
| }, | |
| "check" : { | |
| "_shards" : { | |
| "total" : 1, | |
| "failed" : 0, | |
| "successful" : 1, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ | |
| { | |
| "_index" : ".monitoring-es-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "cluster_state" : { | |
| "nodes_hash" : 1852667448, | |
| "nodes" : { | |
| "JBVOs_BeTciRXxwbtM1jHA" : { | |
| "name" : "JBVOs_B", | |
| "ephemeral_id" : "0gNUI08_QTeS_BM4HsXHkA" | |
| } | |
| } | |
| } | |
| }, | |
| "_id" : "u9QoRGUB6aPRDY57QzP9", | |
| "sort" : [ | |
| 1534446814196 | |
| ], | |
| "_score" : null, | |
| "fields" : { | |
| "cluster_state.nodes_hash" : [ | |
| 1852667448 | |
| ] | |
| } | |
| }, | |
| { | |
| "_index" : ".monitoring-es-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "cluster_state" : { | |
| "nodes_hash" : -558240336, | |
| "nodes" : { | |
| "JBVOs_BeTciRXxwbtM1jHA" : { | |
| "name" : "JBVOs_B", | |
| "ephemeral_id" : "cIDsSJaWSvSzXJCaVyI5lQ" | |
| } | |
| } | |
| } | |
| }, | |
| "_id" : "DuQcRGUBOlJW3YjpQejW", | |
| "sort" : [ | |
| 1534446027217 | |
| ], | |
| "_score" : null, | |
| "fields" : { | |
| "cluster_state.nodes_hash" : [ | |
| -558240336 | |
| ] | |
| } | |
| } | |
| ], | |
| "total" : 183, | |
| "max_score" : null | |
| }, | |
| "took" : 8, | |
| "timed_out" : false | |
| } | |
| }, | |
| "chain" : { | |
| "check" : { | |
| "type" : "search", | |
| "status" : "success", | |
| "payload" : { | |
| "_shards" : { | |
| "total" : 1, | |
| "failed" : 0, | |
| "successful" : 1, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ | |
| { | |
| "_index" : ".monitoring-es-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "cluster_state" : { | |
| "nodes_hash" : 1852667448, | |
| "nodes" : { | |
| "JBVOs_BeTciRXxwbtM1jHA" : { | |
| "name" : "JBVOs_B", | |
| "ephemeral_id" : "0gNUI08_QTeS_BM4HsXHkA" | |
| } | |
| } | |
| } | |
| }, | |
| "_id" : "u9QoRGUB6aPRDY57QzP9", | |
| "sort" : [ | |
| 1534446814196 | |
| ], | |
| "_score" : null, | |
| "fields" : { | |
| "cluster_state.nodes_hash" : [ | |
| 1852667448 | |
| ] | |
| } | |
| }, | |
| { | |
| "_index" : ".monitoring-es-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "cluster_state" : { | |
| "nodes_hash" : -558240336, | |
| "nodes" : { | |
| "JBVOs_BeTciRXxwbtM1jHA" : { | |
| "name" : "JBVOs_B", | |
| "ephemeral_id" : "cIDsSJaWSvSzXJCaVyI5lQ" | |
| } | |
| } | |
| } | |
| }, | |
| "_id" : "DuQcRGUBOlJW3YjpQejW", | |
| "sort" : [ | |
| 1534446027217 | |
| ], | |
| "_score" : null, | |
| "fields" : { | |
| "cluster_state.nodes_hash" : [ | |
| -558240336 | |
| ] | |
| } | |
| } | |
| ], | |
| "total" : 183, | |
| "max_score" : null | |
| }, | |
| "took" : 8, | |
| "timed_out" : false | |
| }, | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-es-*" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 2, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ], | |
| "_source" : [ | |
| "cluster_state.nodes_hash", | |
| "cluster_state.nodes.*.name", | |
| "cluster_state.nodes.*.ephemeral_id" | |
| ], | |
| "collapse" : { | |
| "field" : "cluster_state.nodes_hash" | |
| }, | |
| "query" : { | |
| "bool" : { | |
| "filter" : [ | |
| { | |
| "term" : { | |
| "cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ" | |
| } | |
| }, | |
| { | |
| "term" : { | |
| "type" : "cluster_stats" | |
| } | |
| }, | |
| { | |
| "range" : { | |
| "timestamp" : { | |
| "gte" : "now-2d" | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "alert" : { | |
| "type" : "search", | |
| "status" : "success", | |
| "payload" : { | |
| "_shards" : { | |
| "total" : 0, | |
| "failed" : 0, | |
| "successful" : 0, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ ], | |
| "total" : 0, | |
| "max_score" : 0.0 | |
| }, | |
| "took" : 0, | |
| "timed_out" : false | |
| }, | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-alerts-6" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 1, | |
| "_source" : [ | |
| "nodes.hash" | |
| ], | |
| "query" : { | |
| "bool" : { | |
| "filter" : [ | |
| { | |
| "term" : { | |
| "metadata.cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ" | |
| } | |
| }, | |
| { | |
| "term" : { | |
| "metadata.watch" : "elasticsearch_nodes" | |
| } | |
| } | |
| ] | |
| } | |
| }, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "kibana_settings" : { | |
| "type" : "search", | |
| "status" : "success", | |
| "payload" : { | |
| "_shards" : { | |
| "total" : 1, | |
| "failed" : 0, | |
| "successful" : 1, | |
| "skipped" : 0 | |
| }, | |
| "hits" : { | |
| "hits" : [ | |
| { | |
| "_index" : ".monitoring-kibana-6-2018.08.16", | |
| "_type" : "doc", | |
| "_source" : { | |
| "interval_ms" : 10000, | |
| "cluster_uuid" : "JYPUvc75TMCSkohHCS5KyQ", | |
| "source_node" : { | |
| "transport_address" : "172.21.0.2:9300", | |
| "ip" : "172.21.0.2", | |
| "host" : "172.21.0.2", | |
| "name" : "JBVOs_B", | |
| "uuid" : "JBVOs_BeTciRXxwbtM1jHA", | |
| "timestamp" : "2018-08-16T19:13:39.490Z" | |
| }, | |
| "kibana_settings" : { | |
| "kibana" : { | |
| "transport_address" : "0:5601", | |
| "name" : "kibana.example.org", | |
| "host" : "0", | |
| "index" : ".kibana", | |
| "uuid" : "26ba524c-4be0-4b7d-bc57-3f07c2f229a9", | |
| "version" : "6.4.0", | |
| "snapshot" : true, | |
| "status" : "green" | |
| } | |
| }, | |
| "type" : "kibana_settings", | |
| "timestamp" : "2018-08-16T19:13:39.490Z" | |
| }, | |
| "_id" : "xdQoRGUB6aPRDY57WDOl", | |
| "sort" : [ | |
| 1534446819490 | |
| ], | |
| "_score" : null | |
| } | |
| ], | |
| "total" : 179, | |
| "max_score" : null | |
| }, | |
| "took" : 1, | |
| "timed_out" : false | |
| }, | |
| "search" : { | |
| "request" : { | |
| "search_type" : "query_then_fetch", | |
| "indices" : [ | |
| ".monitoring-kibana-*" | |
| ], | |
| "types" : [ ], | |
| "body" : { | |
| "size" : 1, | |
| "query" : { | |
| "bool" : { | |
| "filter" : { | |
| "term" : { | |
| "type" : "kibana_settings" | |
| } | |
| } | |
| } | |
| }, | |
| "sort" : [ | |
| { | |
| "timestamp" : { | |
| "order" : "desc" | |
| } | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "condition" : { | |
| "type" : "script", | |
| "status" : "success", | |
| "met" : true | |
| }, | |
| "transform" : { | |
| "type" : "script", | |
| "status" : "failure", | |
| "reason" : "runtime error", | |
| "error" : { | |
| "root_cause" : [ | |
| { | |
| "type" : "script_exception", | |
| "reason" : "runtime error", | |
| "script_stack" : [ | |
| "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def ", | |
| " ^---- HERE" | |
| ], | |
| "script" : "void formatResults(StringBuilder message, String type, Map typeMap) {if (typeMap.empty == false) {message.append(' Node');if (typeMap.size() != 1) {message.append('s were');} else {message.append(' was');}message.append(' ').append(type).append(' [').append(typeMap.size()).append(']: ').append(typeMap.values().stream().collect(Collectors.joining(', ', '[', ']'))).append('.');}}ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def clusterState = ctx.payload.check.hits.hits[0]._source.cluster_state;def persistentUuidToName = [:];def latestNodes = clusterState.nodes;def ephemeralUuidToPersistentUuid = [:];def payload = ['timestamp': ctx.execution_time,'updated_timestamp': ctx.execution_time,'resolved_timestamp': ctx.execution_time,'metadata': ctx.metadata.xpack,'prefix': 'Elasticsearch cluster nodes have changed!','nodes': ['hash': clusterState.nodes_hash,'added': persistentUuidToName,'removed': [:],'restarted': [:]]];for (def latestNode : latestNodes.entrySet()) {persistentUuidToName[latestNode.key] = latestNode.value.name;ephemeralUuidToPersistentUuid[latestNode.value.ephemeral_id] = latestNode.key;}def previousNodes = ctx.payload.check.hits.hits[1]._source.cluster_state.nodes;def previousPersistentUuidToName = [:];for (def previousNode : previousNodes.entrySet()){if (persistentUuidToName.containsKey(previousNode.key) == false){payload.nodes.removed[previousNode.key] = previousNode.value.name;}else{if (ephemeralUuidToPersistentUuid.containsKey(previousNode.value.ephemeral_id) == false) {payload.nodes.restarted[previousNode.key] = persistentUuidToName[previousNode.key];}persistentUuidToName.remove(previousNode.key);}}StringBuilder message = new StringBuilder();formatResults(message, 'removed', payload.nodes.removed);formatResults(message, 'added', payload.nodes.added);formatResults(message, 'restarted', payload.nodes.restarted);payload.message = message.toString().trim();return payload;", | |
| "lang" : "painless" | |
| } | |
| ], | |
| "type" : "script_exception", | |
| "reason" : "runtime error", | |
| "script_stack" : [ | |
| "ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def ", | |
| " ^---- HERE" | |
| ], | |
| "script" : "void formatResults(StringBuilder message, String type, Map typeMap) {if (typeMap.empty == false) {message.append(' Node');if (typeMap.size() != 1) {message.append('s were');} else {message.append(' was');}message.append(' ').append(type).append(' [').append(typeMap.size()).append(']: ').append(typeMap.values().stream().collect(Collectors.joining(', ', '[', ']'))).append('.');}}ctx.vars.email_recipient = (ctx.payload.kibana_settings.hits.total > 0) ? ctx.payload.kibana_settings.hits.hits[0]._source.kibana_settings.xpack.default_admin_email : null;def clusterState = ctx.payload.check.hits.hits[0]._source.cluster_state;def persistentUuidToName = [:];def latestNodes = clusterState.nodes;def ephemeralUuidToPersistentUuid = [:];def payload = ['timestamp': ctx.execution_time,'updated_timestamp': ctx.execution_time,'resolved_timestamp': ctx.execution_time,'metadata': ctx.metadata.xpack,'prefix': 'Elasticsearch cluster nodes have changed!','nodes': ['hash': clusterState.nodes_hash,'added': persistentUuidToName,'removed': [:],'restarted': [:]]];for (def latestNode : latestNodes.entrySet()) {persistentUuidToName[latestNode.key] = latestNode.value.name;ephemeralUuidToPersistentUuid[latestNode.value.ephemeral_id] = latestNode.key;}def previousNodes = ctx.payload.check.hits.hits[1]._source.cluster_state.nodes;def previousPersistentUuidToName = [:];for (def previousNode : previousNodes.entrySet()){if (persistentUuidToName.containsKey(previousNode.key) == false){payload.nodes.removed[previousNode.key] = previousNode.value.name;}else{if (ephemeralUuidToPersistentUuid.containsKey(previousNode.value.ephemeral_id) == false) {payload.nodes.restarted[previousNode.key] = persistentUuidToName[previousNode.key];}persistentUuidToName.remove(previousNode.key);}}StringBuilder message = new StringBuilder();formatResults(message, 'removed', payload.nodes.removed);formatResults(message, 'added', payload.nodes.added);formatResults(message, 'restarted', payload.nodes.restarted);payload.message = message.toString().trim();return payload;", | |
| "lang" : "painless", | |
| "caused_by" : { | |
| "type" : "null_pointer_exception", | |
| "reason" : null | |
| } | |
| } | |
| }, | |
| "actions" : [ ] | |
| }, | |
| "messages" : [ | |
| "failed to execute watch transform" | |
| ] | |
| }, | |
| "sort" : [ | |
| 1534446821446 | |
| ] | |
| } | |
| ] | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment