Created
January 12, 2015 11:01
-
-
Save mikepea/511ebb8b723f69cea3a4 to your computer and use it in GitHub Desktop.
ElasticSearch output of _nodes/stats?pretty -- for analysis of why heap needs to be over 2GB.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cluster_name" : "elasticsearch", | |
| "nodes" : { | |
| "REDACTED" : { | |
| "timestamp" : 1421060315273, | |
| "name" : "monitoring-02", | |
| "transport_address" : "inet[/REDACTED:9300]", | |
| "host" : "monitoring-02", | |
| "ip" : [ "inet[/REDACTED:9300]", "NONE" ], | |
| "indices" : { | |
| "docs" : { | |
| "count" : 33572665, | |
| "deleted" : 0 | |
| }, | |
| "store" : { | |
| "size_in_bytes" : 35325488265, | |
| "throttle_time_in_millis" : 1001644 | |
| }, | |
| "indexing" : { | |
| "index_total" : 2385254, | |
| "index_time_in_millis" : 1056177, | |
| "index_current" : 0, | |
| "delete_total" : 0, | |
| "delete_time_in_millis" : 0, | |
| "delete_current" : 0 | |
| }, | |
| "get" : { | |
| "total" : 2, | |
| "time_in_millis" : 1, | |
| "exists_total" : 2, | |
| "exists_time_in_millis" : 1, | |
| "missing_total" : 0, | |
| "missing_time_in_millis" : 0, | |
| "current" : 0 | |
| }, | |
| "search" : { | |
| "open_contexts" : 0, | |
| "query_total" : 5980, | |
| "query_time_in_millis" : 23715, | |
| "query_current" : 0, | |
| "fetch_total" : 686, | |
| "fetch_time_in_millis" : 1142, | |
| "fetch_current" : 0 | |
| }, | |
| "merges" : { | |
| "current" : 0, | |
| "current_docs" : 0, | |
| "current_size_in_bytes" : 0, | |
| "total" : 25591, | |
| "total_time_in_millis" : 8145961, | |
| "total_docs" : 89042897, | |
| "total_size_in_bytes" : 105938452318 | |
| }, | |
| "refresh" : { | |
| "total" : 235568, | |
| "total_time_in_millis" : 2608859 | |
| }, | |
| "flush" : { | |
| "total" : 841, | |
| "total_time_in_millis" : 519917 | |
| }, | |
| "warmer" : { | |
| "current" : 0, | |
| "total" : 492264, | |
| "total_time_in_millis" : 115589 | |
| }, | |
| "filter_cache" : { | |
| "memory_size_in_bytes" : 100688, | |
| "evictions" : 0 | |
| }, | |
| "id_cache" : { | |
| "memory_size_in_bytes" : 0 | |
| }, | |
| "fielddata" : { | |
| "memory_size_in_bytes" : 3713608, | |
| "evictions" : 0 | |
| }, | |
| "percolate" : { | |
| "total" : 0, | |
| "time_in_millis" : 0, | |
| "current" : 0, | |
| "memory_size_in_bytes" : -1, | |
| "memory_size" : "-1b", | |
| "queries" : 0 | |
| }, | |
| "completion" : { | |
| "size_in_bytes" : 0 | |
| }, | |
| "segments" : { | |
| "count" : 2883, | |
| "memory_in_bytes" : 1275165840, | |
| "index_writer_memory_in_bytes" : 475612, | |
| "version_map_memory_in_bytes" : 6536 | |
| }, | |
| "translog" : { | |
| "operations" : 21629, | |
| "size_in_bytes" : 0 | |
| }, | |
| "suggest" : { | |
| "total" : 0, | |
| "time_in_millis" : 0, | |
| "current" : 0 | |
| } | |
| }, | |
| "os" : { | |
| "timestamp" : 1421060316390, | |
| "uptime_in_millis" : 308426, | |
| "load_average" : [ 2.16, 1.71, 1.55 ], | |
| "cpu" : { | |
| "sys" : 1, | |
| "user" : 9, | |
| "idle" : 84, | |
| "usage" : 10, | |
| "stolen" : 0 | |
| }, | |
| "mem" : { | |
| "free_in_bytes" : 1263923200, | |
| "used_in_bytes" : 7108706304, | |
| "free_percent" : 45, | |
| "used_percent" : 54, | |
| "actual_free_in_bytes" : 3821703168, | |
| "actual_used_in_bytes" : 4550926336 | |
| }, | |
| "swap" : { | |
| "used_in_bytes" : 159744, | |
| "free_in_bytes" : 821919744 | |
| } | |
| }, | |
| "process" : { | |
| "timestamp" : 1421060316391, | |
| "open_file_descriptors" : 9482, | |
| "cpu" : { | |
| "percent" : 32, | |
| "sys_in_millis" : 7198300, | |
| "user_in_millis" : 49129050, | |
| "total_in_millis" : 56327350 | |
| }, | |
| "mem" : { | |
| "resident_in_bytes" : 2771247104, | |
| "share_in_bytes" : 38199296, | |
| "total_virtual_in_bytes" : 19026513920 | |
| } | |
| }, | |
| "jvm" : { | |
| "timestamp" : 1421060316396, | |
| "uptime_in_millis" : 303072733, | |
| "mem" : { | |
| "heap_used_in_bytes" : 2264790672, | |
| "heap_used_percent" : 71, | |
| "heap_committed_in_bytes" : 3186360320, | |
| "heap_max_in_bytes" : 3186360320, | |
| "non_heap_used_in_bytes" : 52775528, | |
| "non_heap_committed_in_bytes" : 80363520, | |
| "pools" : { | |
| "young" : { | |
| "used_in_bytes" : 211515000, | |
| "max_in_bytes" : 279183360, | |
| "peak_used_in_bytes" : 279183360, | |
| "peak_max_in_bytes" : 279183360 | |
| }, | |
| "survivor" : { | |
| "used_in_bytes" : 16317936, | |
| "max_in_bytes" : 34865152, | |
| "peak_used_in_bytes" : 34865152, | |
| "peak_max_in_bytes" : 34865152 | |
| }, | |
| "old" : { | |
| "used_in_bytes" : 2036957736, | |
| "max_in_bytes" : 2872311808, | |
| "peak_used_in_bytes" : 2160199936, | |
| "peak_max_in_bytes" : 2872311808 | |
| } | |
| } | |
| }, | |
| "threads" : { | |
| "count" : 72, | |
| "peak_count" : 76 | |
| }, | |
| "gc" : { | |
| "collectors" : { | |
| "young" : { | |
| "collection_count" : 83026, | |
| "collection_time_in_millis" : 1033012 | |
| }, | |
| "old" : { | |
| "collection_count" : 274, | |
| "collection_time_in_millis" : 21758 | |
| } | |
| } | |
| }, | |
| "buffer_pools" : { | |
| "direct" : { | |
| "count" : 64, | |
| "used_in_bytes" : 13061311, | |
| "total_capacity_in_bytes" : 13061311 | |
| }, | |
| "mapped" : { | |
| "count" : 3978, | |
| "used_in_bytes" : 13254262144, | |
| "total_capacity_in_bytes" : 13254262144 | |
| } | |
| } | |
| }, | |
| "thread_pool" : { | |
| "generic" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 5, | |
| "completed" : 91516 | |
| }, | |
| "index" : { | |
| "threads" : 1, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 1, | |
| "completed" : 1 | |
| }, | |
| "snapshot_data" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "bench" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "get" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 2, | |
| "completed" : 2 | |
| }, | |
| "snapshot" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "merge" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 2, | |
| "completed" : 496234 | |
| }, | |
| "suggest" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "bulk" : { | |
| "threads" : 4, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 4, | |
| "completed" : 2385254 | |
| }, | |
| "optimize" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "warmer" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 2, | |
| "completed" : 256252 | |
| }, | |
| "flush" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 2, | |
| "completed" : 229764 | |
| }, | |
| "search" : { | |
| "threads" : 12, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 12, | |
| "completed" : 6666 | |
| }, | |
| "percolate" : { | |
| "threads" : 0, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 0, | |
| "completed" : 0 | |
| }, | |
| "management" : { | |
| "threads" : 3, | |
| "queue" : 0, | |
| "active" : 1, | |
| "rejected" : 0, | |
| "largest" : 3, | |
| "completed" : 50530 | |
| }, | |
| "refresh" : { | |
| "threads" : 2, | |
| "queue" : 0, | |
| "active" : 0, | |
| "rejected" : 0, | |
| "largest" : 2, | |
| "completed" : 235334 | |
| } | |
| }, | |
| "network" : { | |
| "tcp" : { | |
| "active_opens" : 440155, | |
| "passive_opens" : 872034, | |
| "curr_estab" : 130, | |
| "in_segs" : 43919765, | |
| "out_segs" : 42829383, | |
| "retrans_segs" : 4198, | |
| "estab_resets" : 4099, | |
| "attempt_fails" : 204, | |
| "in_errs" : 0, | |
| "out_rsts" : 3735 | |
| } | |
| }, | |
| "fs" : { | |
| "timestamp" : 1421060316396, | |
| "total" : { | |
| "total_in_bytes" : 221333442560, | |
| "free_in_bytes" : 184169394176, | |
| "available_in_bytes" : 172902658048, | |
| "disk_reads" : 66460, | |
| "disk_writes" : 6116675, | |
| "disk_io_op" : 6183135, | |
| "disk_read_size_in_bytes" : 1265398784, | |
| "disk_write_size_in_bytes" : 133966233600, | |
| "disk_io_size_in_bytes" : 135231632384 | |
| }, | |
| "data" : [ { | |
| "path" : "/data/elasticsearch/elasticsearch/nodes/0", | |
| "mount" : "/data", | |
| "dev" : "/dev/mapper/vg_autodata-lv_data", | |
| "total_in_bytes" : 221333442560, | |
| "free_in_bytes" : 184169394176, | |
| "available_in_bytes" : 172902658048, | |
| "disk_reads" : 66460, | |
| "disk_writes" : 6116675, | |
| "disk_io_op" : 6183135, | |
| "disk_read_size_in_bytes" : 1265398784, | |
| "disk_write_size_in_bytes" : 133966233600, | |
| "disk_io_size_in_bytes" : 135231632384 | |
| } ] | |
| }, | |
| "transport" : { | |
| "server_open" : 13, | |
| "rx_count" : 0, | |
| "rx_size_in_bytes" : 0, | |
| "tx_count" : 0, | |
| "tx_size_in_bytes" : 0 | |
| }, | |
| "http" : { | |
| "current_open" : 3, | |
| "total_opened" : 61850 | |
| }, | |
| "fielddata_breaker" : { | |
| "maximum_size_in_bytes" : 1911816192, | |
| "maximum_size" : "1.7gb", | |
| "estimated_size_in_bytes" : 3713608, | |
| "estimated_size" : "3.5mb", | |
| "overhead" : 1.03, | |
| "tripped" : 0 | |
| } | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
As a follow-up, I configured curator to run daily as
curator optimize --older-than 3 --max_num_segments 1This dropped segments (on a test system) down from 600ish to 300ish, and heap seemed to reduce to 40% (from 80%). Interestingly segment memory (on the test system) only reduced from 250MB to 225MB.
I'll post results for the production system above when it has been applied.