kubelet-service-kill.log

$ k logs kubelet-service-kill-t29n79-785ck
ansible-playbook 2.7.3
  config file = /etc/ansible/ansible.cfg
  configured module search path = [u'/root/.ansible/plugins/modules', u'/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/local/lib/python2.7/dist-packages/ansible
  executable location = /usr/local/bin/ansible-playbook
  python version = 2.7.17 (default, Apr 15 2020, 17:20:14) [GCC 7.5.0]
Using /etc/ansible/ansible.cfg as config file
/etc/ansible/hosts did not meet host_list requirements, check plugin documentation if this is unexpected
/etc/ansible/hosts did not meet script requirements, check plugin documentation if this is unexpected
statically imported: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_prerequisites.yml

2020-06-19 07:11:33.399800 PLAYBOOK: kubelet_service_kill_ansible_logic.yml
1 plays in ./experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml

2020-06-19 07:11:33.419075 ************ BRACE YOURSELF, EXPERIMENT BEGINS! ************

2020-06-19 07:11:37.071059 Step: Gathering Facts
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:2
META: ran handlers

2020-06-19 07:11:37.579216 Step: [PreReq] Identify the chaos util for kubelet-service-kill experiment
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_prerequisites.yml:1
details => {"changed": true, "checksum": "2d76d919fa3c20d76eff0b8e3148897f4319c491", "dest": "/tmp/chaosutil.yml", "gid": 0, "group": "root", "md5sum": "c855258e0f920687b5ce930bafa965a7", "mode": "0644", "owner": "root", "size": 73, "src": "/tmp/ansible-tmp-1592550697.13-219531094582819/source", "state": "file", "uid": 0}

2020-06-19 07:11:37.643989 Step: [PreReq]: Including the chaos util for the kubelet-service-kill experiment
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:27
details => {"ansible_facts": {"c_util": "/chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml"}, "ansible_included_var_files": ["/tmp/chaosutil.yml"], "changed": false}

2020-06-19 07:11:37.701078 Step: [PreReq]: Constructing the chaos result name
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:32
details => {"ansible_facts": {"c_result": "nginx-chaos-kubelet-service-kill"}, "changed": false}
included: /utils/runtime/update_chaos_result_resource.yml for 127.0.0.1

2020-06-19 07:11:38.047136 Step: [PreReq]: Generate the chaos result CR to reflect SOT (Start of Test)
task path: /utils/runtime/update_chaos_result_resource.yml:3
details => {"changed": true, "checksum": "f54406394c14b6d04508f43f4586bfcc1effd15b", "dest": "/tmp/chaos-result.yml", "gid": 0, "group": "root", "md5sum": "2849d849b8853fba1f2171919135eaf0", "mode": "0644", "owner": "root", "size": 335, "src": "/tmp/ansible-tmp-1592550697.82-174010284860201/source", "state": "file", "uid": 0}

2020-06-19 07:11:39.809924 Step: [PreReq]: Apply the chaos result CR for kubelet-service-kill experiment
task path: /utils/runtime/update_chaos_result_resource.yml:13
details => {"changed": true, "cmd": "kubectl apply -f /tmp/chaos-result.yml -n default", "delta": "0:00:01.450632", "end": "2020-06-19 07:11:39.789224", "failed_when_result": false, "rc": 0, "start": "2020-06-19 07:11:38.338592", "stderr": "", "stderr_lines": [], "stdout": "chaosresult.litmuschaos.io/nginx-chaos-kubelet-service-kill configured", "stdout_lines": ["chaosresult.litmuschaos.io/nginx-chaos-kubelet-service-kill configured"]}

2020-06-19 07:11:39.936977 Step: [Info]: Display the application information passed via the test job
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:45
details => {
    "msg": [
        "The application info is as follows:",
        "Namespace    : default",
        "Label        : app=nginx",
        "Ramp Time    : "
    ]
}
included: /utils/common/status_app_pod.yml for 127.0.0.1

2020-06-19 07:11:40.591295 Step: [Status]: Checking whether application pods are in running state
task path: /utils/common/status_app_pod.yml:2
details => {"attempts": 1, "changed": true, "cmd": "kubectl get pods -n default -l app=nginx -o custom-columns=:.status.phase --no-headers", "delta": "0:00:00.443171", "end": "2020-06-19 07:11:40.562554", "rc": 0, "start": "2020-06-19 07:11:40.119383", "stderr": "", "stderr_lines": [], "stdout": "Running", "stdout_lines": ["Running"]}

2020-06-19 07:11:41.273193 Step: [Status]: Checking whether application containers are in running state
task path: /utils/common/status_app_pod.yml:9
details => {"attempts": 1, "changed": true, "cmd": "kubectl get pod -n default -l app=nginx --no-headers -o jsonpath='{.items[*].status.containerStatuses[*].ready}' | tr ' ' '\\n' | uniq", "delta": "0:00:00.519444", "end": "2020-06-19 07:11:41.250401", "rc": 0, "start": "2020-06-19 07:11:40.730957", "stderr": "", "stderr_lines": [], "stdout": "true", "stdout_lines": ["true"]}
included: /utils/common/generate-kubernetes-chaos-events.yml for 127.0.0.1

2020-06-19 07:11:42.039182 Step: [Event]: Obtain the current date in ISO8601 format
task path: /utils/common/generate-kubernetes-chaos-events.yml:1
details => {"changed": true, "cmd": "date --iso-8601=seconds", "delta": "0:00:00.449483", "end": "2020-06-19 07:11:42.012580", "rc": 0, "start": "2020-06-19 07:11:41.563097", "stderr": "", "stderr_lines": [], "stdout": "2020-06-19T07:11:42+00:00", "stdout_lines": ["2020-06-19T07:11:42+00:00"]}

2020-06-19 07:11:42.095802 Step: set_fact
task path: /utils/common/generate-kubernetes-chaos-events.yml:5
details => {"ansible_facts": {"ts": "2020-06-19T07:11:42Z"}, "changed": false}

2020-06-19 07:11:42.859290 Step: [Event]: Creating a PreChaosCheck event in the chaosengine
task path: /utils/common/generate-kubernetes-chaos-events.yml:8
details => {"changed": true, "method": "patch", "result": {"apiVersion": "v1", "count": 1, "eventTime": null, "firstTimestamp": "2020-06-19T07:11:42Z", "involvedObject": {"apiVersion": "litmuschaos.io/v1alpha1", "kind": "ChaosEngine", "name": "nginx-chaos", "namespace": "default", "uid": "02a70455-d7c9-4346-8d64-e50ccdc3fb08"}, "kind": "Event", "lastTimestamp": "2020-06-19T07:11:42Z", "message": "AUT is Running successfully", "metadata": {"creationTimestamp": "2020-06-19T04:23:45Z", "managedFields": [{"apiVersion": "v1", "fieldsType": "FieldsV1", "fieldsV1": {"f:count": {}, "f:firstTimestamp": {}, "f:involvedObject": {"f:apiVersion": {}, "f:kind": {}, "f:name": {}, "f:namespace": {}, "f:uid": {}}, "f:lastTimestamp": {}, "f:message": {}, "f:reason": {}, "f:source": {"f:component": {}}, "f:type": {}}, "manager": "OpenAPI-Generator", "operation": "Update", "time": "2020-06-19T07:11:42Z"}], "name": "event-\"PreChaosCheck\"", "namespace": "default", "resourceVersion": "70798", "selfLink": "/api/v1/namespaces/default/events/event-%22PreChaosCheck%22", "uid": "d4f4ebda-3987-46bb-ae21-d7c68b05b683"}, "reason": "PreChaosCheck", "reportingComponent": "", "reportingInstance": "", "source": {"component": "kubelet-service-kill-t29n79-785ck"}, "type": "Normal"}}
included: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml for 127.0.0.1

2020-06-19 07:11:43.660616 Step: [Prepare]: Select the application pod name
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:6
details => {"changed": true, "cmd": "kubectl get pod -l app=nginx -n default -o=custom-columns=:metadata.name --no-headers | shuf | head -1", "delta": "0:00:00.545175", "end": "2020-06-19 07:11:43.641499", "rc": 0, "start": "2020-06-19 07:11:43.096324", "stderr": "", "stderr_lines": [], "stdout": "nginx-cbd5c7d97-9jpmb", "stdout_lines": ["nginx-cbd5c7d97-9jpmb"]}

2020-06-19 07:11:43.716275 Step: [Prepare]: Recording the application pod name
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:15
details => {"ansible_facts": {"app_pod": "nginx-cbd5c7d97-9jpmb"}, "changed": false}

2020-06-19 07:11:44.364495 Step: [Prepare]: Identify the application node name
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:21
details => {"changed": true, "cmd": "kubectl get pod nginx-cbd5c7d97-9jpmb -n default --no-headers -o custom-columns=:spec.nodeName", "delta": "0:00:00.500018", "end": "2020-06-19 07:11:44.344409", "rc": 0, "start": "2020-06-19 07:11:43.844391", "stderr": "", "stderr_lines": [], "stdout": "aks-default-40839087-vmss000000", "stdout_lines": ["aks-default-40839087-vmss000000"]}

2020-06-19 07:11:44.422044 Step: [Prepare]: Record the application node name
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:29
details => {"ansible_facts": {"app_node": "aks-default-40839087-vmss000000"}, "changed": false}

2020-06-19 07:11:45.034760 Step: [Prepare:] Generate a run id if not passed from the engine/experiment
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:34
details => {"changed": true, "cmd": "echo $(mktemp) | cut -d '.' -f 2 | cut -c -6", "delta": "0:00:00.437670", "end": "2020-06-19 07:11:44.989955", "rc": 0, "start": "2020-06-19 07:11:44.552285", "stderr": "", "stderr_lines": [], "stdout": "udJurH", "stdout_lines": ["udJurH"]}

2020-06-19 07:11:45.123545 Step: set_fact
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:38
details => {"ansible_facts": {"run_id": "udjurh"}, "changed": false}
included: /utils/common/generate-kubernetes-chaos-events.yml for 127.0.0.1

2020-06-19 07:11:45.814134 Step: [Event]: Obtain the current date in ISO8601 format
task path: /utils/common/generate-kubernetes-chaos-events.yml:1
details => {"changed": true, "cmd": "date --iso-8601=seconds", "delta": "0:00:00.393090", "end": "2020-06-19 07:11:45.792039", "rc": 0, "start": "2020-06-19 07:11:45.398949", "stderr": "", "stderr_lines": [], "stdout": "2020-06-19T07:11:45+00:00", "stdout_lines": ["2020-06-19T07:11:45+00:00"]}

2020-06-19 07:11:45.871176 Step: set_fact
task path: /utils/common/generate-kubernetes-chaos-events.yml:5
details => {"ansible_facts": {"ts": "2020-06-19T07:11:45Z"}, "changed": false}

2020-06-19 07:11:46.514192 Step: [Event]: Creating a ChaosInject event in the chaosengine
task path: /utils/common/generate-kubernetes-chaos-events.yml:8
details => {"changed": true, "method": "patch", "result": {"apiVersion": "v1", "count": 1, "eventTime": null, "firstTimestamp": "2020-06-19T07:11:45Z", "involvedObject": {"apiVersion": "litmuschaos.io/v1alpha1", "kind": "ChaosEngine", "name": "nginx-chaos", "namespace": "default", "uid": "02a70455-d7c9-4346-8d64-e50ccdc3fb08"}, "kind": "Event", "lastTimestamp": "2020-06-19T07:11:45Z", "message": "Injecting kubelet-service-kill chaos on aks-default-40839087-vmss000000 node", "metadata": {"creationTimestamp": "2020-06-19T04:23:51Z", "managedFields": [{"apiVersion": "v1", "fieldsType": "FieldsV1", "fieldsV1": {"f:count": {}, "f:firstTimestamp": {}, "f:involvedObject": {"f:apiVersion": {}, "f:kind": {}, "f:name": {}, "f:namespace": {}, "f:uid": {}}, "f:lastTimestamp": {}, "f:message": {}, "f:reason": {}, "f:source": {"f:component": {}}, "f:type": {}}, "manager": "OpenAPI-Generator", "operation": "Update", "time": "2020-06-19T07:11:46Z"}], "name": "event-\"ChaosInject\"", "namespace": "default", "resourceVersion": "70808", "selfLink": "/api/v1/namespaces/default/events/event-%22ChaosInject%22", "uid": "706e3609-4a9a-413a-8308-f19a0fd80786"}, "reason": "ChaosInject", "reportingComponent": "", "reportingInstance": "", "source": {"component": "kubelet-service-kill-t29n79-785ck"}, "type": "Normal"}}

2020-06-19 07:11:46.761864 Step: [Prepare]: Patch the run_id to kubelet service kill helper pod template
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:51
details => {"changed": true, "checksum": "30785a4c7895afe68ee7f344a4df7637c2420c97", "dest": "/tmp/kubelet-service-kill.yml", "gid": 0, "group": "root", "md5sum": "144fb2bbdda393181df168b84d0dfa9e", "mode": "0644", "owner": "root", "size": 867, "src": "/tmp/ansible-tmp-1592550706.57-226505511513681/source", "state": "file", "uid": 0}

2020-06-19 07:11:46.818101 Step: set_fact
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:59
details => {"ansible_facts": {"pod_running_status": ""}, "changed": false}
included: /utils/common/create_chaos_pod.yml for 127.0.0.1 => (item=1)
included: /utils/common/create_chaos_pod.yml for 127.0.0.1 => (item=2)
included: /utils/common/create_chaos_pod.yml for 127.0.0.1 => (item=3)

2020-06-19 07:11:48.002109 Step: [Inject]: Creating the chaos pod in default namespace
task path: /utils/common/create_chaos_pod.yml:3
fatal: [127.0.0.1]: FAILED! => {"changed": true, "cmd": "kubectl apply -f /tmp/kubelet-service-kill.yml -n default", "delta": "0:00:00.865666", "end": "2020-06-19 07:11:47.982637", "msg": "non-zero return code", "rc": 1, "start": "2020-06-19 07:11:47.116971", "stderr": "error: error parsing /tmp/kubelet-service-kill.yml: error converting YAML to JSON: yaml: line 8: mapping values are not allowed in this context", "stderr_lines": ["error: error parsing /tmp/kubelet-service-kill.yml: error converting YAML to JSON: yaml: line 8: mapping values are not allowed in this context"], "stdout": "", "stdout_lines": []}

2020-06-19 07:11:48.128589 Step: fail
task path: /chaoslib/litmus/kubelet_service_kill/kubelet_service_kill.yml:142
fatal: [127.0.0.1]: FAILED! => {"changed": false, "msg": "kubelet_service_kill lib failed"}

2020-06-19 07:11:48.178707 Step: set_fact
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:139
details => {"ansible_facts": {"flag": "Fail"}, "changed": false}

2020-06-19 07:11:48.251052 Step: [Result]: Getting the final result of kubelet-service-kill experiment
task path: /experiments/generic/kubelet_service_kill/kubelet_service_kill_ansible_logic.yml:142
details => {
    "msg": "kubelet-service-kill experiment has been Failed"
}
included: /utils/runtime/getting_failure_step.yml for 127.0.0.1

2020-06-19 07:11:48.405280 Step: [Failure-Detection]: Recording the offset, on the basis of verbosity
task path: /utils/runtime/getting_failure_step.yml:8
details => {"ansible_facts": {"offset": 2}, "changed": false}

2020-06-19 07:11:49.054043 Step: [Failure-Detection]: Getting name of failure step from experiment pod
task path: /utils/runtime/getting_failure_step.yml:14
details => {"changed": true, "cmd": "kubectl logs kubelet-service-kill-t29n79-785ck -n default | grep \"FAILED!\" -B 2 | head -1 | awk -F \"Step:\" '{print $2}'", "delta": "0:00:00.502243", "end": "2020-06-19 07:11:49.034693", "rc": 0, "start": "2020-06-19 07:11:48.532450", "stderr": "", "stderr_lines": [], "stdout": " [Inject]: Creating the chaos pod in default namespace ", "stdout_lines": [" [Inject]: Creating the chaos pod in default namespace "]}

2020-06-19 07:11:49.117359 Step: [Failure-Detection]: Recording the name of failed step
task path: /utils/runtime/getting_failure_step.yml:21
details => {"ansible_facts": {"failStep": " [Inject]: Creating the chaos pod in default namespace "}, "changed": false}

2020-06-19 07:11:49.184800 Step: [Failure-Detection]: Printing the name of failed step
task path: /utils/runtime/getting_failure_step.yml:25
details => {
    "msg": "failStep:  [Inject]: Creating the chaos pod in default namespace "
}
included: /utils/common/generate-kubernetes-chaos-events.yml for 127.0.0.1

2020-06-19 07:11:49.849972 Step: [Event]: Obtain the current date in ISO8601 format
task path: /utils/common/generate-kubernetes-chaos-events.yml:1
details => {"changed": true, "cmd": "date --iso-8601=seconds", "delta": "0:00:00.390399", "end": "2020-06-19 07:11:49.830147", "rc": 0, "start": "2020-06-19 07:11:49.439748", "stderr": "", "stderr_lines": [], "stdout": "2020-06-19T07:11:49+00:00", "stdout_lines": ["2020-06-19T07:11:49+00:00"]}

2020-06-19 07:11:49.915544 Step: set_fact
task path: /utils/common/generate-kubernetes-chaos-events.yml:5
details => {"ansible_facts": {"ts": "2020-06-19T07:11:49Z"}, "changed": false}

2020-06-19 07:11:50.595592 Step: [Event]: Creating a Summary event in the chaosengine
task path: /utils/common/generate-kubernetes-chaos-events.yml:8
details => {"changed": true, "method": "patch", "result": {"apiVersion": "v1", "count": 1, "eventTime": null, "firstTimestamp": "2020-06-19T07:11:49Z", "involvedObject": {"apiVersion": "litmuschaos.io/v1alpha1", "kind": "ChaosEngine", "name": "nginx-chaos", "namespace": "default", "uid": "02a70455-d7c9-4346-8d64-e50ccdc3fb08"}, "kind": "Event", "lastTimestamp": "2020-06-19T07:11:49Z", "message": "kubelet-service-kill Experiment Failed!", "metadata": {"creationTimestamp": "2020-06-19T04:25:14Z", "managedFields": [{"apiVersion": "v1", "fieldsType": "FieldsV1", "fieldsV1": {"f:count": {}, "f:firstTimestamp": {}, "f:involvedObject": {"f:apiVersion": {}, "f:kind": {}, "f:name": {}, "f:namespace": {}, "f:uid": {}}, "f:lastTimestamp": {}, "f:message": {}, "f:reason": {}, "f:source": {"f:component": {}}, "f:type": {}}, "manager": "OpenAPI-Generator", "operation": "Update", "time": "2020-06-19T07:11:50Z"}], "name": "event-\"Summary\"", "namespace": "default", "resourceVersion": "70823", "selfLink": "/api/v1/namespaces/default/events/event-%22Summary%22", "uid": "0944d2aa-0c67-4de1-a3e6-a49db2f0fe56"}, "reason": "Summary", "reportingComponent": "", "reportingInstance": "", "source": {"component": "kubelet-service-kill-t29n79-785ck"}, "type": "Normal"}}
included: /utils/runtime/update_chaos_result_resource.yml for 127.0.0.1

2020-06-19 07:11:51.583135 Step: [Result]: Update the chaos result CR to reflect EOT (End of Test)
task path: /utils/runtime/update_chaos_result_resource.yml:23
details => {"changed": true, "checksum": "ea1fa3b2ef57a2b590f54a9d9d1588b6c0ca7422", "dest": "/tmp/chaos-result.yml", "gid": 0, "group": "root", "md5sum": "38beb5b9ac55f786448bb64c4eff0b37", "mode": "0644", "owner": "root", "size": 406, "src": "/tmp/ansible-tmp-1592550710.91-47322188869259/source", "state": "file", "uid": 0}

2020-06-19 07:11:52.751951 Step: [The END]: Apply the chaos result CR for kubelet-service-kill experiment
task path: /utils/runtime/update_chaos_result_resource.yml:33
details => {"changed": true, "cmd": "kubectl apply -f /tmp/chaos-result.yml -n default", "delta": "0:00:00.967984", "end": "2020-06-19 07:11:52.722779", "failed_when_result": false, "rc": 0, "start": "2020-06-19 07:11:51.754795", "stderr": "", "stderr_lines": [], "stdout": "chaosresult.litmuschaos.io/nginx-chaos-kubelet-service-kill configured", "stdout_lines": ["chaosresult.litmuschaos.io/nginx-chaos-kubelet-service-kill configured"]}
META: ran handlers
META: ran handlers

2020-06-19 07:11:52.755058 ************ RELAX, EXPERIMENT ENDS! ************
127.0.0.1                  : ok=45   changed=18   unreachable=0    failed=2