Created
August 29, 2017 17:09
-
-
Save cezarsa/f5ab23ad975cc47b488c15e4969c5936 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/provision/kubernetes/deploy.go b/provision/kubernetes/deploy.go | |
index 587eac2e0..f729c2682 100644 | |
--- a/provision/kubernetes/deploy.go | |
+++ b/provision/kubernetes/deploy.go | |
@@ -455,8 +455,12 @@ func monitorDeployment(client *clusterClient, dep *v1beta1.Deployment, a provisi | |
fmt.Fprintf(w, " ---> %d of %d new units created\n", dep.Status.UpdatedReplicas, specReplicas) | |
} | |
if healthcheckTimeout == nil && dep.Status.UpdatedReplicas == specReplicas { | |
- healthcheckTimeout = time.After(maxWaitTimeDuration) | |
- fmt.Fprintf(w, " ---> waiting healthcheck on %d created units\n", specReplicas) | |
+ var allInit bool | |
+ allInit, err = allPodsInitialized(client, a, processName) | |
+ if allInit && err == nil { | |
+ healthcheckTimeout = time.After(maxWaitTimeDuration) | |
+ fmt.Fprintf(w, " ---> waiting healthcheck on %d created units\n", specReplicas) | |
+ } | |
} | |
readyUnits := dep.Status.UpdatedReplicas - dep.Status.UnavailableReplicas | |
if oldReadyUnits != readyUnits && readyUnits >= 0 { | |
diff --git a/provision/kubernetes/deploy_test.go b/provision/kubernetes/deploy_test.go | |
index 8ddf6fe84..1f78205ba 100644 | |
--- a/provision/kubernetes/deploy_test.go | |
+++ b/provision/kubernetes/deploy_test.go | |
@@ -622,11 +622,11 @@ func (s *S) TestServiceManagerDeployServiceRollback(c *check.C) { | |
err = servicecommon.RunServicePipeline(&m, a, "myimg", servicecommon.ProcessSpec{ | |
"p1": servicecommon.ProcessState{Start: true}, | |
}) | |
- c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units$") | |
+ c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-1-1: invalid pod phase \"Running\"$") | |
c.Assert(rollbackObj, check.DeepEquals, &v1beta1.DeploymentRollback{ | |
Name: "myapp-p1", | |
}) | |
- c.Assert(buf.String(), check.Matches, `(?s).*---- Updating units \[p1\] ----.*ROLLING BACK AFTER FAILURE.*---> timeout after .* waiting for units <---\s*$`) | |
+ c.Assert(buf.String(), check.Matches, `(?s).*---- Updating units \[p1\] ----.*ROLLING BACK AFTER FAILURE.*---> timeout after .* waiting for units: Pod myapp-p1-pod-1-1: invalid pod phase \"Running\" <---\s*$`) | |
cleanupDeployment(s.client.clusterClient, a, "p1") | |
_, err = s.client.Core().Events(s.client.Namespace()).Create(&apiv1.Event{ | |
ObjectMeta: metav1.ObjectMeta{ | |
@@ -640,7 +640,7 @@ func (s *S) TestServiceManagerDeployServiceRollback(c *check.C) { | |
err = servicecommon.RunServicePipeline(&m, a, "myimg", servicecommon.ProcessSpec{ | |
"p1": servicecommon.ProcessState{Start: true}, | |
}) | |
- c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-2-1: Unhealthy - my evt message$") | |
+ c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-2-1: invalid pod phase \"Running\" - last event: my evt message$") | |
} | |
func (s *S) TestServiceManagerRemoveService(c *check.C) { | |
diff --git a/provision/kubernetes/helpers.go b/provision/kubernetes/helpers.go | |
index 784cf01ee..f6dc18132 100644 | |
--- a/provision/kubernetes/helpers.go | |
+++ b/provision/kubernetes/helpers.go | |
@@ -94,50 +94,67 @@ func waitFor(timeout time.Duration, fn func() (bool, error), onTimeout func() er | |
} | |
} | |
+func podsForAppProcess(client *clusterClient, a provision.App, process string) (*apiv1.PodList, error) { | |
+ labelOpts := provision.ServiceLabelsOpts{ | |
+ App: a, | |
+ Process: process, | |
+ ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{ | |
+ Prefix: tsuruLabelPrefix, | |
+ Provisioner: provisionerName, | |
+ }, | |
+ } | |
+ l, err := provision.ServiceLabels(labelOpts) | |
+ if err != nil { | |
+ return nil, errors.WithStack(err) | |
+ } | |
+ var selector map[string]string | |
+ if process == "" { | |
+ selector = l.ToAppSelector() | |
+ } else { | |
+ selector = l.ToSelector() | |
+ } | |
+ return client.Core().Pods(client.Namespace()).List(metav1.ListOptions{ | |
+ LabelSelector: labels.SelectorFromSet(labels.Set(selector)).String(), | |
+ }) | |
+} | |
+ | |
+func allPodsInitialized(client *clusterClient, a provision.App, process string) (bool, error) { | |
+ pods, err := podsForAppProcess(client, a, process) | |
+ if err != nil { | |
+ return false, errors.WithStack(err) | |
+ } | |
+ for _, pod := range pods.Items { | |
+ for _, cond := range pod.Status.Conditions { | |
+ if cond.Type != apiv1.PodInitialized { | |
+ continue | |
+ } | |
+ if cond.Status != apiv1.ConditionTrue { | |
+ return false, nil | |
+ } | |
+ } | |
+ } | |
+ return true, nil | |
+} | |
+ | |
func notReadyPodEvents(client *clusterClient, a provision.App, process string) ([]string, error) { | |
- l, err := provision.ServiceLabels(provision.ServiceLabelsOpts{ | |
- App: a, | |
- Process: process, | |
- ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{ | |
- Prefix: tsuruLabelPrefix, | |
- Provisioner: provisionerName, | |
- }, | |
- }) | |
+ pods, err := podsForAppProcess(client, a, process) | |
if err != nil { | |
return nil, errors.WithStack(err) | |
} | |
- pods, err := client.Core().Pods(client.Namespace()).List(metav1.ListOptions{ | |
- LabelSelector: labels.SelectorFromSet(labels.Set(l.ToSelector())).String(), | |
- }) | |
- if err != nil { | |
- return nil, errors.WithStack(err) | |
- } | |
- var podsForEvts []string | |
+ var podsForEvts []*apiv1.Pod | |
podsLoop: | |
- for _, pod := range pods.Items { | |
+ for i, pod := range pods.Items { | |
for _, cond := range pod.Status.Conditions { | |
if cond.Type == apiv1.PodReady && cond.Status != apiv1.ConditionTrue { | |
- podsForEvts = append(podsForEvts, pod.Name) | |
+ podsForEvts = append(podsForEvts, &pods.Items[i]) | |
continue podsLoop | |
} | |
} | |
} | |
var messages []string | |
- for _, podName := range podsForEvts { | |
- eventsInterface := client.Core().Events(client.Namespace()) | |
- ns := client.Namespace() | |
- selector := eventsInterface.GetFieldSelector(&podName, &ns, nil, nil) | |
- options := metav1.ListOptions{FieldSelector: selector.String()} | |
- var events *apiv1.EventList | |
- events, err = eventsInterface.List(options) | |
- if err != nil { | |
- return nil, errors.WithStack(err) | |
- } | |
- if len(events.Items) == 0 { | |
- continue | |
- } | |
- lastEvt := events.Items[len(events.Items)-1] | |
- messages = append(messages, fmt.Sprintf("Pod %s: %s - %s", podName, lastEvt.Reason, lastEvt.Message)) | |
+ for _, pod := range podsForEvts { | |
+ err = newInvalidPodPhaseError(client, pod) | |
+ messages = append(messages, fmt.Sprintf("Pod %s: %v", pod.Name, err)) | |
} | |
return messages, nil | |
} | |
@@ -395,21 +412,8 @@ func execCommand(opts execOpts) error { | |
return errors.WithStack(err) | |
} | |
} else { | |
- var l *provision.LabelSet | |
- l, err = provision.ServiceLabels(provision.ServiceLabelsOpts{ | |
- App: opts.app, | |
- ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{ | |
- Prefix: tsuruLabelPrefix, | |
- Provisioner: provisionerName, | |
- }, | |
- }) | |
- if err != nil { | |
- return errors.WithStack(err) | |
- } | |
var pods *apiv1.PodList | |
- pods, err = client.Core().Pods(client.Namespace()).List(metav1.ListOptions{ | |
- LabelSelector: labels.SelectorFromSet(labels.Set(l.ToAppSelector())).String(), | |
- }) | |
+ pods, err = podsForAppProcess(client, opts.app, "") | |
if err != nil { | |
return errors.WithStack(err) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment