feat(dtc-install): waitFor输出hang住

pull/31674/head
huanghong 10 months ago
parent df0ee22be7
commit a2e7238320

@ -1087,9 +1087,9 @@ func (i *Install) executeWaitFor(chartName, waitFor string) error {
const ( const (
maxRetries = 100 maxRetries = 100
retryInterval = 5 * time.Second retryInterval = 5 * time.Second
cmdTimeout = 25 * time.Minute
) )
// 可重试的错误关键字
retryableErrors := []string{ retryableErrors := []string{
"no matching resources found", "no matching resources found",
"etcdserver: leader changed", "etcdserver: leader changed",
@ -1103,16 +1103,29 @@ func (i *Install) executeWaitFor(chartName, waitFor string) error {
"context deadline exceeded", "context deadline exceeded",
"EOF", "EOF",
"no route to host", "no route to host",
"etcdserver: request timed out",
"http2: client connection lost", "http2: client connection lost",
"timed out waiting for the condition",
} }
fmt.Fprintf(os.Stdout, "Executing waitFor: %s\n", waitFor) fmt.Fprintf(os.Stdout, "Executing waitFor: %s\n", waitFor)
os.Stdout.Sync()
for attempt := 1; attempt <= maxRetries; attempt++ { for attempt := 1; attempt <= maxRetries; attempt++ {
cmd := exec.Command("bash", "-c", waitFor) ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout)
stdoutStderr, err := cmd.CombinedOutput()
cmd := exec.CommandContext(ctx, "bash", "-c", waitFor)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
output := strings.TrimSpace(stdout.String() + stderr.String())
ctxErr := ctx.Err()
cancel()
// 成功
if err == nil { if err == nil {
i.cfg.Log("Wait completed for %s", chartName) i.cfg.Log("Wait completed for %s", chartName)
fmt.Fprintf(os.Stdout, "Wait completed for %s\n", chartName) fmt.Fprintf(os.Stdout, "Wait completed for %s\n", chartName)
@ -1120,31 +1133,48 @@ func (i *Install) executeWaitFor(chartName, waitFor string) error {
return nil return nil
} }
output := string(stdoutStderr) errString := err.Error()
// 检查是否是可重试的错误 // context
if ctxErr == context.DeadlineExceeded {
fmt.Fprintf(os.Stdout, "Command context timed out (%d/%d), output: %s\n", attempt, maxRetries, output)
os.Stdout.Sync()
time.Sleep(retryInterval)
continue
}
// 被 signal kill(context 取消时会发生)
if strings.Contains(errString, "signal: killed") {
fmt.Fprintf(os.Stdout, "Command was killed (%d/%d), output: %s\n", attempt, maxRetries, output)
os.Stdout.Sync()
time.Sleep(retryInterval)
continue
}
// 检查可重试错误
shouldRetry := false shouldRetry := false
for _, retryableErr := range retryableErrors { for _, retryableErr := range retryableErrors {
if strings.Contains(output, retryableErr) { if strings.Contains(output, retryableErr) || strings.Contains(errString, retryableErr) {
shouldRetry = true shouldRetry = true
break break
} }
} }
if shouldRetry { if shouldRetry {
fmt.Fprintf(os.Stdout, "Retryable error, retrying (%d/%d): %s\n", attempt, maxRetries, strings.TrimSpace(output)) fmt.Fprintf(os.Stdout, "Retryable error (%d/%d): %s\n", attempt, maxRetries, output)
os.Stdout.Sync()
time.Sleep(retryInterval) time.Sleep(retryInterval)
continue continue
} }
// 不可重试的错误,直接返回 // 不可重试错误
fmt.Fprintf(os.Stdout, "Wait command failed: %s, output: %s\n", err.Error(), output) fmt.Fprintf(os.Stdout, "Wait command failed: %s, output: %s\n", errString, output)
os.Stdout.Sync()
return fmt.Errorf("wait for %s failed: %s", chartName, output) return fmt.Errorf("wait for %s failed: %s", chartName, output)
} }
return fmt.Errorf("wait for %s timed out after %d retries", chartName, maxRetries) return fmt.Errorf("wait for %s timed out after %d retries", chartName, maxRetries)
} }
func (i *Install) handleContext(ctx context.Context, c chan<- resultMessage, done chan struct{}, rel *release.Release) { func (i *Install) handleContext(ctx context.Context, c chan<- resultMessage, done chan struct{}, rel *release.Release) {
select { select {
case <-ctx.Done(): case <-ctx.Done():

Loading…
Cancel
Save