From ada108126ca45af6f3fbf501e8421a8bdaeeee4a Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Wed, 1 Jun 2016 10:47:26 -0700 Subject: [PATCH 1/5] Add HTTP server for liveness and readiness probes --- cmd/tiller/probes.go | 20 ++++++++++++++++++++ cmd/tiller/tiller.go | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 cmd/tiller/probes.go diff --git a/cmd/tiller/probes.go b/cmd/tiller/probes.go new file mode 100644 index 000000000..3b2286c94 --- /dev/null +++ b/cmd/tiller/probes.go @@ -0,0 +1,20 @@ +package main + +import ( + "net/http" +) + +func readinessProbe(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func livenessProbe(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func runProbesServer(addr string) error { + mux := http.NewServeMux() + mux.HandleFunc("/readiness", readinessProbe) + mux.HandleFunc("/liveness", livenessProbe) + return http.ListenAndServe(addr, mux) +} diff --git a/cmd/tiller/tiller.go b/cmd/tiller/tiller.go index aba7561ea..a1b51a827 100644 --- a/cmd/tiller/tiller.go +++ b/cmd/tiller/tiller.go @@ -21,6 +21,7 @@ var rootServer = grpc.NewServer() var env = environment.New() var addr = ":44134" +var probe = ":44135" var namespace = "" const globalUsage = `The Kubernetes Helm server. @@ -53,10 +54,28 @@ func start(c *cobra.Command, args []string) { } fmt.Printf("Tiller is running on %s\n", addr) + fmt.Printf("Tiller probes server is running on %s\n", probe) - if err := rootServer.Serve(lstn); err != nil { + srvErrCh := make(chan error) + probeErrCh := make(chan error) + go func() { + if err := rootServer.Serve(lstn); err != nil { + srvErrCh <- err + } + }() + + go func() { + if err := runProbesServer(probe); err != nil { + probeErrCh <- err + } + }() + + select { + case err := <-srvErrCh: fmt.Fprintf(os.Stderr, "Server died: %s\n", err) os.Exit(1) + case err := <-probeErrCh: + fmt.Fprintf(os.Stderr, "Probes server died: %s\n", err) } } From fe3afa8012588c6efa036ab2eb662cd15a83f215 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Wed, 1 Jun 2016 10:54:33 -0700 Subject: [PATCH 2/5] Added test for probes ServeMux --- cmd/tiller/probes.go | 4 ++-- cmd/tiller/probes_test.go | 28 ++++++++++++++++++++++++++++ cmd/tiller/tiller.go | 4 +++- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 cmd/tiller/probes_test.go diff --git a/cmd/tiller/probes.go b/cmd/tiller/probes.go index 3b2286c94..fcbb20665 100644 --- a/cmd/tiller/probes.go +++ b/cmd/tiller/probes.go @@ -12,9 +12,9 @@ func livenessProbe(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func runProbesServer(addr string) error { +func newProbesMux() *http.ServeMux { mux := http.NewServeMux() mux.HandleFunc("/readiness", readinessProbe) mux.HandleFunc("/liveness", livenessProbe) - return http.ListenAndServe(addr, mux) + return mux } diff --git a/cmd/tiller/probes_test.go b/cmd/tiller/probes_test.go new file mode 100644 index 000000000..263164648 --- /dev/null +++ b/cmd/tiller/probes_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestProbesServer(t *testing.T) { + mux := newProbesMux() + srv := httptest.NewServer(mux) + defer srv.Close() + resp, err := http.Get("/readiness") + if err != nil { + t.Fatalf("GET /readiness returned an error (%s)", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET /readiness returned status code %d, expected %d", resp.StatusCode, http.StatusOK) + } + + resp, err = http.Get("/liveness") + if err != nil { + t.Fatalf("GET /liveness returned an error (%s)", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET /liveness returned status code %d, expected %d", resp.StatusCode, http.StatusOK) + } +} diff --git a/cmd/tiller/tiller.go b/cmd/tiller/tiller.go index a1b51a827..f1b209a10 100644 --- a/cmd/tiller/tiller.go +++ b/cmd/tiller/tiller.go @@ -3,6 +3,7 @@ package main import ( "fmt" "net" + "net/http" "os" "github.com/kubernetes/helm/cmd/tiller/environment" @@ -65,7 +66,8 @@ func start(c *cobra.Command, args []string) { }() go func() { - if err := runProbesServer(probe); err != nil { + mux := newProbesMux() + if err := http.ListenAndServe(addr, mux); err != nil { probeErrCh <- err } }() From 56069ae4741046e6fad98f9f5d56207a8a819f9c Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Wed, 1 Jun 2016 11:39:31 -0700 Subject: [PATCH 3/5] Fix tests --- cmd/tiller/probes_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tiller/probes_test.go b/cmd/tiller/probes_test.go index 263164648..5b625fcc4 100644 --- a/cmd/tiller/probes_test.go +++ b/cmd/tiller/probes_test.go @@ -10,7 +10,7 @@ func TestProbesServer(t *testing.T) { mux := newProbesMux() srv := httptest.NewServer(mux) defer srv.Close() - resp, err := http.Get("/readiness") + resp, err := http.Get(srv.URL + "/readiness") if err != nil { t.Fatalf("GET /readiness returned an error (%s)", err) } @@ -18,7 +18,7 @@ func TestProbesServer(t *testing.T) { t.Fatalf("GET /readiness returned status code %d, expected %d", resp.StatusCode, http.StatusOK) } - resp, err = http.Get("/liveness") + resp, err = http.Get(srv.URL + "/liveness") if err != nil { t.Fatalf("GET /liveness returned an error (%s)", err) } From 9940222d34b59e8c91782e6ce7e6be6b1d64e0e1 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Wed, 1 Jun 2016 12:18:18 -0700 Subject: [PATCH 4/5] Add probes to the the embedded Tiller manifest --- pkg/client/install.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/client/install.go b/pkg/client/install.go index ae3a4f6ea..4dc5243d3 100644 --- a/pkg/client/install.go +++ b/pkg/client/install.go @@ -107,4 +107,16 @@ spec: - containerPort: 44134 name: tiller imagePullPolicy: Always + livenessProbe: + httpGet: + path: /liveness + port: 44135 + initialDelaySeconds: 1 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /readiness + port: 44135 + initialDelaySeconds: 1 + timeoutSeconds:1 ` From 9e1df63e7701c1630a4e0f8a7179b70e1b21c174 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Fri, 3 Jun 2016 14:48:06 -0700 Subject: [PATCH 5/5] using spaces in the tiller manifest yaml --- pkg/client/install.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/client/install.go b/pkg/client/install.go index 4dc5243d3..0e4ecd9c3 100644 --- a/pkg/client/install.go +++ b/pkg/client/install.go @@ -107,16 +107,16 @@ spec: - containerPort: 44134 name: tiller imagePullPolicy: Always - livenessProbe: - httpGet: - path: /liveness - port: 44135 - initialDelaySeconds: 1 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /readiness - port: 44135 - initialDelaySeconds: 1 - timeoutSeconds:1 + livenessProbe: + httpGet: + path: /liveness + port: 44135 + initialDelaySeconds: 1 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /readiness + port: 44135 + initialDelaySeconds: 1 + timeoutSeconds: 1 `