From dcd804ac7b644ffe2ccdcced59f1841da6cdd865 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 13:16:28 -0800 Subject: [PATCH 1/6] Fix expandybird build. --- dm/dm.go | 2 +- expandybird/Dockerfile | 67 +++++++++++++++++++++++++++++++++++++---- expandybird/Makefile | 16 +++------- manager/Dockerfile | 1 + resourcifier/Dockerfile | 1 + 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index 365b90358..03ee06dbb 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -275,7 +275,7 @@ func callService(path, method, action string, reader io.ReadCloser) { panic(fmt.Errorf("cannot parse url (%s): %s\n", path, err)) } - URL.Path += path + URL.Path = strings.TrimRight(URL.Path, "/") + "/" + strings.TrimLeft(path, "/") resp := callHTTP(URL.String(), method, action, reader) var j interface{} if err := json.Unmarshal([]byte(resp), &j); err != nil { diff --git a/expandybird/Dockerfile b/expandybird/Dockerfile index 6d95f9709..e445fd27b 100644 --- a/expandybird/Dockerfile +++ b/expandybird/Dockerfile @@ -1,16 +1,71 @@ -FROM python:2-onbuild +# Copyright 2015 Google, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2 +MAINTAINER Jack Greenfield RUN ln -s /usr/local/bin/python /usr/bin/python +RUN apt-get update \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# following lines copied from golang:1.4 + +# gcc for cgo +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc \ + libc6-dev \ + make \ + && rm -rf /var/lib/apt/lists/* + +ENV GOLANG_VERSION 1.4.3 +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz +ENV GOLANG_DOWNLOAD_SHA1 486db10dc571a55c8d795365070f66d343458c48 + +RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ + && echo "$GOLANG_DOWNLOAD_SHA1 golang.tar.gz" | sha1sum -c - \ + && tar -C /usr/src -xzf golang.tar.gz \ + && rm golang.tar.gz \ + && cd /usr/src/go/src && ./make.bash --no-clean 2>&1 + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/src/go/bin:$PATH + +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +#end copied lines + +COPY . "$GOPATH"/src/github.com/kubernetes/deployment-manager +WORKDIR "$GOPATH"/src/github.com/kubernetes/deployment-manager/expandybird + +RUN go get -v -d . +RUN go install -v . + RUN mkdir -p /var/expandybird/expansion WORKDIR /var/expandybird -ADD expandybird ./expandybird -ADD expansion ./expansion +COPY ./expandybird/expansion /var/expandybird/expansion + +COPY ./expandybird/requirements.txt /var/expandybird/requirements.txt +RUN pip install --no-cache-dir -r /var/expandybird/requirements.txt -ADD requirements.txt ./requirements.txt -RUN pip install -r ./requirements.txt +RUN cp "$GOPATH"/bin/expandybird /var/expandybird/expandybird +RUN /bin/rm -rf "$GOPATH" EXPOSE 8080 -ENTRYPOINT ["./expandybird", "-expansion_binary", "./expansion/expansion.py"] +ENTRYPOINT ["/var/expandybird/expandybird", "-expansion_binary", "/var/expandybird/expansion/expansion.py"] diff --git a/expandybird/Makefile b/expandybird/Makefile index 8490e7bf3..df97481fd 100644 --- a/expandybird/Makefile +++ b/expandybird/Makefile @@ -14,14 +14,15 @@ include ../include.mk -.PHONY : all build test push container clean +.PHONY : all build push container clean .project DOCKER_REGISTRY := gcr.io PREFIX := $(DOCKER_REGISTRY)/$(PROJECT) IMAGE := expandybird TAG := latest -DIR := . +ROOT_DIR := $(abspath ./..) +DIR = $(ROOT_DIR) push: container ifeq ($(DOCKER_REGISTRY),gcr.io) @@ -30,18 +31,11 @@ else docker push $(PREFIX)/$(IMAGE):$(TAG) endif -container: expandybird - cp $(shell which expandybird) . - docker build -t $(PREFIX)/$(IMAGE):$(TAG) $(DIR) - rm -f expandybird - -expandybird: - go get -v ./... - go install -v ./... +container: + docker build -t $(PREFIX)/$(IMAGE):$(TAG) -f Dockerfile $(DIR) clean: -docker rmi $(PREFIX)/$(IMAGE):$(TAG) - rm -f expandybird .PHONY: test test: lint vet test-unit diff --git a/manager/Dockerfile b/manager/Dockerfile index e2c586a92..34bd5de4f 100644 --- a/manager/Dockerfile +++ b/manager/Dockerfile @@ -33,6 +33,7 @@ WORKDIR "$GOPATH"/src/github.com/kubernetes/deployment-manager/manager RUN go-wrapper download RUN go-wrapper install +WORKDIR /usr/local/bin RUN cp "$GOPATH"/bin/manager /usr/local/bin RUN /bin/rm -rf "$GOPATH" diff --git a/resourcifier/Dockerfile b/resourcifier/Dockerfile index 881c77d7e..6760254c9 100644 --- a/resourcifier/Dockerfile +++ b/resourcifier/Dockerfile @@ -32,6 +32,7 @@ WORKDIR "$GOPATH"/src/github.com/kubernetes/deployment-manager/resourcifier RUN go-wrapper download RUN go-wrapper install +WORKDIR /usr/local/bin RUN cp "$GOPATH"/bin/resourcifier /usr/local/bin RUN /bin/rm -rf "$GOPATH" From 46984eb02a1875f263897a1f804a1380dd2d7045 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 15:22:05 -0800 Subject: [PATCH 2/6] Fix test failure in expandybird. --- expandybird/service/service.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/expandybird/service/service.go b/expandybird/service/service.go index d770db127..239170cb8 100644 --- a/expandybird/service/service.go +++ b/expandybird/service/service.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,8 +17,8 @@ limitations under the License. package service import ( - "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/util" "errors" @@ -44,7 +44,8 @@ func NewService(handler restful.RouteFunction) *Service { webService.Produces(restful.MIME_JSON, restful.MIME_XML) webService.Route(webService.POST("/expand").To(handler). Doc("Expand a template."). - Reads(&common.Template{})) + Reads(&common.Template{}). + Writes(&expander.ExpansionResponse{})) return &Service{webService} } From 808641c7f712d3560fb3433059a3e9062efb14af Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 16:08:06 -0800 Subject: [PATCH 3/6] Log expansion and write accept headers for tests. --- expandybird/service/service.go | 2 ++ manager/manager/expander.go | 2 +- util/httputil.go | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/expandybird/service/service.go b/expandybird/service/service.go index 239170cb8..ac75d47e6 100644 --- a/expandybird/service/service.go +++ b/expandybird/service/service.go @@ -86,6 +86,8 @@ func NewExpansionHandler(backend expander.Expander) restful.RouteFunction { } util.LogHandlerExit("expandybird", http.StatusOK, "OK", resp.ResponseWriter) + message := fmt.Sprintf("\nConfig:\n%s\nLayout:\n%s\n", response.Config, response.Layout) + util.LogHandlerText("expandybird", message) resp.WriteEntity(response) } } diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 49557ac39..333804dab 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -192,7 +192,7 @@ func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) response, err := http.Post(e.getBaseURL(), "application/json", ioutil.NopCloser(bytes.NewReader(j))) if err != nil { - e := fmt.Errorf("http POST failed: %s", err) + e := fmt.Errorf("call failed (%s) with payload:\n%s\n", e.getBaseURL(), err, string(j)) return nil, e } diff --git a/util/httputil.go b/util/httputil.go index 540bc86a2..cba3988a7 100644 --- a/util/httputil.go +++ b/util/httputil.go @@ -64,6 +64,7 @@ func NewHandlerTester(handler http.Handler) HandlerTester { } r.Header.Set("Content-Type", ctype) + r.Header.Set("Accept", "*/*") w := httptest.NewRecorder() handler.ServeHTTP(w, r) return w, nil @@ -87,6 +88,7 @@ func NewServerTester(handler http.Handler) ServerTester { } r.Header.Set("Content-Type", ctype) + r.Header.Set("Accept", "*/*") return http.DefaultClient.Do(r) } } @@ -112,6 +114,10 @@ func TestHandlerWithURL(handler http.Handler, method, urlString string) (*httpte return NewHandlerTester(handler).TestWithURL(method, urlString) } +func LogHandlerText(handler string, v string) { + log.Printf("%s: %s\n", handler, v) +} + // LogHandlerEntry logs the start of the given handler handling the given request. func LogHandlerEntry(handler string, r *http.Request) { log.Printf("%s: handling request:%s %s\n", handler, r.Method, r.URL.RequestURI()) From 64aea8540329855a0dece159afc653f5f0375e5c Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 18:24:13 -0800 Subject: [PATCH 4/6] Move to golang:1.6 --- dm/Dockerfile | 2 +- expandybird/Dockerfile | 18 ++++++++---------- manager/Dockerfile | 2 +- manager/manager/expander.go | 2 +- resourcifier/Dockerfile | 2 +- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/dm/Dockerfile b/dm/Dockerfile index 75f49814c..8590cd65a 100644 --- a/dm/Dockerfile +++ b/dm/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.4-cross +FROM golang:1.6 ENV GOLANG_CROSSPLATFORMS \ darwin/amd64 \ diff --git a/expandybird/Dockerfile b/expandybird/Dockerfile index e445fd27b..f671ea7f4 100644 --- a/expandybird/Dockerfile +++ b/expandybird/Dockerfile @@ -22,9 +22,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# following lines copied from golang:1.4 +# following lines copied from golang:1.6 -# gcc for cgo RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ gcc \ @@ -32,18 +31,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ make \ && rm -rf /var/lib/apt/lists/* -ENV GOLANG_VERSION 1.4.3 -ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz -ENV GOLANG_DOWNLOAD_SHA1 486db10dc571a55c8d795365070f66d343458c48 +ENV GOLANG_VERSION 1.6 +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz +ENV GOLANG_DOWNLOAD_SHA256 5470eac05d273c74ff8bac7bef5bad0b5abbd1c4052efbdbc8db45332e836b0b RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ - && echo "$GOLANG_DOWNLOAD_SHA1 golang.tar.gz" | sha1sum -c - \ - && tar -C /usr/src -xzf golang.tar.gz \ - && rm golang.tar.gz \ - && cd /usr/src/go/src && ./make.bash --no-clean 2>&1 + && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ + && tar -C /usr/local -xzf golang.tar.gz \ + && rm golang.tar.gz ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/src/go/bin:$PATH +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" diff --git a/manager/Dockerfile b/manager/Dockerfile index 34bd5de4f..5e62c9358 100644 --- a/manager/Dockerfile +++ b/manager/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.4 +FROM golang:1.6 MAINTAINER Jack Greenfield RUN apt-get update \ diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 333804dab..e61ec2b82 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -192,7 +192,7 @@ func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) response, err := http.Post(e.getBaseURL(), "application/json", ioutil.NopCloser(bytes.NewReader(j))) if err != nil { - e := fmt.Errorf("call failed (%s) with payload:\n%s\n", e.getBaseURL(), err, string(j)) + e := fmt.Errorf("call failed (%s) with payload:\n%s\n", err, string(j)) return nil, e } diff --git a/resourcifier/Dockerfile b/resourcifier/Dockerfile index 6760254c9..c02984055 100644 --- a/resourcifier/Dockerfile +++ b/resourcifier/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.4 +FROM golang:1.6 RUN apt-get update \ && apt-get autoremove -y \ From af0bdc512f8be327a574ca9dfd57c846e57e6692 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 19:52:03 -0800 Subject: [PATCH 5/6] Fix TLS mismatch issue. --- manager/deployments.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manager/deployments.go b/manager/deployments.go index 193f718fa..37de85909 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -113,7 +113,7 @@ func newManager(cp common.CredentialProvider) manager.Manager { resolver := manager.NewTypeResolver(registryProvider, util.DefaultHTTPClient()) expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName, expanderPort), resolver) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName, deployerPort)) - address := strings.TrimPrefix(getServiceURL(*mongoAddress, *mongoName, *mongoPort), "https://") + address := strings.TrimPrefix(getServiceURL(*mongoAddress, *mongoName, *mongoPort), "http://") repository := createRepository(address) return manager.NewManager(expander, deployer, repository, registryProvider, service, cp) } @@ -136,7 +136,7 @@ func getServiceURL(serviceURL, serviceName, servicePort string) string { log.Fatalf("cannot resolve service:%v. environment:%v\n", serviceName, os.Environ()) } - serviceURL = fmt.Sprintf("https://%s:%s", addrs[0], servicePort) + serviceURL = fmt.Sprintf("http://%s:%s", addrs[0], servicePort) } } @@ -149,7 +149,7 @@ func getServiceURL(serviceURL, serviceName, servicePort string) string { func makeEnvVariableURL(str string) string { prefix := makeEnvVariableName(str) url := os.Getenv(prefix + "_PORT") - return strings.Replace(url, "tcp", "https", 1) + return strings.Replace(url, "tcp", "http", 1) } // makeEnvVariableName is copied from the Kubernetes source, From 5ca0db7d9f773f415a28334b9a1ccd5c2b05caee Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 22 Feb 2016 20:49:56 -0800 Subject: [PATCH 6/6] Set expandybird client headers. --- manager/manager/expander.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/manager/manager/expander.go b/manager/manager/expander.go index e61ec2b82..f41406cfe 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -190,7 +190,16 @@ func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) return nil, err } - response, err := http.Post(e.getBaseURL(), "application/json", ioutil.NopCloser(bytes.NewReader(j))) + reader := ioutil.NopCloser(bytes.NewReader(j)) + request, err := http.NewRequest("POST", e.getBaseURL(), reader) + if err != nil { + return nil, err + } + + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "*/*") + + response, err := http.DefaultClient.Do(request) if err != nil { e := fmt.Errorf("call failed (%s) with payload:\n%s\n", err, string(j)) return nil, e