From edd4e9dcba5048528ba7427e2a6f66782928ee00 Mon Sep 17 00:00:00 2001 From: jackgr Date: Sun, 14 Feb 2016 11:58:27 -0800 Subject: [PATCH 1/4] Escape deployment and manifest names. --- dm/dm.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index 6f69fac4f..c1e25a5c0 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -206,26 +206,19 @@ func execute() { os.Exit(1) } - path := fmt.Sprintf("deployments/%s", args[1]) + path := fmt.Sprintf("deployments/%s", url.QueryEscape(args[1])) action := fmt.Sprintf("get deployment named %s", args[1]) callService(path, "GET", action, nil) case "manifest": - msg := "Must specify manifest in the form / or to list." - if len(args) < 2 { + msg := "Must specify manifest in the form or just to list." + if len(args) < 2 || len(args) > 3 { fmt.Fprintln(os.Stderr, msg) os.Exit(1) } - s := strings.Split(args[1], "/") - ls := len(s) - if ls < 1 || ls > 2 { - fmt.Fprintln(os.Stderr, fmt.Sprintf("Invalid manifest (%s), %s", args[1], msg)) - os.Exit(1) - } - - path := fmt.Sprintf("deployments/%s/manifests", s[0]) - if ls == 2 { - path = path + fmt.Sprintf("/%s", s[1]) + path := fmt.Sprintf("deployments/%s/manifests", url.QueryEscape(args[1])) + if len(args) > 2 { + path = path + fmt.Sprintf("/%s", url.QueryEscape(args[2])) } action := fmt.Sprintf("get manifest %s", args[1]) @@ -236,12 +229,12 @@ func execute() { os.Exit(1) } - path := fmt.Sprintf("deployments/%s", args[1]) + path := fmt.Sprintf("deployments/%s", url.QueryEscape(args[1])) action := fmt.Sprintf("delete deployment named %s", args[1]) callService(path, "DELETE", action, nil) case "update": template := loadTemplate(args) - path := fmt.Sprintf("deployments/%s", template.Name) + path := fmt.Sprintf("deployments/%s", url.QueryEscape(template.Name)) action := fmt.Sprintf("delete deployment named %s", template.Name) callService(path, "PUT", action, marshalTemplate(template)) case "deployed-types": @@ -274,9 +267,14 @@ func execute() { } func callService(path, method, action string, reader io.ReadCloser) { - u := fmt.Sprintf("%s/%s", *service, path) + var URL *url.URL + URL, err := url.Parse(*service) + if err != nil { + panic(fmt.Errorf("cannot parse url (%s): %s\n", path, err)) + } - resp := callHTTP(u, method, action, reader) + URL.Path += path + resp := callHTTP(URL.String(), method, action, reader) var j interface{} if err := json.Unmarshal([]byte(resp), &j); err != nil { panic(fmt.Errorf("Failed to parse JSON response from service: %s", resp)) From f14c53980fa255d53da76ba64e8c04329359442a Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 15 Feb 2016 11:52:43 -0800 Subject: [PATCH 2/4] Remove unneccessary download url retrieval. --- dm/dm.go | 28 ++++--------------------- expandybird/expander/expander.go | 24 ++++++++++++++++++++- expandybird/expander/expander_test.go | 30 ++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index c1e25a5c0..ee7c2954b 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -245,15 +245,7 @@ func execute() { os.Exit(1) } - tUrls := getDownloadURLs(args[1]) - var tURL = "" - if len(tUrls) == 0 { - // Type is most likely a primitive. - tURL = args[1] - } else { - // TODO(vaikas): Support packages properly. - tURL = tUrls[0] - } + tURL := args[1] path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tURL)) action := fmt.Sprintf("list deployed instances of type %s", tURL) callService(path, "GET", action, nil) @@ -431,24 +423,12 @@ func buildTemplateFromType(t string) *common.Template { } // Name the deployment after the type name. - name := t - - config := common.Configuration{Resources: []*common.Resource{&common.Resource{ - Name: name, - Type: getDownloadURLs(t)[0], - Properties: props, - }}} - - y, err := yaml.Marshal(config) + template, err := expander.NewTemplateFromType(t, t, props) if err != nil { - panic(fmt.Errorf("error: %s\ncannot create configuration for deployment: %v\n", err, config)) + panic(fmt.Errorf("cannot create configuration from type (%s): %s\n", t, err)) } - return &common.Template{ - Name: name, - Content: string(y), - // No imports, as this is a single type from repository. - } + return template } func marshalTemplate(template *common.Template) io.ReadCloser { diff --git a/expandybird/expander/expander.go b/expandybird/expander/expander.go index 758a71474..86ad3bf6b 100644 --- a/expandybird/expander/expander.go +++ b/expandybird/expander/expander.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,6 +46,28 @@ func NewExpander(binary string) Expander { return &expander{binary} } +func NewTemplateFromType(name, typeName string, properties map[string]interface{}) (*common.Template, error) { + resource := &common.Resource{ + Name: name, + Type: typeName, + Properties: properties, + } + + config := common.Configuration{Resources: []*common.Resource{resource}} + content, err := yaml.Marshal(config) + if err != nil { + return nil, fmt.Errorf("error: %s\ncannot marshal configuration: %v\n", err, config) + } + + template := &common.Template{ + Name: name, + Content: string(content), + Imports: []*common.ImportFile{}, + } + + return template, nil +} + // NewTemplateFromArchive creates and returns a new template whose content // and imported files are read from the supplied archive. func NewTemplateFromArchive(name string, r io.Reader, importFileNames []string) (*common.Template, error) { diff --git a/expandybird/expander/expander_test.go b/expandybird/expander/expander_test.go index 64aef8aef..9c4677f78 100644 --- a/expandybird/expander/expander_test.go +++ b/expandybird/expander/expander_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,6 +28,7 @@ import ( "strings" "testing" + "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/common" ) @@ -107,6 +108,33 @@ func testExpandTemplateFromFile(t *testing.T, fileName, baseName string, importF expandAndVerifyOutput(t, actualOutput, description) } +var ( + testTemplateName = "expandybird" + testTemplateType = "replicatedservice.py" + testTemplateProperties = ` +service_port: 8080 +target_port: 8080 +container_port: 8080 +external_service: true +replicas: 3 +image: gcr.io/dm-k8s-testing/expandybird +labels: + app: expandybird +` +) + +func TestNewTemplateFromType(t *testing.T) { + var properties map[string]interface{} + if err := yaml.Unmarshal([]byte(testTemplateProperties), &properties); err != nil { + t.Fatalf("cannot unmarshal test data: %s", err) + } + + _, err := NewTemplateFromType(testTemplateName, testTemplateType, properties) + if err != nil { + t.Fatalf("cannot create template from type %s: %s", testTemplateType, err) + } +} + func TestNewTemplateFromReader(t *testing.T) { r := bytes.NewReader([]byte{}) if _, err := NewTemplateFromReader("test", r, nil); err == nil { From 440af2fabd2f9c075c1eaf2fca1bf719d434ce74 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 15 Feb 2016 17:16:15 -0800 Subject: [PATCH 3/4] Move metadata retrieval to server. --- dm/dm.go | 44 +++++--------------------------- expandybird/expander/expander.go | 2 ++ manager/deployments.go | 36 ++++++++++++++++++++++++++ manager/manager/manager.go | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 37 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index ee7c2954b..bbe343221 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -168,7 +168,13 @@ func execute() { path := fmt.Sprintf("registries/%s/types", *templateRegistry) callService(path, "GET", "list templates", nil) case "describe": - describeType(args) + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "No type name or URL supplied") + os.Exit(1) + } + + path := fmt.Sprintf("types/%s/metadata", url.QueryEscape(args[1])) + callService(path, "GET", "get metadata for type", nil) case "expand": template := loadTemplate(args) callService("expand", "POST", "expand configuration", marshalTemplate(template)) @@ -308,42 +314,6 @@ func callHTTP(path, method, action string, reader io.ReadCloser) string { return string(body) } -// describeType prints the schema for a type specified by either a -// template URL or a fully qualified registry type name (e.g., -// :) -func describeType(args []string) { - if len(args) != 2 { - fmt.Fprintln(os.Stderr, "No type name or URL supplied") - os.Exit(1) - } - - tUrls := getDownloadURLs(url.QueryEscape(args[1])) - if len(tUrls) == 0 { - panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \":\": %s", args[1])) - } - - if !strings.Contains(tUrls[0], ".prov") { - // It's not a chart, so grab the schema - path := fmt.Sprintf("registries/%s/download?file=%s.schema", *templateRegistry, url.QueryEscape(tUrls[0])) - callService(path, "GET", "get schema for type ("+tUrls[0]+")", nil) - } else { - // It's a chart, so grab the provenance file - path := fmt.Sprintf("registries/%s/download?file=%s", *templateRegistry, url.QueryEscape(tUrls[0])) - callService(path, "GET", "get file", nil) - } -} - -// getDownloadURLs returns URLs for a type in the given registry -func getDownloadURLs(tName string) []string { - path := fmt.Sprintf("%s/registries/%s/types/%s", *service, *templateRegistry, url.QueryEscape(tName)) - resp := callHTTP(path, "GET", "get download urls", nil) - u := []string{} - if err := json.Unmarshal([]byte(resp), &u); err != nil { - panic(fmt.Errorf("Failed to parse JSON response from service: %s", resp)) - } - return u -} - func loadTemplate(args []string) *common.Template { var template *common.Template var err error diff --git a/expandybird/expander/expander.go b/expandybird/expander/expander.go index 86ad3bf6b..0dfb6ac7d 100644 --- a/expandybird/expander/expander.go +++ b/expandybird/expander/expander.go @@ -46,6 +46,8 @@ func NewExpander(binary string) Expander { return &expander{binary} } +// NewTemplateFromType creates and returns a new template whose content +// is a YAML marshaled resource assembled from the supplied arguments. func NewTemplateFromType(name, typeName string, properties map[string]interface{}) (*common.Template, error) { resource := &common.Resource{ Name: name, diff --git a/manager/deployments.go b/manager/deployments.go index 8134c8e63..421693af9 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -53,6 +53,8 @@ var deployments = []Route{ {"Expand", "/expand", "POST", expandHandlerFunc, ""}, {"ListTypes", "/types", "GET", listTypesHandlerFunc, ""}, {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, + {"GetRegistryForType", "/types/{type}/registry", "GET", getRegistryForTypeHandlerFunc, ""}, + {"GetMetadataForType", "/types/{type}/metadata", "GET", getMetadataForTypeHandlerFunc, ""}, {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, {"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""}, {"CreateRegistry", "/registries/{registry}", "POST", createRegistryHandlerFunc, "JSON"}, @@ -378,6 +380,40 @@ func listTypeInstancesHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, instances, http.StatusOK) } +func getRegistryForTypeHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get type registry" + util.LogHandlerEntry(handler, r) + typeName, err := getPathVariable(w, r, "type", handler) + if err != nil { + return + } + + registry, err := backend.GetRegistryForType(typeName) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, registry, http.StatusOK) +} + +func getMetadataForTypeHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get type metadata" + util.LogHandlerEntry(handler, r) + typeName, err := getPathVariable(w, r, "type", handler) + if err != nil { + return + } + + metadata, err := backend.GetMetadataForType(typeName) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, metadata, http.StatusOK) +} + // Putting Registry handlers here for now because deployments.go // currently owns its own Manager backend and doesn't like to share. func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) { diff --git a/manager/manager/manager.go b/manager/manager/manager.go index 1245445aa..801b09fab 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -21,6 +21,7 @@ import ( "log" "net/url" "regexp" + "strings" "time" "github.com/kubernetes/deployment-manager/common" @@ -46,6 +47,8 @@ type Manager interface { // Types ListTypes() ([]string, error) ListInstances(typeName string) ([]*common.TypeInstance, error) + GetRegistryForType(typeName string) (string, error) + GetMetadataForType(typeName string) (string, error) // Registries ListRegistries() ([]*common.Registry, error) @@ -334,6 +337,42 @@ func (m *manager) ListInstances(typeName string) ([]*common.TypeInstance, error) return m.repository.GetTypeInstances(typeName) } +// GetRegistryForType returns the registry where a type resides. +func (m *manager) GetRegistryForType(typeName string) (string, error) { + _, r, err := registry.GetDownloadURLs(m.registryProvider, typeName) + if err != nil { + return "", err + } + + return r.GetRegistryName(), nil +} + +// GetMetadataForType returns the metadata for type. +func (m *manager) GetMetadataForType(typeName string) (string, error) { + URLs, r, err := registry.GetDownloadURLs(m.registryProvider, typeName) + if err != nil { + return "", err + } + + if len(URLs) < 1 { + return "", nil + } + + // If it's a chart, we want the provenance file + fPath := URLs[0] + if !strings.Contains(fPath, ".prov") { + // It's not a chart, so we want the schema + fPath += ".schema" + } + + metadata, err := getFileFromRegistry(fPath, r) + if err != nil { + return "", fmt.Errorf("cannot get metadata for type (%s): %s", typeName, err) + } + + return metadata, nil +} + // ListRegistries returns the list of registries func (m *manager) ListRegistries() ([]*common.Registry, error) { return m.service.List() @@ -403,11 +442,16 @@ func (m *manager) GetFile(registryName string, url string) (string, error) { return "", err } + return getFileFromRegistry(url, r) +} + +func getFileFromRegistry(url string, r registry.Registry) (string, error) { getter := util.NewHTTPClient(3, r, util.NewSleeper()) body, _, err := getter.Get(url) if err != nil { return "", err } + return body, nil } From 2cdb34fd96f980e06836da2b13bbb7053c3aad3b Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 15 Feb 2016 18:40:40 -0800 Subject: [PATCH 4/4] Clean up registry types. --- dm/dm.go | 4 ++++ manager/deployments.go | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index bbe343221..3e0534197 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -166,6 +166,10 @@ func execute() { switch args[0] { case "templates": path := fmt.Sprintf("registries/%s/types", *templateRegistry) + if *regexString != "" { + path += fmt.Sprintf("?%s", url.QueryEscape(*regexString)) + } + callService(path, "GET", "list templates", nil) case "describe": if len(args) != 2 { diff --git a/manager/deployments.go b/manager/deployments.go index 421693af9..193f718fa 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -495,12 +495,18 @@ func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) { return } + values, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + var regex *regexp.Regexp - regexString, err := getPathVariable(w, r, "regex", handler) - if err == nil { + regexString := values.Get("regex") + if regexString != "" { regex, err = regexp.Compile(regexString) if err != nil { - util.LogAndReturnError(handler, http.StatusInternalServerError, err, w) + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) return } }