From 8ba0eb4f4df657656c814e775f82786fa0d635d0 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Wed, 9 Dec 2015 01:15:25 -0800 Subject: [PATCH 01/11] Add support for helm packages. Quite hacky in parts, needs to resolve naming collisions --- dm/dm.go | 1 - examples/package/package.yaml | 4 ++++ expandybird/expansion/expansion.py | 10 +++++----- manager/manager/typeresolver.go | 5 +++++ registry/github_package_registry.go | 1 - 5 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 examples/package/package.yaml diff --git a/dm/dm.go b/dm/dm.go index 1c24f3e7a..85051741a 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -214,7 +214,6 @@ func execute() { // Type is most likely a primitive. tUrl = args[1] } else { - // TODO(vaikas): Support packages properly. tUrl = tUrls[0] } path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl)) diff --git a/examples/package/package.yaml b/examples/package/package.yaml new file mode 100644 index 000000000..70c08b6f6 --- /dev/null +++ b/examples/package/package.yaml @@ -0,0 +1,4 @@ +resources: +- name: cassandra + type: github.com/helm/charts/cassandra + properties: null diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index ef012f90d..849fa681a 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -50,12 +50,12 @@ def Expand(config, imports=None, env=None, validate_schema=False): Raises: ExpansionError: if there is any error occurred during expansion """ - try: - return _Expand(config, imports=imports, env=env, - validate_schema=validate_schema) - except Exception as e: +# try: + return _Expand(config, imports=imports, env=env, + validate_schema=validate_schema) +# except Exception as e: # print traceback.format_exc() - raise ExpansionError('config', str(e)) +# raise ExpansionError('config', str(e)) def _Expand(config, imports=None, env=None, validate_schema=False): diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index fbc291050..165cf2bfa 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -18,6 +18,7 @@ package manager import ( "fmt" + "log" "net/http" "time" @@ -68,6 +69,7 @@ func resolverError(c *common.Configuration, err error) error { } func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) { + log.Printf("Fetching %s", u) r, code, err := g.Get(u) if err != nil { return "", err @@ -99,6 +101,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co // TODO(vaikas): Need to account for multiple URLs being fetched for a given type. toFetch := make([]*fetchUnit, 0, tr.maxUrls) for _, r := range config.Resources { + log.Printf("checking: %s", r.Type) // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). urls, err := tr.MapFetchableURLs(r.Type) if err != nil { @@ -121,7 +124,9 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co } count := 0 + log.Printf("toFetch %#v", toFetch) for len(toFetch) > 0 { + log.Printf("toFetch2 %#v", toFetch) // 1. If short github URL, resolve to a download URL // 2. Fetch import URL. Exit if no URLs left // 3. Check/handle HTTP status diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 096c3c23a..664a7e096 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -48,7 +48,6 @@ func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry { } // List the types from the Registry. -// TODO(vaikas): Figure out how the versions work here. func (g *GithubPackageRegistry) List() ([]Type, error) { // Just list all the types at the top level. types, err := g.getDirs("") From e1cb0ede863e056082d2b9c82a9310b15ea1613e Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Wed, 9 Dec 2015 11:18:17 -0800 Subject: [PATCH 02/11] handle fetching number of urls as a unit that get pooled into a single template so that helm works out of the box --- manager/manager/expander.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 95875ea14..47316a4ff 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "log" "net/http" "github.com/ghodss/yaml" @@ -140,6 +141,8 @@ func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) t.Imports = append(t.Imports, newImp...) + log.Printf("Got Imports as %#v", t.Imports) + for { // Now expand with everything imported. result, err := e.expandTemplate(t) @@ -166,6 +169,7 @@ func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) // If the new imports contain nothing, we are done. Everything is fully expanded. if len(newImp) == 0 { result.Layout = finalLayout + log.Printf("Returning config as %#v", result) return result, nil } From c3831f021ac136df8a6523a214dae84e7a4195fd Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Sun, 13 Dec 2015 16:31:46 -0800 Subject: [PATCH 03/11] debug stuff --- manager/manager/typeresolver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 165cf2bfa..e8f3b5b06 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -125,6 +125,9 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co count := 0 log.Printf("toFetch %#v", toFetch) + for _, jj := range toFetch { + log.Printf("TOFETCH UNIT: %#v", jj) + } for len(toFetch) > 0 { log.Printf("toFetch2 %#v", toFetch) // 1. If short github URL, resolve to a download URL From 125bad99cfbb0c56d23241dbe429ea64297ace31 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Sun, 13 Dec 2015 21:24:41 -0800 Subject: [PATCH 04/11] unit tests pass, can deploy packages --- expandybird/expansion/expansion.py | 10 +++++----- manager/manager/typeresolver.go | 8 -------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index 849fa681a..ef012f90d 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -50,12 +50,12 @@ def Expand(config, imports=None, env=None, validate_schema=False): Raises: ExpansionError: if there is any error occurred during expansion """ -# try: - return _Expand(config, imports=imports, env=env, - validate_schema=validate_schema) -# except Exception as e: + try: + return _Expand(config, imports=imports, env=env, + validate_schema=validate_schema) + except Exception as e: # print traceback.format_exc() -# raise ExpansionError('config', str(e)) + raise ExpansionError('config', str(e)) def _Expand(config, imports=None, env=None, validate_schema=False): diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index e8f3b5b06..fbc291050 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -18,7 +18,6 @@ package manager import ( "fmt" - "log" "net/http" "time" @@ -69,7 +68,6 @@ func resolverError(c *common.Configuration, err error) error { } func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) { - log.Printf("Fetching %s", u) r, code, err := g.Get(u) if err != nil { return "", err @@ -101,7 +99,6 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co // TODO(vaikas): Need to account for multiple URLs being fetched for a given type. toFetch := make([]*fetchUnit, 0, tr.maxUrls) for _, r := range config.Resources { - log.Printf("checking: %s", r.Type) // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). urls, err := tr.MapFetchableURLs(r.Type) if err != nil { @@ -124,12 +121,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co } count := 0 - log.Printf("toFetch %#v", toFetch) - for _, jj := range toFetch { - log.Printf("TOFETCH UNIT: %#v", jj) - } for len(toFetch) > 0 { - log.Printf("toFetch2 %#v", toFetch) // 1. If short github URL, resolve to a download URL // 2. Fetch import URL. Exit if no URLs left // 3. Check/handle HTTP status From 93f74d0c8aefe3ff269fb65697751875fc41bfc1 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Tue, 15 Dec 2015 11:26:37 -0800 Subject: [PATCH 05/11] bring back to par with master --- dm/dm.go | 1 + examples/package/package.yaml | 4 ---- manager/manager/expander.go | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 examples/package/package.yaml diff --git a/dm/dm.go b/dm/dm.go index 85051741a..1c24f3e7a 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -214,6 +214,7 @@ func execute() { // Type is most likely a primitive. tUrl = args[1] } else { + // TODO(vaikas): Support packages properly. tUrl = tUrls[0] } path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl)) diff --git a/examples/package/package.yaml b/examples/package/package.yaml deleted file mode 100644 index 70c08b6f6..000000000 --- a/examples/package/package.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- name: cassandra - type: github.com/helm/charts/cassandra - properties: null diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 47316a4ff..95875ea14 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" "net/http" "github.com/ghodss/yaml" @@ -141,8 +140,6 @@ func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) t.Imports = append(t.Imports, newImp...) - log.Printf("Got Imports as %#v", t.Imports) - for { // Now expand with everything imported. result, err := e.expandTemplate(t) @@ -169,7 +166,6 @@ func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) // If the new imports contain nothing, we are done. Everything is fully expanded. if len(newImp) == 0 { result.Layout = finalLayout - log.Printf("Returning config as %#v", result) return result, nil } From 616a8c387719848530bb4413b029ab0db2ae0d5c Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Wed, 6 Jan 2016 16:33:55 -0800 Subject: [PATCH 06/11] First cut of unifying the registries, just puts the scaffolding in place --- common/types.go | 36 +++++++++++++- dm/dm.go | 27 +++++------ manager/manager/typeresolver.go | 14 ++++-- manager/manager/typeresolver_test.go | 12 ++--- registry/github_package_registry.go | 6 +-- registry/github_registry.go | 6 +-- registry/inmem_repository_service.go | 70 ++++++++++++++++++++++++++++ registry/registry.go | 19 +++++++- registry/registryprovider.go | 49 +++++++++++++++---- util/templateutil.go | 2 +- 10 files changed, 197 insertions(+), 44 deletions(-) create mode 100644 registry/inmem_repository_service.go diff --git a/common/types.go b/common/types.go index 2f07783a7..72681820d 100644 --- a/common/types.go +++ b/common/types.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -166,3 +166,37 @@ type KubernetesObject struct { Metadata map[string]interface{} `json:"metadata"` Spec map[string]interface{} `json:"spec"` } + +// Repository related types +type BasicAuthCredential struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// Credentials used to access the repository +type RegistryCredential struct { + BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` +} + +type Registry struct { + Name string `json:"name,omitempty"` // Friendly name for the repo + Type RegistryType `json:"type,omitempty"` // Technology implementing the registry + URL string `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts + Credential RegistryCredential `json:"credential,omitempty"` + Format RegistryFormat `json:"format,omitempty"` +} + +// RegistryType defines the technology that implements the registry +type RegistryType string + +const ( + Github RegistryType = "github" +) + +// RegistryFormat defines the format of the registry +type RegistryFormat string + +const ( + VersionedRegistry RegistryFormat = "versioned" + UnversionedRegistry RegistryFormat = "unversioned" +) diff --git a/dm/dm.go b/dm/dm.go index 1c24f3e7a..9a9a0d22b 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -81,22 +81,13 @@ var usage = func() { panic("\n") } -func getGitRegistry() registry.Registry { +func getGitRegistry() (registry.Registry, error) { + rs := registry.NewDefaultRegistryProvider() s := strings.Split(*template_registry, "/") if len(s) < 2 { panic(fmt.Errorf("invalid template registry: %s", *template_registry)) } - - var path = "" - if len(s) > 2 { - path = strings.Join(s[2:], "/") - } - - if s[0] == "helm" { - return registry.NewGithubPackageRegistry(s[0], s[1]) - } else { - return registry.NewGithubRegistry(s[0], s[1], path) - } + return rs.GetRegistry("github.com/" + s[0] + "/" + s[1]) } func main() { @@ -121,7 +112,10 @@ func execute() { switch args[0] { case "templates": - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Cannot get registry %v", err)) + } templates, err := git.List() if err != nil { panic(fmt.Errorf("Cannot list %v", err)) @@ -305,7 +299,10 @@ func getTypeURLs(tName string) []string { } func getDownloadURLs(t registry.Type) []string { - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Failed to get registry")) + } urls, err := git.GetURLs(t) if err != nil { panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index fbc291050..2d7fcb5b1 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ func NewTypeResolver() TypeResolver { client.Timeout = timeout ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.maxUrls = maxURLImports - ret.rp = ®istry.DefaultRegistryProvider{} + ret.rp = registry.NewDefaultRegistryProvider() return ret } @@ -244,7 +244,10 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro if len(m) != 6 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + if err != nil { + return []string{}, err + } t := registry.Type{m[3], m[4], m[5]} return r.GetURLs(t) } @@ -259,7 +262,10 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin if len(m) != 4 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubPackageRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + if err != nil { + return []string{}, err + } t := registry.Type{Name: m[3]} return r.GetURLs(t) } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index b2e9f78ee..a17b03ca4 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,16 +71,12 @@ type testRegistryProvider struct { func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { r := make(map[string]registry.Registry) - r[owner+repository] = &testGithubRegistry{tests, count} + r["github.com/"+owner+"/"+repository] = &testGithubRegistry{tests, count} return &testRegistryProvider{owner, repository, r} } -func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] -} - -func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] +func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { + return trp.r[URL], nil } type testGithubRegistry struct { diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 664a7e096..ba8c903a2 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,11 +39,11 @@ type GithubPackageRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry { +func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry { return &GithubPackageRegistry{ owner: owner, repository: repository, - client: github.NewClient(nil), + client: client, } } diff --git a/registry/github_registry.go b/registry/github_registry.go index 0f06dd3ea..c3cf70f01 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,12 +55,12 @@ type GithubRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubRegistry(owner, repository, path string) *GithubRegistry { +func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry { return &GithubRegistry{ owner: owner, repository: repository, path: path, - client: github.NewClient(nil), + client: client, } } diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go new file mode 100644 index 000000000..714e570f6 --- /dev/null +++ b/registry/inmem_repository_service.go @@ -0,0 +1,70 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "strings" + + "github.com/kubernetes/deployment-manager/common" +) + +type inmemRepositoryService struct { + repositories map[string]*common.Registry +} + +func NewInmemRepositoryService() RegistryService { + return &inmemRepositoryService{ + repositories: make(map[string]*common.Registry), + } +} + +func (rs *inmemRepositoryService) List() ([]*common.Registry, error) { + return nil, nil +} + +func (rs *inmemRepositoryService) Create(repository *common.Registry) error { + rs.repositories[repository.URL] = repository + return nil +} + +func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) { + return &common.Registry{}, nil +} + +func (rs *inmemRepositoryService) Delete(name string) error { + return nil +} +func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { + if !strings.HasPrefix(URL, "github.com/") { + return nil, fmt.Errorf("Failed to parse short github url: %s", URL) + } + s := strings.Split(URL, "/") + if len(s) < 3 { + panic(fmt.Errorf("invalid template registry: %s", URL)) + } + + toFind := "github.com/" + s[1] + "/" + s[2] + fmt.Printf("toFind: %s", toFind) + for _, r := range rs.repositories { + fmt.Printf("Checking: %s", r.URL) + if r.URL == toFind { + return r, nil + } + } + return nil, fmt.Errorf("Failed to find registry for github url: %s", URL) +} diff --git a/registry/registry.go b/registry/registry.go index e53844e74..8e323fe4e 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +16,23 @@ limitations under the License. package registry +import ( + "github.com/kubernetes/deployment-manager/common" +) + +type RegistryService interface { + // List all the registries + List() ([]*common.Registry, error) + // Create a new registry + Create(repository *common.Registry) error + // Get a registry + Get(name string) (*common.Registry, error) + // Delete a registry + Delete(name string) error + // Find a registry that backs the given URL + GetByURL(URL string) (*common.Registry, error) +} + // Registry abstracts a registry that holds templates, which can be // used in a Deployment Manager configurations. There can be multiple // implementations of a registry. Currently we support Deployment Manager diff --git a/registry/registryprovider.go b/registry/registryprovider.go index d4454451c..03e4f5129 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,19 +16,52 @@ limitations under the License. package registry +import ( + "fmt" + + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" +) + // RegistryProvider returns factories for creating registries for a given RegistryType. type RegistryProvider interface { - GetGithubRegistry(owner string, repository string) Registry - GetGithubPackageRegistry(owner string, repository string) Registry + GetRegistry(prefix string) (Registry, error) } -type DefaultRegistryProvider struct { +func NewDefaultRegistryProvider() RegistryProvider { + rs := NewInmemRepositoryService() + rs.Create(&common.Registry{ + Name: "charts", + Type: common.Github, + URL: "github.com/helm/charts", + Format: common.UnversionedRegistry, + }) + rs.Create(&common.Registry{ + Name: "application-dm-templates", + Type: common.Github, + URL: "github.com/kubernetes/application-dm-templates", + Format: common.VersionedRegistry, + }) + return &DefaultRegistryProvider{rs: rs} + } -func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { - return NewGithubRegistry(owner, repository, "") +type DefaultRegistryProvider struct { + rs RegistryService } -func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry { - return NewGithubPackageRegistry(owner, repository) +func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) { + r, err := drp.rs.GetByURL(URL) + if err != nil { + return nil, err + } + if r.Type == common.Github { + if r.Format == common.UnversionedRegistry { + return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil + } + if r.Format == common.VersionedRegistry { + return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil + } + } + return nil, fmt.Errorf("cannot find registry backing url %s", URL) } diff --git a/util/templateutil.go b/util/templateutil.go index 40c9419d8..3ce844c5d 100644 --- a/util/templateutil.go +++ b/util/templateutil.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From f3fd7b14dcf7eda81a71284e1f4c1b2ef8deaf48 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Wed, 6 Jan 2016 17:01:02 -0800 Subject: [PATCH 07/11] wire in the in memory registry provider so that we can show how it works --- manager/deployments.go | 21 ++++++++++++++--- manager/manager/manager.go | 35 +++++++++++++++++++++++----- registry/inmem_repository_service.go | 21 +++++++++++++++-- registry/registryprovider.go | 12 ---------- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/manager/deployments.go b/manager/deployments.go index dceb966c1..460020ba5 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,9 +31,10 @@ import ( "github.com/ghodss/yaml" "github.com/gorilla/mux" - "github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/manager/repository" + "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/util" ) @@ -48,6 +49,7 @@ var deployments = []Route{ {"Expand", "/expand", "POST", expandHandlerFunc, ""}, {"ListTypes", "/types", "GET", listTypesHandlerFunc, ""}, {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, + {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, } var ( @@ -72,8 +74,9 @@ func init() { func newManager() manager.Manager { expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver()) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName)) + registryService := registry.NewInmemRepositoryService() r := repository.NewMapBasedRepository() - return manager.NewManager(expander, deployer, r) + return manager.NewManager(expander, deployer, r, registryService) } func getServiceURL(serviceURL, serviceName string) string { @@ -329,3 +332,15 @@ func listTypeInstancesHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, backend.ListInstances(typeName), http.StatusOK) } + +// Putting Registry handlers here for now because deployments.go +// currently owns its own Manager backend and doesn't like to share. +func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: list registries" + util.LogHandlerEntry(handler, r) + registries, err := backend.ListRegistries() + if err != nil { + return + } + util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK) +} diff --git a/manager/manager/manager.go b/manager/manager/manager.go index d1c6c518a..1953198ba 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/manager/repository" + "github.com/kubernetes/deployment-manager/registry" ) // Manager manages a persistent set of Deployments. @@ -37,17 +38,23 @@ type Manager interface { Expand(t *common.Template) (*common.Manifest, error) ListTypes() []string ListInstances(typeName string) []*common.TypeInstance + // Registry related functions + ListRegistries() ([]*common.Registry, error) + CreateRegistry(pr *common.Registry) error + GetRegistry(name string) (*common.Registry, error) + DeleteRegistry(name string) error } type manager struct { - expander Expander - deployer Deployer - repository repository.Repository + expander Expander + deployer Deployer + repository repository.Repository + registryService registry.RegistryService } // NewManager returns a new initialized Manager. -func NewManager(expander Expander, deployer Deployer, repository repository.Repository) Manager { - return &manager{expander, deployer, repository} +func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager { + return &manager{expander, deployer, repository, registryService} } // ListDeployments returns the list of deployments @@ -302,6 +309,22 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance { return m.repository.GetTypeInstances(typeName) } +func (m *manager) ListRegistries() ([]*common.Registry, error) { + return m.registryService.List() +} + +func (m *manager) CreateRegistry(pr *common.Registry) error { + return m.registryService.Create(pr) +} + +func (m *manager) GetRegistry(name string) (*common.Registry, error) { + return m.registryService.Get(name) +} + +func (m *manager) DeleteRegistry(name string) error { + return m.registryService.Delete(name) +} + func generateManifestName() string { return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano()) } diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go index 714e570f6..3bac784b9 100644 --- a/registry/inmem_repository_service.go +++ b/registry/inmem_repository_service.go @@ -28,13 +28,30 @@ type inmemRepositoryService struct { } func NewInmemRepositoryService() RegistryService { - return &inmemRepositoryService{ + rs := &inmemRepositoryService{ repositories: make(map[string]*common.Registry), } + rs.Create(&common.Registry{ + Name: "charts", + Type: common.Github, + URL: "github.com/helm/charts", + Format: common.UnversionedRegistry, + }) + rs.Create(&common.Registry{ + Name: "application-dm-templates", + Type: common.Github, + URL: "github.com/kubernetes/application-dm-templates", + Format: common.VersionedRegistry, + }) + return rs } func (rs *inmemRepositoryService) List() ([]*common.Registry, error) { - return nil, nil + ret := []*common.Registry{} + for _, r := range rs.repositories { + ret = append(ret, r) + } + return ret, nil } func (rs *inmemRepositoryService) Create(repository *common.Registry) error { diff --git a/registry/registryprovider.go b/registry/registryprovider.go index 03e4f5129..f767e896d 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -30,18 +30,6 @@ type RegistryProvider interface { func NewDefaultRegistryProvider() RegistryProvider { rs := NewInmemRepositoryService() - rs.Create(&common.Registry{ - Name: "charts", - Type: common.Github, - URL: "github.com/helm/charts", - Format: common.UnversionedRegistry, - }) - rs.Create(&common.Registry{ - Name: "application-dm-templates", - Type: common.Github, - URL: "github.com/kubernetes/application-dm-templates", - Format: common.VersionedRegistry, - }) return &DefaultRegistryProvider{rs: rs} } From 1ee72edf31399782ebd8e6cd592b095669bb5357 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Thu, 7 Jan 2016 09:32:47 -0800 Subject: [PATCH 08/11] use the inmem service registry for the manager_test --- manager/manager/manager_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index a52e73d55..d0e59efda 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import ( "testing" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/registry" ) var template = common.Template{Name: "test", Content: "test"} @@ -251,7 +252,8 @@ func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.Type var testExpander = &expanderStub{} var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() -var testManager = NewManager(testExpander, testDeployer, testRepository) +var testRegistryService = registry.NewInmemRepositoryService() +var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService) func TestListDeployments(t *testing.T) { testRepository.reset() From 3db24d853170f9fd0bc4cfe5837e96e222faac79 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Thu, 7 Jan 2016 10:25:43 -0800 Subject: [PATCH 09/11] clean up the code for fetching registries --- dm/dm.go | 9 ++------- registry/inmem_repository_service.go | 15 +++------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/dm/dm.go b/dm/dm.go index 9a9a0d22b..2c7e56151 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -44,7 +44,7 @@ var ( deployment_name = flag.String("name", "", "Name of deployment, used for deploy and update commands (defaults to template name)") stdin = flag.Bool("stdin", false, "Reads a configuration from the standard input") properties = flag.String("properties", "", "Properties to use when deploying a template (e.g., --properties k1=v1,k2=v2)") - template_registry = flag.String("registry", "kubernetes/application-dm-templates", "Github based template registry (owner/repo[/path])") + template_registry = flag.String("registry", "github.com/kubernetes/application-dm-templates", "Registry (github.com/owner/repo)") service = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager") binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") timeout = flag.Int("timeout", 10, "Time in seconds to wait for response") @@ -82,12 +82,7 @@ var usage = func() { } func getGitRegistry() (registry.Registry, error) { - rs := registry.NewDefaultRegistryProvider() - s := strings.Split(*template_registry, "/") - if len(s) < 2 { - panic(fmt.Errorf("invalid template registry: %s", *template_registry)) - } - return rs.GetRegistry("github.com/" + s[0] + "/" + s[1]) + return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry) } func main() { diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go index 3bac784b9..a0a18c614 100644 --- a/registry/inmem_repository_service.go +++ b/registry/inmem_repository_service.go @@ -66,20 +66,11 @@ func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) { func (rs *inmemRepositoryService) Delete(name string) error { return nil } -func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { - if !strings.HasPrefix(URL, "github.com/") { - return nil, fmt.Errorf("Failed to parse short github url: %s", URL) - } - s := strings.Split(URL, "/") - if len(s) < 3 { - panic(fmt.Errorf("invalid template registry: %s", URL)) - } - toFind := "github.com/" + s[1] + "/" + s[2] - fmt.Printf("toFind: %s", toFind) +// GetByURL returns a registry that handles the types for a given URL. +func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { for _, r := range rs.repositories { - fmt.Printf("Checking: %s", r.URL) - if r.URL == toFind { + if strings.HasPrefix(r.URL, URL) { return r, nil } } From 0b82b48cc4f6997e7359620b8c72a0f6b855a0d4 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Thu, 7 Jan 2016 10:35:50 -0800 Subject: [PATCH 10/11] clean up the code for fetching registries --- manager/manager/typeresolver.go | 4 ++-- registry/inmem_repository_service.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 2d7fcb5b1..73b75bd9a 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -244,7 +244,7 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro if len(m) != 6 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + r, err := tr.rp.GetRegistry(template) if err != nil { return []string{}, err } @@ -262,7 +262,7 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin if len(m) != 4 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + r, err := tr.rp.GetRegistry(template) if err != nil { return []string{}, err } diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go index a0a18c614..47845ac26 100644 --- a/registry/inmem_repository_service.go +++ b/registry/inmem_repository_service.go @@ -70,7 +70,7 @@ func (rs *inmemRepositoryService) Delete(name string) error { // GetByURL returns a registry that handles the types for a given URL. func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { for _, r := range rs.repositories { - if strings.HasPrefix(r.URL, URL) { + if strings.HasPrefix(URL, r.URL) { return r, nil } } From 89669d0d14591372daa567370f5cbd0b7f48cd12 Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Thu, 7 Jan 2016 11:11:14 -0800 Subject: [PATCH 11/11] clean up the code for fetching registries --- manager/deployments.go | 4 ++-- manager/manager/typeresolver.go | 4 ++-- manager/manager/typeresolver_test.go | 24 ++++++++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manager/deployments.go b/manager/deployments.go index 460020ba5..02c21f8b3 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -72,10 +72,10 @@ func init() { } func newManager() manager.Manager { - expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver()) + expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider())) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName)) - registryService := registry.NewInmemRepositoryService() r := repository.NewMapBasedRepository() + registryService := registry.NewInmemRepositoryService() return manager.NewManager(expander, deployer, r, registryService) } diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 73b75bd9a..3da460c23 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -50,7 +50,7 @@ type fetchUnit struct { } // NewTypeResolver returns a new initialized TypeResolver. -func NewTypeResolver() TypeResolver { +func NewTypeResolver(rp registry.RegistryProvider) TypeResolver { ret := &typeResolver{} client := http.DefaultClient //TODO (iantw): Make this a flag @@ -58,7 +58,7 @@ func NewTypeResolver() TypeResolver { client.Timeout = timeout ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.maxUrls = maxURLImports - ret.rp = registry.NewDefaultRegistryProvider() + ret.rp = rp return ret } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index a17b03ca4..96010f4f8 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -64,19 +64,23 @@ type urlAndError struct { } type testRegistryProvider struct { - owner string - repo string - r map[string]registry.Registry + URLPrefix string + r map[string]registry.Registry } -func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { +func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { r := make(map[string]registry.Registry) - r["github.com/"+owner+"/"+repository] = &testGithubRegistry{tests, count} - return &testRegistryProvider{owner, repository, r} + r[URLPrefix] = &testGithubRegistry{tests, count} + return &testRegistryProvider{URLPrefix, r} } func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { - return trp.r[URL], nil + for key, r := range trp.r { + if strings.HasPrefix(URL, key) { + return r, nil + } + } + return nil, fmt.Errorf("No registry found for %s", URL) } type testGithubRegistry struct { @@ -352,7 +356,7 @@ func TestShortGithubUrlMapping(t *testing.T) { } test := resolverTestCase{ - registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), } testUrlConversionDriver(test, tests, t) } @@ -369,7 +373,7 @@ func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { } test := resolverTestCase{ - registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2), } testUrlConversionDriver(test, tests, t) } @@ -411,7 +415,7 @@ func TestShortGithubUrl(t *testing.T) { importOut: finalImports, urlcount: 4, responses: responses, - registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), } testDriver(test, t) }