From 616a8c387719848530bb4413b029ab0db2ae0d5c Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Wed, 6 Jan 2016 16:33:55 -0800 Subject: [PATCH] First cut of unifying the registries, just puts the scaffolding in place --- common/types.go | 36 +++++++++++++- dm/dm.go | 27 +++++------ manager/manager/typeresolver.go | 14 ++++-- manager/manager/typeresolver_test.go | 12 ++--- registry/github_package_registry.go | 6 +-- registry/github_registry.go | 6 +-- registry/inmem_repository_service.go | 70 ++++++++++++++++++++++++++++ registry/registry.go | 19 +++++++- registry/registryprovider.go | 49 +++++++++++++++---- util/templateutil.go | 2 +- 10 files changed, 197 insertions(+), 44 deletions(-) create mode 100644 registry/inmem_repository_service.go diff --git a/common/types.go b/common/types.go index 2f07783a7..72681820d 100644 --- a/common/types.go +++ b/common/types.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -166,3 +166,37 @@ type KubernetesObject struct { Metadata map[string]interface{} `json:"metadata"` Spec map[string]interface{} `json:"spec"` } + +// Repository related types +type BasicAuthCredential struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// Credentials used to access the repository +type RegistryCredential struct { + BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` +} + +type Registry struct { + Name string `json:"name,omitempty"` // Friendly name for the repo + Type RegistryType `json:"type,omitempty"` // Technology implementing the registry + URL string `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts + Credential RegistryCredential `json:"credential,omitempty"` + Format RegistryFormat `json:"format,omitempty"` +} + +// RegistryType defines the technology that implements the registry +type RegistryType string + +const ( + Github RegistryType = "github" +) + +// RegistryFormat defines the format of the registry +type RegistryFormat string + +const ( + VersionedRegistry RegistryFormat = "versioned" + UnversionedRegistry RegistryFormat = "unversioned" +) diff --git a/dm/dm.go b/dm/dm.go index 1c24f3e7a..9a9a0d22b 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -81,22 +81,13 @@ var usage = func() { panic("\n") } -func getGitRegistry() registry.Registry { +func getGitRegistry() (registry.Registry, error) { + rs := registry.NewDefaultRegistryProvider() s := strings.Split(*template_registry, "/") if len(s) < 2 { panic(fmt.Errorf("invalid template registry: %s", *template_registry)) } - - var path = "" - if len(s) > 2 { - path = strings.Join(s[2:], "/") - } - - if s[0] == "helm" { - return registry.NewGithubPackageRegistry(s[0], s[1]) - } else { - return registry.NewGithubRegistry(s[0], s[1], path) - } + return rs.GetRegistry("github.com/" + s[0] + "/" + s[1]) } func main() { @@ -121,7 +112,10 @@ func execute() { switch args[0] { case "templates": - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Cannot get registry %v", err)) + } templates, err := git.List() if err != nil { panic(fmt.Errorf("Cannot list %v", err)) @@ -305,7 +299,10 @@ func getTypeURLs(tName string) []string { } func getDownloadURLs(t registry.Type) []string { - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Failed to get registry")) + } urls, err := git.GetURLs(t) if err != nil { panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index fbc291050..2d7fcb5b1 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ func NewTypeResolver() TypeResolver { client.Timeout = timeout ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.maxUrls = maxURLImports - ret.rp = ®istry.DefaultRegistryProvider{} + ret.rp = registry.NewDefaultRegistryProvider() return ret } @@ -244,7 +244,10 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro if len(m) != 6 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + if err != nil { + return []string{}, err + } t := registry.Type{m[3], m[4], m[5]} return r.GetURLs(t) } @@ -259,7 +262,10 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin if len(m) != 4 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubPackageRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry("github.com/" + m[1] + "/" + m[2]) + if err != nil { + return []string{}, err + } t := registry.Type{Name: m[3]} return r.GetURLs(t) } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index b2e9f78ee..a17b03ca4 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,16 +71,12 @@ type testRegistryProvider struct { func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { r := make(map[string]registry.Registry) - r[owner+repository] = &testGithubRegistry{tests, count} + r["github.com/"+owner+"/"+repository] = &testGithubRegistry{tests, count} return &testRegistryProvider{owner, repository, r} } -func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] -} - -func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] +func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { + return trp.r[URL], nil } type testGithubRegistry struct { diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 664a7e096..ba8c903a2 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,11 +39,11 @@ type GithubPackageRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry { +func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry { return &GithubPackageRegistry{ owner: owner, repository: repository, - client: github.NewClient(nil), + client: client, } } diff --git a/registry/github_registry.go b/registry/github_registry.go index 0f06dd3ea..c3cf70f01 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,12 +55,12 @@ type GithubRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubRegistry(owner, repository, path string) *GithubRegistry { +func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry { return &GithubRegistry{ owner: owner, repository: repository, path: path, - client: github.NewClient(nil), + client: client, } } diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go new file mode 100644 index 000000000..714e570f6 --- /dev/null +++ b/registry/inmem_repository_service.go @@ -0,0 +1,70 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "strings" + + "github.com/kubernetes/deployment-manager/common" +) + +type inmemRepositoryService struct { + repositories map[string]*common.Registry +} + +func NewInmemRepositoryService() RegistryService { + return &inmemRepositoryService{ + repositories: make(map[string]*common.Registry), + } +} + +func (rs *inmemRepositoryService) List() ([]*common.Registry, error) { + return nil, nil +} + +func (rs *inmemRepositoryService) Create(repository *common.Registry) error { + rs.repositories[repository.URL] = repository + return nil +} + +func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) { + return &common.Registry{}, nil +} + +func (rs *inmemRepositoryService) Delete(name string) error { + return nil +} +func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { + if !strings.HasPrefix(URL, "github.com/") { + return nil, fmt.Errorf("Failed to parse short github url: %s", URL) + } + s := strings.Split(URL, "/") + if len(s) < 3 { + panic(fmt.Errorf("invalid template registry: %s", URL)) + } + + toFind := "github.com/" + s[1] + "/" + s[2] + fmt.Printf("toFind: %s", toFind) + for _, r := range rs.repositories { + fmt.Printf("Checking: %s", r.URL) + if r.URL == toFind { + return r, nil + } + } + return nil, fmt.Errorf("Failed to find registry for github url: %s", URL) +} diff --git a/registry/registry.go b/registry/registry.go index e53844e74..8e323fe4e 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +16,23 @@ limitations under the License. package registry +import ( + "github.com/kubernetes/deployment-manager/common" +) + +type RegistryService interface { + // List all the registries + List() ([]*common.Registry, error) + // Create a new registry + Create(repository *common.Registry) error + // Get a registry + Get(name string) (*common.Registry, error) + // Delete a registry + Delete(name string) error + // Find a registry that backs the given URL + GetByURL(URL string) (*common.Registry, error) +} + // Registry abstracts a registry that holds templates, which can be // used in a Deployment Manager configurations. There can be multiple // implementations of a registry. Currently we support Deployment Manager diff --git a/registry/registryprovider.go b/registry/registryprovider.go index d4454451c..03e4f5129 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,19 +16,52 @@ limitations under the License. package registry +import ( + "fmt" + + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" +) + // RegistryProvider returns factories for creating registries for a given RegistryType. type RegistryProvider interface { - GetGithubRegistry(owner string, repository string) Registry - GetGithubPackageRegistry(owner string, repository string) Registry + GetRegistry(prefix string) (Registry, error) } -type DefaultRegistryProvider struct { +func NewDefaultRegistryProvider() RegistryProvider { + rs := NewInmemRepositoryService() + rs.Create(&common.Registry{ + Name: "charts", + Type: common.Github, + URL: "github.com/helm/charts", + Format: common.UnversionedRegistry, + }) + rs.Create(&common.Registry{ + Name: "application-dm-templates", + Type: common.Github, + URL: "github.com/kubernetes/application-dm-templates", + Format: common.VersionedRegistry, + }) + return &DefaultRegistryProvider{rs: rs} + } -func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { - return NewGithubRegistry(owner, repository, "") +type DefaultRegistryProvider struct { + rs RegistryService } -func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry { - return NewGithubPackageRegistry(owner, repository) +func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) { + r, err := drp.rs.GetByURL(URL) + if err != nil { + return nil, err + } + if r.Type == common.Github { + if r.Format == common.UnversionedRegistry { + return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil + } + if r.Format == common.VersionedRegistry { + return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil + } + } + return nil, fmt.Errorf("cannot find registry backing url %s", URL) } diff --git a/util/templateutil.go b/util/templateutil.go index 40c9419d8..3ce844c5d 100644 --- a/util/templateutil.go +++ b/util/templateutil.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.