diff --git a/common/types.go b/common/types.go index 2f07783a7..72681820d 100644 --- a/common/types.go +++ b/common/types.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -166,3 +166,37 @@ type KubernetesObject struct { Metadata map[string]interface{} `json:"metadata"` Spec map[string]interface{} `json:"spec"` } + +// Repository related types +type BasicAuthCredential struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// Credentials used to access the repository +type RegistryCredential struct { + BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` +} + +type Registry struct { + Name string `json:"name,omitempty"` // Friendly name for the repo + Type RegistryType `json:"type,omitempty"` // Technology implementing the registry + URL string `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts + Credential RegistryCredential `json:"credential,omitempty"` + Format RegistryFormat `json:"format,omitempty"` +} + +// RegistryType defines the technology that implements the registry +type RegistryType string + +const ( + Github RegistryType = "github" +) + +// RegistryFormat defines the format of the registry +type RegistryFormat string + +const ( + VersionedRegistry RegistryFormat = "versioned" + UnversionedRegistry RegistryFormat = "unversioned" +) diff --git a/dm/dm.go b/dm/dm.go index 1c24f3e7a..2c7e56151 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,7 +44,7 @@ var ( deployment_name = flag.String("name", "", "Name of deployment, used for deploy and update commands (defaults to template name)") stdin = flag.Bool("stdin", false, "Reads a configuration from the standard input") properties = flag.String("properties", "", "Properties to use when deploying a template (e.g., --properties k1=v1,k2=v2)") - template_registry = flag.String("registry", "kubernetes/application-dm-templates", "Github based template registry (owner/repo[/path])") + template_registry = flag.String("registry", "github.com/kubernetes/application-dm-templates", "Registry (github.com/owner/repo)") service = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager") binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") timeout = flag.Int("timeout", 10, "Time in seconds to wait for response") @@ -81,22 +81,8 @@ var usage = func() { panic("\n") } -func getGitRegistry() registry.Registry { - s := strings.Split(*template_registry, "/") - if len(s) < 2 { - panic(fmt.Errorf("invalid template registry: %s", *template_registry)) - } - - var path = "" - if len(s) > 2 { - path = strings.Join(s[2:], "/") - } - - if s[0] == "helm" { - return registry.NewGithubPackageRegistry(s[0], s[1]) - } else { - return registry.NewGithubRegistry(s[0], s[1], path) - } +func getGitRegistry() (registry.Registry, error) { + return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry) } func main() { @@ -121,7 +107,10 @@ func execute() { switch args[0] { case "templates": - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Cannot get registry %v", err)) + } templates, err := git.List() if err != nil { panic(fmt.Errorf("Cannot list %v", err)) @@ -305,7 +294,10 @@ func getTypeURLs(tName string) []string { } func getDownloadURLs(t registry.Type) []string { - git := getGitRegistry() + git, err := getGitRegistry() + if err != nil { + panic(fmt.Errorf("Failed to get registry")) + } urls, err := git.GetURLs(t) if err != nil { panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) diff --git a/manager/deployments.go b/manager/deployments.go index dceb966c1..02c21f8b3 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,9 +31,10 @@ import ( "github.com/ghodss/yaml" "github.com/gorilla/mux" - "github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/manager/repository" + "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/util" ) @@ -48,6 +49,7 @@ var deployments = []Route{ {"Expand", "/expand", "POST", expandHandlerFunc, ""}, {"ListTypes", "/types", "GET", listTypesHandlerFunc, ""}, {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, + {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, } var ( @@ -70,10 +72,11 @@ func init() { } func newManager() manager.Manager { - expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver()) + expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider())) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName)) r := repository.NewMapBasedRepository() - return manager.NewManager(expander, deployer, r) + registryService := registry.NewInmemRepositoryService() + return manager.NewManager(expander, deployer, r, registryService) } func getServiceURL(serviceURL, serviceName string) string { @@ -329,3 +332,15 @@ func listTypeInstancesHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, backend.ListInstances(typeName), http.StatusOK) } + +// Putting Registry handlers here for now because deployments.go +// currently owns its own Manager backend and doesn't like to share. +func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: list registries" + util.LogHandlerEntry(handler, r) + registries, err := backend.ListRegistries() + if err != nil { + return + } + util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK) +} diff --git a/manager/manager/manager.go b/manager/manager/manager.go index d1c6c518a..1953198ba 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/manager/repository" + "github.com/kubernetes/deployment-manager/registry" ) // Manager manages a persistent set of Deployments. @@ -37,17 +38,23 @@ type Manager interface { Expand(t *common.Template) (*common.Manifest, error) ListTypes() []string ListInstances(typeName string) []*common.TypeInstance + // Registry related functions + ListRegistries() ([]*common.Registry, error) + CreateRegistry(pr *common.Registry) error + GetRegistry(name string) (*common.Registry, error) + DeleteRegistry(name string) error } type manager struct { - expander Expander - deployer Deployer - repository repository.Repository + expander Expander + deployer Deployer + repository repository.Repository + registryService registry.RegistryService } // NewManager returns a new initialized Manager. -func NewManager(expander Expander, deployer Deployer, repository repository.Repository) Manager { - return &manager{expander, deployer, repository} +func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager { + return &manager{expander, deployer, repository, registryService} } // ListDeployments returns the list of deployments @@ -302,6 +309,22 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance { return m.repository.GetTypeInstances(typeName) } +func (m *manager) ListRegistries() ([]*common.Registry, error) { + return m.registryService.List() +} + +func (m *manager) CreateRegistry(pr *common.Registry) error { + return m.registryService.Create(pr) +} + +func (m *manager) GetRegistry(name string) (*common.Registry, error) { + return m.registryService.Get(name) +} + +func (m *manager) DeleteRegistry(name string) error { + return m.registryService.Delete(name) +} + func generateManifestName() string { return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano()) } diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index a52e73d55..d0e59efda 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import ( "testing" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/registry" ) var template = common.Template{Name: "test", Content: "test"} @@ -251,7 +252,8 @@ func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.Type var testExpander = &expanderStub{} var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() -var testManager = NewManager(testExpander, testDeployer, testRepository) +var testRegistryService = registry.NewInmemRepositoryService() +var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService) func TestListDeployments(t *testing.T) { testRepository.reset() diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index fbc291050..3da460c23 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,7 +50,7 @@ type fetchUnit struct { } // NewTypeResolver returns a new initialized TypeResolver. -func NewTypeResolver() TypeResolver { +func NewTypeResolver(rp registry.RegistryProvider) TypeResolver { ret := &typeResolver{} client := http.DefaultClient //TODO (iantw): Make this a flag @@ -58,7 +58,7 @@ func NewTypeResolver() TypeResolver { client.Timeout = timeout ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.maxUrls = maxURLImports - ret.rp = ®istry.DefaultRegistryProvider{} + ret.rp = rp return ret } @@ -244,7 +244,10 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro if len(m) != 6 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry(template) + if err != nil { + return []string{}, err + } t := registry.Type{m[3], m[4], m[5]} return r.GetURLs(t) } @@ -259,7 +262,10 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin if len(m) != 4 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r := tr.rp.GetGithubPackageRegistry(m[1], m[2]) + r, err := tr.rp.GetRegistry(template) + if err != nil { + return []string{}, err + } t := registry.Type{Name: m[3]} return r.GetURLs(t) } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index b2e9f78ee..96010f4f8 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,23 +64,23 @@ type urlAndError struct { } type testRegistryProvider struct { - owner string - repo string - r map[string]registry.Registry + URLPrefix string + r map[string]registry.Registry } -func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { +func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { r := make(map[string]registry.Registry) - r[owner+repository] = &testGithubRegistry{tests, count} - return &testRegistryProvider{owner, repository, r} -} - -func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] + r[URLPrefix] = &testGithubRegistry{tests, count} + return &testRegistryProvider{URLPrefix, r} } -func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry { - return trp.r[owner+repository] +func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { + for key, r := range trp.r { + if strings.HasPrefix(URL, key) { + return r, nil + } + } + return nil, fmt.Errorf("No registry found for %s", URL) } type testGithubRegistry struct { @@ -356,7 +356,7 @@ func TestShortGithubUrlMapping(t *testing.T) { } test := resolverTestCase{ - registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), } testUrlConversionDriver(test, tests, t) } @@ -373,7 +373,7 @@ func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { } test := resolverTestCase{ - registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2), } testUrlConversionDriver(test, tests, t) } @@ -415,7 +415,7 @@ func TestShortGithubUrl(t *testing.T) { importOut: finalImports, urlcount: 4, responses: responses, - registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), } testDriver(test, t) } diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 096c3c23a..ba8c903a2 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,16 +39,15 @@ type GithubPackageRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry { +func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry { return &GithubPackageRegistry{ owner: owner, repository: repository, - client: github.NewClient(nil), + client: client, } } // List the types from the Registry. -// TODO(vaikas): Figure out how the versions work here. func (g *GithubPackageRegistry) List() ([]Type, error) { // Just list all the types at the top level. types, err := g.getDirs("") diff --git a/registry/github_registry.go b/registry/github_registry.go index 0f06dd3ea..c3cf70f01 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,12 +55,12 @@ type GithubRegistry struct { } // NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubRegistry(owner, repository, path string) *GithubRegistry { +func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry { return &GithubRegistry{ owner: owner, repository: repository, path: path, - client: github.NewClient(nil), + client: client, } } diff --git a/registry/inmem_repository_service.go b/registry/inmem_repository_service.go new file mode 100644 index 000000000..47845ac26 --- /dev/null +++ b/registry/inmem_repository_service.go @@ -0,0 +1,78 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "strings" + + "github.com/kubernetes/deployment-manager/common" +) + +type inmemRepositoryService struct { + repositories map[string]*common.Registry +} + +func NewInmemRepositoryService() RegistryService { + rs := &inmemRepositoryService{ + repositories: make(map[string]*common.Registry), + } + rs.Create(&common.Registry{ + Name: "charts", + Type: common.Github, + URL: "github.com/helm/charts", + Format: common.UnversionedRegistry, + }) + rs.Create(&common.Registry{ + Name: "application-dm-templates", + Type: common.Github, + URL: "github.com/kubernetes/application-dm-templates", + Format: common.VersionedRegistry, + }) + return rs +} + +func (rs *inmemRepositoryService) List() ([]*common.Registry, error) { + ret := []*common.Registry{} + for _, r := range rs.repositories { + ret = append(ret, r) + } + return ret, nil +} + +func (rs *inmemRepositoryService) Create(repository *common.Registry) error { + rs.repositories[repository.URL] = repository + return nil +} + +func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) { + return &common.Registry{}, nil +} + +func (rs *inmemRepositoryService) Delete(name string) error { + return nil +} + +// GetByURL returns a registry that handles the types for a given URL. +func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { + for _, r := range rs.repositories { + if strings.HasPrefix(URL, r.URL) { + return r, nil + } + } + return nil, fmt.Errorf("Failed to find registry for github url: %s", URL) +} diff --git a/registry/registry.go b/registry/registry.go index e53844e74..8e323fe4e 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +16,23 @@ limitations under the License. package registry +import ( + "github.com/kubernetes/deployment-manager/common" +) + +type RegistryService interface { + // List all the registries + List() ([]*common.Registry, error) + // Create a new registry + Create(repository *common.Registry) error + // Get a registry + Get(name string) (*common.Registry, error) + // Delete a registry + Delete(name string) error + // Find a registry that backs the given URL + GetByURL(URL string) (*common.Registry, error) +} + // Registry abstracts a registry that holds templates, which can be // used in a Deployment Manager configurations. There can be multiple // implementations of a registry. Currently we support Deployment Manager diff --git a/registry/registryprovider.go b/registry/registryprovider.go index d4454451c..f767e896d 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,19 +16,40 @@ limitations under the License. package registry +import ( + "fmt" + + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" +) + // RegistryProvider returns factories for creating registries for a given RegistryType. type RegistryProvider interface { - GetGithubRegistry(owner string, repository string) Registry - GetGithubPackageRegistry(owner string, repository string) Registry + GetRegistry(prefix string) (Registry, error) } -type DefaultRegistryProvider struct { +func NewDefaultRegistryProvider() RegistryProvider { + rs := NewInmemRepositoryService() + return &DefaultRegistryProvider{rs: rs} + } -func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { - return NewGithubRegistry(owner, repository, "") +type DefaultRegistryProvider struct { + rs RegistryService } -func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry { - return NewGithubPackageRegistry(owner, repository) +func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) { + r, err := drp.rs.GetByURL(URL) + if err != nil { + return nil, err + } + if r.Type == common.Github { + if r.Format == common.UnversionedRegistry { + return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil + } + if r.Format == common.VersionedRegistry { + return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil + } + } + return nil, fmt.Errorf("cannot find registry backing url %s", URL) } diff --git a/util/templateutil.go b/util/templateutil.go index 40c9419d8..3ce844c5d 100644 --- a/util/templateutil.go +++ b/util/templateutil.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.