From 17f22d777fbe4e4cc1931c7b2cfd3c9b6ff4277f Mon Sep 17 00:00:00 2001 From: jackgr Date: Thu, 7 Jan 2016 18:37:47 -0800 Subject: [PATCH 1/6] Merge changes from #172. --- common/types.go | 2 +- dm/dm.go | 2 +- manager/deployments.go | 68 ++++++++++- manager/manager/manager.go | 58 +++++++-- manager/manager/manager_test.go | 98 ++++++++++++++- manager/manager/typeresolver.go | 4 +- manager/manager/typeresolver_test.go | 32 +++++ registry/github_package_registry.go | 37 +++++- registry/github_registry.go | 54 +++++++++ ...y_service.go => inmem_registry_service.go} | 30 ++--- registry/registry.go | 48 +++++--- registry/registry_provider.go | 114 ++++++++++++++++++ registry/registryprovider.go | 55 --------- registry/semver.go | 80 ++++++++++++ 14 files changed, 575 insertions(+), 107 deletions(-) rename registry/{inmem_repository_service.go => inmem_registry_service.go} (64%) create mode 100644 registry/registry_provider.go delete mode 100644 registry/registryprovider.go create mode 100644 registry/semver.go diff --git a/common/types.go b/common/types.go index 72681820d..0bdda9e7a 100644 --- a/common/types.go +++ b/common/types.go @@ -190,7 +190,7 @@ type Registry struct { type RegistryType string const ( - Github RegistryType = "github" + GithubRegistryType RegistryType = "github" ) // RegistryFormat defines the format of the registry diff --git a/dm/dm.go b/dm/dm.go index 2c7e56151..95f48ffd0 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -82,7 +82,7 @@ var usage = func() { } func getGitRegistry() (registry.Registry, error) { - return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry) + return registry.NewDefaultRegistryProvider().GetRegistryByURL(*template_registry) } func main() { diff --git a/manager/deployments.go b/manager/deployments.go index 02c21f8b3..1d728c829 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -50,6 +50,9 @@ var deployments = []Route{ {"ListTypes", "/types", "GET", listTypesHandlerFunc, ""}, {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, + {"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""}, + {"ListCharts", "/registries/{registry}/charts", "GET", listChartsHandlerFunc, ""}, + {"GetChart", "/registries/{registry}/charts/{chart}", "GET", getChartHandlerFunc, ""}, } var ( @@ -72,11 +75,13 @@ func init() { } func newManager() manager.Manager { - expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider())) + provider := registry.NewDefaultRegistryProvider() + resolver := manager.NewTypeResolver(provider) + expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName)) r := repository.NewMapBasedRepository() - registryService := registry.NewInmemRepositoryService() - return manager.NewManager(expander, deployer, r, registryService) + service := registry.NewInmemRegistryService() + return manager.NewManager(expander, deployer, r, provider, service) } func getServiceURL(serviceURL, serviceName string) string { @@ -342,5 +347,62 @@ func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) { if err != nil { return } + util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK) } + +func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get registry" + util.LogHandlerEntry(handler, r) + registryName, err := getPathVariable(w, r, "registry", handler) + if err != nil { + return + } + + cr, err := backend.GetRegistry(registryName) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK) +} + +func listChartsHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: list charts" + util.LogHandlerEntry(handler, r) + registryName, err := getPathVariable(w, r, "registry", handler) + if err != nil { + return + } + + chartNames, err := backend.ListCharts(registryName) + if err != nil { + util.LogAndReturnError(handler, http.StatusInternalServerError, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, chartNames, http.StatusOK) +} + +func getChartHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get chart" + util.LogHandlerEntry(handler, r) + registryName, err := getPathVariable(w, r, "registry", handler) + if err != nil { + return + } + + chartName, err := getPathVariable(w, r, "chart", handler) + if err != nil { + return + } + + c, err := backend.GetChart(registryName, chartName) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK) +} diff --git a/manager/manager/manager.go b/manager/manager/manager.go index 1953198ba..a8ecee6c3 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -28,33 +28,48 @@ import ( // Manager manages a persistent set of Deployments. type Manager interface { + // Deployments ListDeployments() ([]common.Deployment, error) GetDeployment(name string) (*common.Deployment, error) CreateDeployment(t *common.Template) (*common.Deployment, error) DeleteDeployment(name string, forget bool) (*common.Deployment, error) PutDeployment(name string, t *common.Template) (*common.Deployment, error) + + // Manifests ListManifests(deploymentName string) (map[string]*common.Manifest, error) GetManifest(deploymentName string, manifest string) (*common.Manifest, error) Expand(t *common.Template) (*common.Manifest, error) + + // Types ListTypes() []string ListInstances(typeName string) []*common.TypeInstance - // Registry related functions + + // Registries ListRegistries() ([]*common.Registry, error) CreateRegistry(pr *common.Registry) error GetRegistry(name string) (*common.Registry, error) DeleteRegistry(name string) error + + // Charts + ListCharts(registryName string) ([]string, error) + GetChart(registryName, chartName string) (*registry.Chart, error) } type manager struct { - expander Expander - deployer Deployer - repository repository.Repository - registryService registry.RegistryService + expander Expander + deployer Deployer + repository repository.Repository + provider registry.RegistryProvider + service registry.RegistryService } // NewManager returns a new initialized Manager. -func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager { - return &manager{expander, deployer, repository, registryService} +func NewManager(expander Expander, + deployer Deployer, + repository repository.Repository, + provider registry.RegistryProvider, + service registry.RegistryService) Manager { + return &manager{expander, deployer, repository, provider, service} } // ListDeployments returns the list of deployments @@ -309,20 +324,21 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance { return m.repository.GetTypeInstances(typeName) } +// ListRegistries returns the list of registries func (m *manager) ListRegistries() ([]*common.Registry, error) { - return m.registryService.List() + return m.service.List() } func (m *manager) CreateRegistry(pr *common.Registry) error { - return m.registryService.Create(pr) + return m.service.Create(pr) } func (m *manager) GetRegistry(name string) (*common.Registry, error) { - return m.registryService.Get(name) + return m.service.Get(name) } func (m *manager) DeleteRegistry(name string) error { - return m.registryService.Delete(name) + return m.service.Delete(name) } func generateManifestName() string { @@ -346,3 +362,23 @@ func getResourceErrors(c *common.Configuration) []string { return errs } + +// ListCharts retrieves the names of the charts in a given registry. +func (m *manager) ListCharts(registryName string) ([]string, error) { + r, err := m.provider.GetRegistryByName(registryName) + if err != nil { + return nil, err + } + + return r.ListCharts() +} + +// GetChart retrieves a given chart in a given registry. +func (m *manager) GetChart(registryName, chartName string) (*registry.Chart, error) { + r, err := m.provider.GetRegistryByName(registryName) + if err != nil { + return nil, err + } + + return r.GetChart(chartName) +} diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index d0e59efda..3cfb68891 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -66,7 +66,7 @@ var deploymentList = []common.Deployment{deployment, {Name: "test2"}} var typeInstMap = map[string][]string{"test": []string{"test"}} -var errTest = errors.New("test") +var errTest = errors.New("test error") type expanderStub struct{} @@ -164,6 +164,7 @@ func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error) if repository.FailListDeployments { return deploymentList, errTest } + return deploymentList, nil } @@ -249,11 +250,102 @@ func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.Type } } +type registryStub struct { + FailListCharts bool + FailGetChart bool + ListChartsCalled bool + GetChartCalled bool +} + +func newRegistryStub() *registryStub { + ret := ®istryStub{} + return ret +} + +func (r *registryStub) reset() { + r.FailListCharts = false + r.FailGetChart = false + r.ListChartsCalled = false + r.GetChartCalled = false +} + +var testRegistryName = "TestRegistry" +var testRegistryURL = "https://github.com/helm/charts" +var testChartName = "TestChart" + +var testChart = registry.Chart{ + Name: testChartName, +} + +var testChartList = []string{testChartName, "TestChart2"} + +func (r *registryStub) GetRegistryName() string { + return testRegistryName +} + +func (r *registryStub) GetRegistryType() common.RegistryType { + return common.GithubRegistryType +} + +func (r *registryStub) GetRegistryURL() string { + return testRegistryURL +} + +func (r *registryStub) ListCharts() ([]string, error) { + if r.FailListCharts { + return nil, errTest + } + + return testChartList, nil +} + +func (r *registryStub) GetChart(chartName string) (*registry.Chart, error) { + if !r.FailGetChart { + if chartName == testChartName { + return &testChart, nil + } + } + + return nil, errTest +} + +// Deprecated: Use ListCharts, instead. +func (r *registryStub) List() ([]registry.Type, error) { + return []registry.Type{}, nil +} + +// Deprecated: Use GetChart, instead. +func (r *registryStub) GetURLs(t registry.Type) ([]string, error) { + return []string{}, nil +} + +type registryProviderStub struct { + FailGetGithubRegistry bool + FailGetGithubPackageRegistry bool +} + +var testRegistryOwner = "TestOwner" +var testRegistryRepository = "TestRepository" + +func newRegistryProviderStub() *registryProviderStub { + ret := ®istryProviderStub{} + return ret +} + +func (rp *registryProviderStub) GetRegistryByURL(URL string) (registry.Registry, error) { + return newRegistryStub(), nil +} + +func (rp *registryProviderStub) GetRegistryByName(registryName string) (registry.Registry, error) { + return newRegistryStub(), nil +} + var testExpander = &expanderStub{} var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() -var testRegistryService = registry.NewInmemRepositoryService() -var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService) +var testRegistryService = registry.NewInmemRegistryService() +var testProvider = newRegistryProviderStub() +var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService) func TestListDeployments(t *testing.T) { testRepository.reset() diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 3da460c23..99b4b86fa 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -244,7 +244,7 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro if len(m) != 6 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r, err := tr.rp.GetRegistry(template) + r, err := tr.rp.GetRegistryByURL(template) if err != nil { return []string{}, err } @@ -262,7 +262,7 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin if len(m) != 4 { return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } - r, err := tr.rp.GetRegistry(template) + r, err := tr.rp.GetRegistryByURL(template) if err != nil { return []string{}, err } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index 96010f4f8..a2e1d4f85 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -74,26 +74,58 @@ func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndErr return &testRegistryProvider{URLPrefix, r} } +// Deprecated: Use GetRegistryByURL, instead. func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { + return trp.GetRegistryByURL(URL) +} + +func (trp *testRegistryProvider) GetRegistryByURL(URL string) (registry.Registry, error) { for key, r := range trp.r { if strings.HasPrefix(URL, key) { return r, nil } } + return nil, fmt.Errorf("No registry found for %s", URL) } +func (trp *testRegistryProvider) GetRegistryByName(registryName string) (registry.Registry, error) { + return newRegistryStub(), nil +} + type testGithubRegistry struct { responses map[registry.Type]urlAndError count int } +func (r *testGithubRegistry) GetRegistryName() string { + return "" +} + +func (r *testGithubRegistry) GetRegistryType() common.RegistryType { + return common.GithubRegistryType +} + +func (r *testGithubRegistry) GetRegistryURL() string { + return "" +} + +func (r *testGithubRegistry) ListCharts() ([]string, error) { + return []string{}, fmt.Errorf("ListCharts should not be called in the test") +} + +func (r *testGithubRegistry) GetChart(chartName string) (*registry.Chart, error) { + return nil, fmt.Errorf("GetChart should not be called in the test") +} + +// Deprecated: Use GetChart, instead. func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) { tgr.count = tgr.count + 1 ret := tgr.responses[t] return []string{ret.u}, ret.e } +// Deprecated: Use ListCharts, instead. func (tgr *testGithubRegistry) List() ([]registry.Type, error) { return []registry.Type{}, fmt.Errorf("List should not be called in the test") } diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index ba8c903a2..30fbdee04 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -17,10 +17,11 @@ limitations under the License. package registry import ( + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" + "log" "strings" - - "github.com/google/go-github/github" ) // GithubPackageRegistry implements the Registry interface that talks to github and @@ -47,6 +48,37 @@ func NewGithubPackageRegistry(owner, repository string, client *github.Client) * } } +// GetRegistryName returns the name of this registry +func (g *GithubPackageRegistry) GetRegistryName() string { + // TODO(jackgr): implement this method + return "" +} + +// GetRegistryType returns the type of this registry. +func (g *GithubPackageRegistry) GetRegistryType() common.RegistryType { + // TODO(jackgr): implement this method + return common.GithubRegistryType +} + +// GetRegistryURL returns the URL for this registry. +func (g *GithubPackageRegistry) GetRegistryURL() string { + // TODO(jackgr): implement this method + return "" +} + +// ListCharts lists the versioned chart names in this registry. +func (g *GithubPackageRegistry) ListCharts() ([]string, error) { + // TODO(jackgr): implement this method + return []string{}, nil +} + +// GetChart fetches the contents of a given chart. +func (g *GithubPackageRegistry) GetChart(chartName string) (*Chart, error) { + // TODO(jackgr): implement this method + return nil, nil +} + +// Deprecated: Use ListCharts, instead. // List the types from the Registry. func (g *GithubPackageRegistry) List() ([]Type, error) { // Just list all the types at the top level. @@ -74,6 +106,7 @@ func (g *GithubPackageRegistry) List() ([]Type, error) { return retTypes, nil } +// Deprecated: Use GetChart, instead. // GetURLs fetches the download URLs for a given Type. func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) { path, err := g.MakeRepositoryPath(t) diff --git a/registry/github_registry.go b/registry/github_registry.go index c3cf70f01..970340faa 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -18,6 +18,7 @@ package registry import ( "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" "fmt" "log" @@ -64,6 +65,58 @@ func NewGithubRegistry(owner, repository, path string, client *github.Client) *G } } +// GetRegistryName returns the name of this registry +func (g *GithubRegistry) GetRegistryName() string { + // TODO(jackgr): implement this method + return "" +} + +// GetRegistryType returns the type of this registry. +func (g *GithubRegistry) GetRegistryType() common.RegistryType { + // TODO(jackgr): implement this method + return common.GithubRegistryType +} + +// GetRegistryURL returns the URL for this registry. +func (g *GithubRegistry) GetRegistryURL() string { + // TODO(jackgr): implement this method + return "" +} + +// ListCharts lists the versioned chart names in this registry. +func (g *GithubRegistry) ListCharts() ([]string, error) { + var result []string + names, err := g.getDirs("") + if err != nil { + log.Printf("Failed to fetch chart names from registry: %s/%s/%s", g.owner, g.repository, g.path) + return nil, err + } + + // Fetch the chart names + for _, name := range names { + // Then fetch the versions for each chart name + versions, err := g.getDirs("/" + name) + if err != nil { + log.Printf("Failed to fetch versions for chart name: %s/%s/%s/%s", + g.owner, g.repository, g.path, name) + return nil, err + } + + for _, version := range versions { + result = append(result, fmt.Sprintf("%s#%s", name, version)) + } + } + + return result, nil +} + +// GetChart fetches the contents of a given chart. +func (g *GithubRegistry) GetChart(chartName string) (*Chart, error) { + // TODO(jackgr): implement this method + return nil, nil +} + +// Deprecated: Use ListCharts, instead. // List the types from the Registry. func (g *GithubRegistry) List() ([]Type, error) { // First list all the collections at the top level. @@ -98,6 +151,7 @@ func (g *GithubRegistry) List() ([]Type, error) { return retTypes, nil } +// Deprecated: Use GetChart, instead. // GetURL fetches the download URL for a given Type and checks for existence of a schema file. func (g *GithubRegistry) GetURLs(t Type) ([]string, error) { path, err := g.MakeRepositoryPath(t) diff --git a/registry/inmem_repository_service.go b/registry/inmem_registry_service.go similarity index 64% rename from registry/inmem_repository_service.go rename to registry/inmem_registry_service.go index 47845ac26..71de20818 100644 --- a/registry/inmem_repository_service.go +++ b/registry/inmem_registry_service.go @@ -23,53 +23,53 @@ import ( "github.com/kubernetes/deployment-manager/common" ) -type inmemRepositoryService struct { - repositories map[string]*common.Registry +type inmemRegistryService struct { + registries map[string]*common.Registry } -func NewInmemRepositoryService() RegistryService { - rs := &inmemRepositoryService{ - repositories: make(map[string]*common.Registry), +func NewInmemRegistryService() RegistryService { + rs := &inmemRegistryService{ + registries: make(map[string]*common.Registry), } rs.Create(&common.Registry{ Name: "charts", - Type: common.Github, + Type: common.GithubRegistryType, URL: "github.com/helm/charts", Format: common.UnversionedRegistry, }) rs.Create(&common.Registry{ Name: "application-dm-templates", - Type: common.Github, + Type: common.GithubRegistryType, URL: "github.com/kubernetes/application-dm-templates", Format: common.VersionedRegistry, }) return rs } -func (rs *inmemRepositoryService) List() ([]*common.Registry, error) { +func (rs *inmemRegistryService) List() ([]*common.Registry, error) { ret := []*common.Registry{} - for _, r := range rs.repositories { + for _, r := range rs.registries { ret = append(ret, r) } return ret, nil } -func (rs *inmemRepositoryService) Create(repository *common.Registry) error { - rs.repositories[repository.URL] = repository +func (rs *inmemRegistryService) Create(registry *common.Registry) error { + rs.registries[registry.URL] = registry return nil } -func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) { +func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) { return &common.Registry{}, nil } -func (rs *inmemRepositoryService) Delete(name string) error { +func (rs *inmemRegistryService) Delete(name string) error { return nil } // GetByURL returns a registry that handles the types for a given URL. -func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) { - for _, r := range rs.repositories { +func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) { + for _, r := range rs.registries { if strings.HasPrefix(URL, r.URL) { return r, nil } diff --git a/registry/registry.go b/registry/registry.go index 9e8269bb4..44c037d4d 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -20,13 +20,37 @@ import ( "strings" "github.com/kubernetes/deployment-manager/common" + + "net/url" ) +// Registry abstracts a registry that holds charts, which can be +// used in a Deployment Manager configuration. There can be multiple +// registry implementations. +type Registry interface { + // GetRegistryName returns the name of this registry + GetRegistryName() string + // GetRegistryType returns the type of this registry. + GetRegistryType() common.RegistryType + // GetRegistryURL returns the URL for this registry. + GetRegistryURL() string + + // ListCharts lists the versioned chart names in this registry. + ListCharts() ([]string, error) + // GetChart fetches the contents of a given chart. + GetChart(chartName string) (*Chart, error) + + // Deprecated: Use ListCharts, instead. + List() ([]Type, error) + // Deprecated: Use GetChart, instead. + GetURLs(t Type) ([]string, error) +} + type RegistryService interface { // List all the registries List() ([]*common.Registry, error) // Create a new registry - Create(repository *common.Registry) error + Create(registry *common.Registry) error // Get a registry Get(name string) (*common.Registry, error) // Delete a registry @@ -35,13 +59,7 @@ type RegistryService interface { GetByURL(URL string) (*common.Registry, error) } -// Registry abstracts a registry that holds templates, which can be -// used in a Deployment Manager configurations. There can be multiple -// implementations of a registry. Currently we support Deployment Manager -// github.com/kubernetes/application-dm-templates -// and helm packages -// github.com/helm/charts -// +// Deprecated: Use Chart, instead type Type struct { Collection string Name string @@ -68,10 +86,12 @@ func ParseType(name string) *Type { return tt } -// Registry abstracts type interactions. -type Registry interface { - // List all the templates at the given path - List() ([]Type, error) - // Get the download URL(s) for a given type - GetURLs(t Type) ([]string, error) +type Chart struct { + Name string + Version SemVer + RegistryURL string + DownloadURLs []url.URL + + // TODO(jackgr): Should the metadata be strongly typed? + Metadata map[string]interface{} } diff --git a/registry/registry_provider.go b/registry/registry_provider.go new file mode 100644 index 000000000..9e97d62bc --- /dev/null +++ b/registry/registry_provider.go @@ -0,0 +1,114 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "strings" + "sync" + + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" +) + +// RegistryProvider returns factories for creating registry clients. +type RegistryProvider interface { + GetRegistryByURL(URL string) (Registry, error) + GetRegistryByName(registryName string) (Registry, error) +} + +func NewDefaultRegistryProvider() RegistryProvider { + registries := make(map[string]Registry) + rs := NewInmemRegistryService() + return &DefaultRegistryProvider{registries: registries, rs: rs} +} + +type DefaultRegistryProvider struct { + sync.RWMutex + registries map[string]Registry + rs RegistryService +} + +func (drp *DefaultRegistryProvider) GetRegistryByURL(URL string) (Registry, error) { + drp.RLock() + defer drp.RUnlock() + + ghr := drp.findRegistryByURL(URL) + if ghr == nil { + cr, err := drp.rs.GetByURL(URL) + if err != nil { + return nil, err + } + + ghr, err := drp.getGithubRegistry(cr) + if err != nil { + return nil, err + } + + drp.registries[ghr.GetRegistryName()] = ghr + } + + return ghr, nil +} + +func (drp *DefaultRegistryProvider) findRegistryByURL(URL string) Registry { + for _, ghr := range drp.registries { + if strings.HasPrefix(URL, ghr.GetRegistryURL()) { + return ghr + } + } + + return nil +} + +func (drp *DefaultRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { + drp.RLock() + defer drp.RUnlock() + + ghr, ok := drp.registries[registryName] + if !ok { + cr, err := drp.rs.Get(registryName) + if err != nil { + return nil, err + } + + ghr, err := drp.getGithubRegistry(cr) + if err != nil { + return nil, err + } + + drp.registries[ghr.GetRegistryName()] = ghr + } + + return ghr, nil +} + +func (drp *DefaultRegistryProvider) getGithubRegistry(cr *common.Registry) (Registry, error) { + // TODO(jackgr): Take owner and repository from cr instead of hard wiring + if cr.Type == common.GithubRegistryType { + switch cr.Format { + case common.UnversionedRegistry: + return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil + case common.VersionedRegistry: + return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil + default: + return nil, fmt.Errorf("unknown registry format: %s", cr.Format) + } + } + + return nil, fmt.Errorf("unknown registry type: %s", cr.Type) +} diff --git a/registry/registryprovider.go b/registry/registryprovider.go deleted file mode 100644 index f767e896d..000000000 --- a/registry/registryprovider.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package registry - -import ( - "fmt" - - "github.com/google/go-github/github" - "github.com/kubernetes/deployment-manager/common" -) - -// RegistryProvider returns factories for creating registries for a given RegistryType. -type RegistryProvider interface { - GetRegistry(prefix string) (Registry, error) -} - -func NewDefaultRegistryProvider() RegistryProvider { - rs := NewInmemRepositoryService() - return &DefaultRegistryProvider{rs: rs} - -} - -type DefaultRegistryProvider struct { - rs RegistryService -} - -func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) { - r, err := drp.rs.GetByURL(URL) - if err != nil { - return nil, err - } - if r.Type == common.Github { - if r.Format == common.UnversionedRegistry { - return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil - } - if r.Format == common.VersionedRegistry { - return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil - } - } - return nil, fmt.Errorf("cannot find registry backing url %s", URL) -} diff --git a/registry/semver.go b/registry/semver.go new file mode 100644 index 000000000..f5c1ff596 --- /dev/null +++ b/registry/semver.go @@ -0,0 +1,80 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "strconv" + "strings" +) + +type SemVer struct { + Major uint + Minor uint + Patch uint +} + +func NewSemVer(version string) (*SemVer, error) { + result := &SemVer{} + parts := strings.SplitN(version, ".", 3) + if len(parts) > 3 { + return nil, fmt.Errorf("invalid semantic version: %s", version) + } + + major, err := strconv.ParseUint(parts[0], 10, 0) + if err != nil { + return nil, fmt.Errorf("invalid semantic version: %s", version) + } + + result.Major = uint(major) + if len(parts) < 3 { + if len(parts) < 2 { + if len(parts) < 1 { + return nil, fmt.Errorf("invalid semantic version: %s", version) + } + } else { + minor, err := strconv.ParseUint(parts[1], 10, 0) + if err != nil { + return nil, fmt.Errorf("invalid semantic version: %s", version) + } + + result.Minor = uint(minor) + } + } else { + patch, err := strconv.ParseUint(parts[2], 10, 0) + if err != nil { + return nil, fmt.Errorf("invalid semantic version: %s", version) + } + + result.Patch = uint(patch) + } + + return result, nil +} + +func (s *SemVer) String() string { + result := strconv.Itoa(int(s.Major)) + if s.Minor != 0 || s.Patch != 0 { + result = result + "." + strconv.Itoa(int(s.Minor)) + } + + if s.Patch != 0 { + result = result + "." + strconv.Itoa(int(s.Patch)) + } + + return result +} From 4ffe342272729c0bab583fc10e8575f06e649645 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 11 Jan 2016 01:32:45 -0800 Subject: [PATCH 2/6] Work in progress to refactor registry interface. --- common/types.go | 46 ++++- dm/dm.go | 110 +++-------- manager/deployments.go | 53 +++-- manager/manager/expander.go | 16 +- manager/manager/manager.go | 27 +-- manager/manager/manager_test.go | 128 +++--------- manager/manager/registryprovider_test.go | 106 ++++++++++ manager/manager/typeresolver.go | 54 +----- manager/manager/typeresolver_test.go | 143 +------------- registry/github_package_registry.go | 83 +++----- registry/github_registry.go | 187 ++++++++++-------- registry/github_template_registry.go | 200 +++++++++++++++++++ registry/inmem_registry_service.go | 10 +- registry/registry.go | 133 ++++++++----- registry/registry_provider.go | 114 ----------- registry/registry_test.go | 31 ++- registry/registryprovider.go | 237 +++++++++++++++++++++++ registry/registryprovider_test.go | 140 +++++++++++++ registry/semver.go | 67 ++++--- registry/semver_test.go | 91 +++++++++ resourcifier/configurations.go | 6 +- util/httputil.go | 22 ++- util/templateutil.go | 57 ------ 23 files changed, 1261 insertions(+), 800 deletions(-) create mode 100644 manager/manager/registryprovider_test.go create mode 100644 registry/github_template_registry.go delete mode 100644 registry/registry_provider.go create mode 100644 registry/registryprovider.go create mode 100644 registry/registryprovider_test.go create mode 100644 registry/semver_test.go delete mode 100644 util/templateutil.go diff --git a/common/types.go b/common/types.go index 0bdda9e7a..fc6a3a909 100644 --- a/common/types.go +++ b/common/types.go @@ -178,12 +178,21 @@ type RegistryCredential struct { BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` } +// Registry describes a template registry +// TODO(jackr): Fix ambiguity re: whether or not URL has a scheme. type Registry struct { - Name string `json:"name,omitempty"` // Friendly name for the repo - Type RegistryType `json:"type,omitempty"` // Technology implementing the registry - URL string `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts + Name string `json:"name,omitempty"` // Friendly name for the registry + Type RegistryType `json:"type,omitempty"` // Technology implementing the registry + URL string `json:"name,omitempty"` // URL to the root of the registry + Format RegistryFormat `json:"format,omitempty"` // Format of the registry +} + +// AuthenticatedRegistry describes a type registry with credentials. +// Broke this out of Registry, so that we can pass around instances of Registry +// without worrying about secrets. +type AuthenticatedRegistry struct { + Registry Credential RegistryCredential `json:"credential,omitempty"` - Format RegistryFormat `json:"format,omitempty"` } // RegistryType defines the technology that implements the registry @@ -193,10 +202,35 @@ const ( GithubRegistryType RegistryType = "github" ) -// RegistryFormat defines the format of the registry +// RegistryFormat is a semi-colon delimited string that describes the format +// of a registry. type RegistryFormat string const ( - VersionedRegistry RegistryFormat = "versioned" + // Versioning. + // If a registry if versioned, then types appear under versions. + VersionedRegistry RegistryFormat = "versioned" + // If a registry is unversioned, then types appear under their names. UnversionedRegistry RegistryFormat = "unversioned" + + // Organization. + // In a collection registry, types are grouped into collections. + CollectionRegistry RegistryFormat = "collection" + // In a one level registry, all types appear at the top level. + OneLevelRegistry RegistryFormat = "onelevel" ) + +// RegistryService maintains a set of registries that defines the scope of all +// registry based operations, such as search and type resolution. +type RegistryService interface { + // List all the registries + List() ([]*Registry, error) + // Create a new registry + Create(registry *Registry) error + // Get a registry + Get(name string) (*Registry, error) + // Delete a registry + Delete(name string) error + // Find a registry that backs the given URL + GetByURL(URL string) (*Registry, error) +} diff --git a/dm/dm.go b/dm/dm.go index 95f48ffd0..677e8e9bc 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -22,7 +22,6 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/registry" - "github.com/kubernetes/deployment-manager/util" "archive/tar" "bytes" @@ -35,6 +34,7 @@ import ( "net/http" "net/url" "os" + "path" "strconv" "strings" "time" @@ -81,8 +81,15 @@ var usage = func() { panic("\n") } -func getGitRegistry() (registry.Registry, error) { - return registry.NewDefaultRegistryProvider().GetRegistryByURL(*template_registry) +var provider = registry.NewDefaultRegistryProvider() + +func getGithubRegistry() registry.Registry { + git, err := provider.GetRegistryByShortURL(*template_registry) + if err != nil { + panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err)) + } + + return git } func main() { @@ -107,30 +114,21 @@ func execute() { switch args[0] { case "templates": - git, err := getGitRegistry() + git := getGithubRegistry() + types, err := git.ListTypes(nil) if err != nil { - panic(fmt.Errorf("Cannot get registry %v", err)) - } - templates, err := git.List() - if err != nil { - panic(fmt.Errorf("Cannot list %v", err)) + panic(fmt.Errorf("cannot list types in registry %s: %s", *template_registry, err)) } fmt.Printf("Templates:\n") - for _, t := range templates { - var typeSpec = "" - if len(t.Collection) > 0 { - typeSpec = t.Collection + "/" - } - typeSpec = typeSpec + t.Name - if len(t.Version) > 0 { - typeSpec = typeSpec + ":" + t.Version + for _, t := range types { + fmt.Printf("%s\n", t.String()) + urls, err := git.GetDownloadURLs(t) + if err != nil { + panic(fmt.Errorf("cannot get download urls for %s: %s", t, err)) } - fmt.Printf("%s\n", typeSpec) - fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec) - fmt.Printf("\tdownload URL(s):\n") - for _, downloadURL := range getDownloadURLs(t) { + for _, downloadURL := range urls { fmt.Printf("\t%s\n", downloadURL) } } @@ -197,7 +195,7 @@ func execute() { usage() } - tUrls := getTypeURLs(args[1]) + tUrls := getDownloadURLs(args[1]) var tUrl = "" if len(tUrls) == 0 { // Type is most likely a primitive. @@ -268,46 +266,24 @@ func describeType(args []string) { usage() } - tUrls := getTypeURLs(args[1]) + tUrls := getDownloadURLs(args[1]) if len(tUrls) == 0 { panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \":\": %s", args[1])) } + schemaUrl := tUrls[0] + ".schema" fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil)) } -// getTypeURLs returns URLs or empty list if a primitive type. -func getTypeURLs(tName string) []string { - if util.IsHttpUrl(tName) { - // User can pass raw URL to template. - return []string{tName} - } - - // User can pass registry type. - t := getRegistryType(tName) - if t == nil { - // Primitive types have no associated URL. - return []string{} - } - - return getDownloadURLs(*t) -} - -func getDownloadURLs(t registry.Type) []string { - git, err := getGitRegistry() - if err != nil { - panic(fmt.Errorf("Failed to get registry")) - } - urls, err := git.GetURLs(t) +// getDownloadURLs returns URLs or empty list if a primitive type. +func getDownloadURLs(tName string) []string { + qName := path.Join(*template_registry, tName) + result, err := registry.GetDownloadURLs(provider, qName) if err != nil { - panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) + panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err)) } - return urls -} - -func isHttp(t string) bool { - return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://") + return result } func loadTemplate(args []string) *common.Template { @@ -343,8 +319,8 @@ func loadTemplate(args []string) *common.Template { } } else { if len(args) < 3 { - if t := getRegistryType(args[1]); t != nil { - template = buildTemplateFromType(*t) + if t, err := registry.ParseType(args[1]); err == nil { + template = buildTemplateFromType(t) } else { template, err = expander.NewTemplateFromRootTemplate(args[1]) } @@ -365,28 +341,6 @@ func loadTemplate(args []string) *common.Template { return template } -// TODO: needs better validation that this is actually a registry type. -func getRegistryType(fullType string) *registry.Type { - tList := strings.Split(fullType, ":") - if len(tList) != 2 { - return nil - } - - cList := strings.Split(tList[0], "/") - if len(cList) == 1 { - return ®istry.Type{ - Name: tList[0], - Version: tList[1], - } - } else { - return ®istry.Type{ - Collection: cList[0], - Name: cList[1], - Version: tList[1], - } - } -} - func buildTemplateFromType(t registry.Type) *common.Template { props := make(map[string]interface{}) if *properties != "" { @@ -409,11 +363,11 @@ func buildTemplateFromType(t registry.Type) *common.Template { } // Name the deployment after the type name. - name := fmt.Sprintf("%s:%s", t.Name, t.Version) + name := fmt.Sprintf("%s:%s", t.Name, t.GetVersion()) config := common.Configuration{Resources: []*common.Resource{&common.Resource{ Name: name, - Type: getDownloadURLs(t)[0], + Type: getDownloadURLs(t.String())[0], Properties: props, }}} diff --git a/manager/deployments.go b/manager/deployments.go index 1d728c829..bc2cf1563 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "errors" "flag" "fmt" "io" @@ -25,7 +26,9 @@ import ( "log" "net" "net/http" + "net/url" "os" + "regexp" "strings" "github.com/ghodss/yaml" @@ -51,8 +54,8 @@ var deployments = []Route{ {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, {"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""}, - {"ListCharts", "/registries/{registry}/charts", "GET", listChartsHandlerFunc, ""}, - {"GetChart", "/registries/{registry}/charts/{chart}", "GET", getChartHandlerFunc, ""}, + {"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""}, + {"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""}, } var ( @@ -210,13 +213,21 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) { func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) { vars := mux.Vars(r) - ret, ok := vars[variable] + escaped, ok := vars[variable] if !ok { - e := fmt.Errorf("%s parameter not found in URL", variable) + e := errors.New(fmt.Sprintf("%s name not found in URL", variable)) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) return "", e } - return ret, nil + + unescaped, err := url.QueryUnescape(escaped) + if err != nil { + e := fmt.Errorf("cannot decode name (%v)", variable) + util.LogAndReturnError(handler, http.StatusBadRequest, e, w) + return "", e + } + + return unescaped, nil } func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template { @@ -368,37 +379,53 @@ func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK) } -func listChartsHandlerFunc(w http.ResponseWriter, r *http.Request) { - handler := "manager: list charts" +func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: list registry types" util.LogHandlerEntry(handler, r) registryName, err := getPathVariable(w, r, "registry", handler) if err != nil { return } - chartNames, err := backend.ListCharts(registryName) + var regex *regexp.Regexp + regexString, err := getPathVariable(w, r, "regex", handler) + if err == nil { + regex, err = regexp.Compile(regexString) + if err != nil { + util.LogAndReturnError(handler, http.StatusInternalServerError, err, w) + return + } + } + + registryTypes, err := backend.ListRegistryTypes(registryName, regex) if err != nil { util.LogAndReturnError(handler, http.StatusInternalServerError, err, w) return } - util.LogHandlerExitWithJSON(handler, w, chartNames, http.StatusOK) + util.LogHandlerExitWithJSON(handler, w, registryTypes, http.StatusOK) } -func getChartHandlerFunc(w http.ResponseWriter, r *http.Request) { - handler := "manager: get chart" +func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get download URLs" util.LogHandlerEntry(handler, r) registryName, err := getPathVariable(w, r, "registry", handler) if err != nil { return } - chartName, err := getPathVariable(w, r, "chart", handler) + typeName, err := getPathVariable(w, r, "type", handler) + if err != nil { + return + } + + tt, err := registry.ParseType(typeName) if err != nil { + util.LogAndReturnError(handler, http.StatusInternalServerError, err, w) return } - c, err := backend.GetChart(registryName, chartName) + c, err := backend.GetDownloadURLs(registryName, tt) if err != nil { util.LogAndReturnError(handler, http.StatusBadRequest, err, w) return diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 95875ea14..49557ac39 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,6 @@ import ( "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/common" - "github.com/kubernetes/deployment-manager/util" ) const ( @@ -101,7 +100,7 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st for len(toVisit) > 0 { lr := toVisit[0] nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type - if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) { + if len(lr.Layout.Resources) == 0 && isTemplate(lr.Resource.Type, imports) { ret[nodeKey] = lr } else if toReplace[nodeKey] != nil { toReplace[nodeKey].Resources = lr.Resources @@ -113,6 +112,17 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st return ret } +// isTemplate returns whether a given type is a template. +func isTemplate(t string, imports []*common.ImportFile) bool { + for _, imp := range imports { + if imp.Name == t { + return true + } + } + + return false +} + // ExpandTemplate expands the supplied template, and returns a configuration. // It will also update the imports in the provided template if any were added // during type resolution. diff --git a/manager/manager/manager.go b/manager/manager/manager.go index a8ecee6c3..6ea47c19e 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -19,6 +19,8 @@ package manager import ( "fmt" "log" + "net/url" + "regexp" "time" "github.com/kubernetes/deployment-manager/common" @@ -50,9 +52,9 @@ type Manager interface { GetRegistry(name string) (*common.Registry, error) DeleteRegistry(name string) error - // Charts - ListCharts(registryName string) ([]string, error) - GetChart(registryName, chartName string) (*registry.Chart, error) + // Registry Types + ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) + GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) } type manager struct { @@ -60,7 +62,7 @@ type manager struct { deployer Deployer repository repository.Repository provider registry.RegistryProvider - service registry.RegistryService + service common.RegistryService } // NewManager returns a new initialized Manager. @@ -68,7 +70,7 @@ func NewManager(expander Expander, deployer Deployer, repository repository.Repository, provider registry.RegistryProvider, - service registry.RegistryService) Manager { + service common.RegistryService) Manager { return &manager{expander, deployer, repository, provider, service} } @@ -363,22 +365,25 @@ func getResourceErrors(c *common.Configuration) []string { return errs } -// ListCharts retrieves the names of the charts in a given registry. -func (m *manager) ListCharts(registryName string) ([]string, error) { +// ListRegistryTypes lists types in a given registry whose string values +// conform to the supplied regular expression, or all types, if the regular +// expression is nil. +func (m *manager) ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) { r, err := m.provider.GetRegistryByName(registryName) if err != nil { return nil, err } - return r.ListCharts() + return r.ListTypes(regex) } -// GetChart retrieves a given chart in a given registry. -func (m *manager) GetChart(registryName, chartName string) (*registry.Chart, error) { +// GetDownloadURLs returns the URLs required to download the contents +// of a given type in a given registry. +func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) { r, err := m.provider.GetRegistryByName(registryName) if err != nil { return nil, err } - return r.GetChart(chartName) + return r.GetDownloadURLs(t) } diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index 3cfb68891..9d80a438c 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -230,121 +230,31 @@ func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manif return nil, errTest } -func (r *repositoryStub) ListTypes() []string { - r.ListTypesCalled = true +func (tgr *repositoryStub) ListTypes() []string { + tgr.ListTypesCalled = true return []string{} } -func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance { - r.GetTypeInstancesCalled = true +func (tgr *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance { + tgr.GetTypeInstancesCalled = true return []*common.TypeInstance{} } -func (r *repositoryStub) ClearTypeInstances(d string) { - r.TypeInstancesCleared = true +func (tgr *repositoryStub) ClearTypeInstances(d string) { + tgr.TypeInstancesCleared = true } -func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) { +func (tgr *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) { for k, _ := range is { - r.TypeInstances[d] = append(r.TypeInstances[d], k) + tgr.TypeInstances[d] = append(tgr.TypeInstances[d], k) } } -type registryStub struct { - FailListCharts bool - FailGetChart bool - ListChartsCalled bool - GetChartCalled bool -} - -func newRegistryStub() *registryStub { - ret := ®istryStub{} - return ret -} - -func (r *registryStub) reset() { - r.FailListCharts = false - r.FailGetChart = false - r.ListChartsCalled = false - r.GetChartCalled = false -} - -var testRegistryName = "TestRegistry" -var testRegistryURL = "https://github.com/helm/charts" -var testChartName = "TestChart" - -var testChart = registry.Chart{ - Name: testChartName, -} - -var testChartList = []string{testChartName, "TestChart2"} - -func (r *registryStub) GetRegistryName() string { - return testRegistryName -} - -func (r *registryStub) GetRegistryType() common.RegistryType { - return common.GithubRegistryType -} - -func (r *registryStub) GetRegistryURL() string { - return testRegistryURL -} - -func (r *registryStub) ListCharts() ([]string, error) { - if r.FailListCharts { - return nil, errTest - } - - return testChartList, nil -} - -func (r *registryStub) GetChart(chartName string) (*registry.Chart, error) { - if !r.FailGetChart { - if chartName == testChartName { - return &testChart, nil - } - } - - return nil, errTest -} - -// Deprecated: Use ListCharts, instead. -func (r *registryStub) List() ([]registry.Type, error) { - return []registry.Type{}, nil -} - -// Deprecated: Use GetChart, instead. -func (r *registryStub) GetURLs(t registry.Type) ([]string, error) { - return []string{}, nil -} - -type registryProviderStub struct { - FailGetGithubRegistry bool - FailGetGithubPackageRegistry bool -} - -var testRegistryOwner = "TestOwner" -var testRegistryRepository = "TestRepository" - -func newRegistryProviderStub() *registryProviderStub { - ret := ®istryProviderStub{} - return ret -} - -func (rp *registryProviderStub) GetRegistryByURL(URL string) (registry.Registry, error) { - return newRegistryStub(), nil -} - -func (rp *registryProviderStub) GetRegistryByName(registryName string) (registry.Registry, error) { - return newRegistryStub(), nil -} - var testExpander = &expanderStub{} var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() var testRegistryService = registry.NewInmemRegistryService() -var testProvider = newRegistryProviderStub() +var testProvider = newTestRegistryProvider("", nil) var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService) func TestListDeployments(t *testing.T) { @@ -615,3 +525,23 @@ func TestListInstances(t *testing.T) { t.Fatal("expected repository GetTypeInstances() call.") } } + +// TODO(jackgr): Implement TestListRegistryTypes +func TestListRegistryTypes(t *testing.T) { + /* + types, err := testManager.ListRegistryTypes("", nil) + if err != nil { + t.Fatalf("cannot list registry types: %s", err) + } + */ +} + +// TODO(jackgr): Implement TestGetDownloadURLs +func TestGetDownloadURLs(t *testing.T) { + /* + urls, err := testManager.GetDownloadURLs("", registry.Type{}) + if err != nil { + t.Fatalf("cannot list get download urls: %s", err) + } + */ +} diff --git a/manager/manager/registryprovider_test.go b/manager/manager/registryprovider_test.go new file mode 100644 index 000000000..4a4cab423 --- /dev/null +++ b/manager/manager/registryprovider_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/registry" +) + +type urlAndError struct { + u string + e error +} + +type testRegistryProvider struct { + r map[string]registry.Registry +} + +func newTestRegistryProvider(shortURL string, tests map[registry.Type]urlAndError) registry.RegistryProvider { + r := make(map[string]registry.Registry) + r[shortURL] = &testGithubRegistry{tests} + return testRegistryProvider{r} +} + +func (trp testRegistryProvider) GetRegistryByShortURL(URL string) (registry.Registry, error) { + for key, r := range trp.r { + if strings.HasPrefix(URL, key) { + return r, nil + } + } + + return nil, fmt.Errorf("No registry found for %s", URL) +} + +func (trp testRegistryProvider) GetRegistryByName(registryName string) (registry.Registry, error) { + panic(fmt.Errorf("GetRegistryByName should not be called in the test")) +} + +func (trp testRegistryProvider) GetGithubRegistry(cr common.Registry) (registry.GithubRegistry, error) { + panic(fmt.Errorf("GetGithubRegistry should not be called in the test")) +} + +type testGithubRegistry struct { + responses map[registry.Type]urlAndError +} + +func (tgr testGithubRegistry) GetRegistryName() string { + panic(fmt.Errorf("GetRegistryName should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryType() common.RegistryType { + return common.GithubRegistryType +} + +func (tgr testGithubRegistry) GetRegistryShortURL() string { + panic(fmt.Errorf("GetRegistryShortURL should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryFormat() common.RegistryFormat { + panic(fmt.Errorf("GetRegistryFormat should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryOwner() string { + panic(fmt.Errorf("GetRegistryOwner should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryRepository() string { + panic(fmt.Errorf("GetRegistryRepository should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryPath() string { + panic(fmt.Errorf("GetRegistryPath should not be called in the test")) +} + +func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]registry.Type, error) { + panic(fmt.Errorf("ListTypes should not be called in the test")) +} + +func (tgr testGithubRegistry) GetDownloadURLs(t registry.Type) ([]*url.URL, error) { + ret := tgr.responses[t] + URL, err := url.Parse(ret.u) + if err != nil { + panic(err) + } + + return []*url.URL{URL}, ret.e +} diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 99b4b86fa..5869558c3 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -100,7 +100,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co toFetch := make([]*fetchUnit, 0, tr.maxUrls) for _, r := range config.Resources { // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). - urls, err := tr.MapFetchableURLs(r.Type) + urls, err := registry.GetDownloadURLs(tr.rp, r.Type) if err != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err)) } @@ -168,7 +168,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co for _, v := range s.Imports { i := &common.ImportFile{Name: v.Name} var existingSchema string - urls, conversionErr := tr.MapFetchableURLs(v.Path) + urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path) if conversionErr != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr)) } @@ -220,56 +220,6 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co return ret, nil } -// MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL -// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for -// example) will return empty string. -func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) { - if util.IsGithubShortType(t) { - return tr.ShortTypeToDownloadURLs(t) - } else if util.IsGithubShortPackageType(t) { - return tr.ShortTypeToPackageDownloadURLs(t) - } else if util.IsHttpUrl(t) { - return []string{t}, nil - } - return []string{}, nil -} - -// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github. -// Input must be of the type and is assumed to have been validated before this call: -// github.com/owner/repo/qualifier/type:version -// for example: -// github.com/kubernetes/application-dm-templates/storage/redis:v1 -func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) { - m := util.TemplateRegistryMatcher.FindStringSubmatch(template) - if len(m) != 6 { - return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) - } - r, err := tr.rp.GetRegistryByURL(template) - if err != nil { - return []string{}, err - } - t := registry.Type{m[3], m[4], m[5]} - return r.GetURLs(t) -} - -// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github. -// Input must be of the type and is assumed to have been validated before this call: -// github.com/owner/repo/type -// for example: -// github.com/helm/charts/cassandra -func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) { - m := util.PackageRegistryMatcher.FindStringSubmatch(template) - if len(m) != 4 { - return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) - } - r, err := tr.rp.GetRegistryByURL(template) - if err != nil { - return []string{}, err - } - t := registry.Type{Name: m[3]} - return r.GetURLs(t) -} - func parseContent(templates []string) (string, error) { if len(templates) == 1 { return templates[0], nil diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index a2e1d4f85..321564c23 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -18,7 +18,6 @@ package manager import ( "errors" - "fmt" "net/http" "reflect" "strings" @@ -58,108 +57,6 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) { return ret.resp, ret.code, ret.err } -type urlAndError struct { - u string - e error -} - -type testRegistryProvider struct { - URLPrefix string - r map[string]registry.Registry -} - -func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { - r := make(map[string]registry.Registry) - r[URLPrefix] = &testGithubRegistry{tests, count} - return &testRegistryProvider{URLPrefix, r} -} - -// Deprecated: Use GetRegistryByURL, instead. -func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) { - return trp.GetRegistryByURL(URL) -} - -func (trp *testRegistryProvider) GetRegistryByURL(URL string) (registry.Registry, error) { - for key, r := range trp.r { - if strings.HasPrefix(URL, key) { - return r, nil - } - } - - return nil, fmt.Errorf("No registry found for %s", URL) -} - -func (trp *testRegistryProvider) GetRegistryByName(registryName string) (registry.Registry, error) { - return newRegistryStub(), nil -} - -type testGithubRegistry struct { - responses map[registry.Type]urlAndError - count int -} - -func (r *testGithubRegistry) GetRegistryName() string { - return "" -} - -func (r *testGithubRegistry) GetRegistryType() common.RegistryType { - return common.GithubRegistryType -} - -func (r *testGithubRegistry) GetRegistryURL() string { - return "" -} - -func (r *testGithubRegistry) ListCharts() ([]string, error) { - return []string{}, fmt.Errorf("ListCharts should not be called in the test") -} - -func (r *testGithubRegistry) GetChart(chartName string) (*registry.Chart, error) { - return nil, fmt.Errorf("GetChart should not be called in the test") -} - -// Deprecated: Use GetChart, instead. -func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) { - tgr.count = tgr.count + 1 - ret := tgr.responses[t] - return []string{ret.u}, ret.e -} - -// Deprecated: Use ListCharts, instead. -func (tgr *testGithubRegistry) List() ([]registry.Type, error) { - return []registry.Type{}, fmt.Errorf("List should not be called in the test") -} - -type testGithubPackageRegistry struct { - responses map[registry.Type]urlAndError - count int -} - -func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) { - tgr.count = tgr.count + 1 - ret := tgr.responses[t] - return []string{ret.u}, ret.e -} - -func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) { - return []registry.Type{}, fmt.Errorf("List should not be called in the test") -} - -func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) { - r := &typeResolver{ - rp: c.registryProvider, - } - for in, expected := range tests { - actual, err := r.ShortTypeToDownloadURLs(in) - if err != expected.e { - t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) - } - if actual[0] != expected.u { - t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual) - } - } -} - func testDriver(c resolverTestCase, t *testing.T) { g := &testGetter{test: t, responses: c.responses} r := &typeResolver{ @@ -376,40 +273,6 @@ func TestSharedImport(t *testing.T) { testDriver(test, t) } -func TestShortGithubUrlMapping(t *testing.T) { - githubUrlMaps := map[registry.Type]urlAndError{ - registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, - } - - tests := map[string]urlAndError{ - "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, - } - - test := resolverTestCase{ - registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), - } - testUrlConversionDriver(test, tests, t) -} - -func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { - githubUrlMaps := map[registry.Type]urlAndError{ - registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, - } - - tests := map[string]urlAndError{ - "github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, - } - - test := resolverTestCase{ - registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2), - } - testUrlConversionDriver(test, tests, t) -} - var templateShortGithubTemplate = ` resources: - name: foo @@ -438,8 +301,8 @@ func TestShortGithubUrl(t *testing.T) { } githubUrlMaps := map[registry.Type]urlAndError{ - registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, + registry.NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.NewTypeOrDie("common", "replicatedservice", "v2"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, } test := resolverTestCase{ @@ -447,7 +310,7 @@ func TestShortGithubUrl(t *testing.T) { importOut: finalImports, urlcount: 4, responses: responses, - registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2), + registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps), } testDriver(test, t) } diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 30fbdee04..681eeef18 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -20,12 +20,15 @@ import ( "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" + "fmt" "log" + "net/url" + "regexp" "strings" ) // GithubPackageRegistry implements the Registry interface that talks to github and -// expects packages in helm format without versioning and no qualifier in the path. +// expects packages in helm format without versioning and no collection in the path. // Format of the directory for a type is like so: // package/ // Chart.yaml @@ -34,53 +37,23 @@ import ( // bar.yaml // ... type GithubPackageRegistry struct { - owner string - repository string - client *github.Client + githubRegistry } -// NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry { - return &GithubPackageRegistry{ - owner: owner, - repository: repository, - client: client, +// NewGithubPackageRegistry creates a GithubPackageRegistry. +func NewGithubPackageRegistry(name, shortURL string, client *github.Client) (GithubPackageRegistry, error) { + format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), client) + if err != nil { + return GithubPackageRegistry{}, err } -} - -// GetRegistryName returns the name of this registry -func (g *GithubPackageRegistry) GetRegistryName() string { - // TODO(jackgr): implement this method - return "" -} - -// GetRegistryType returns the type of this registry. -func (g *GithubPackageRegistry) GetRegistryType() common.RegistryType { - // TODO(jackgr): implement this method - return common.GithubRegistryType -} - -// GetRegistryURL returns the URL for this registry. -func (g *GithubPackageRegistry) GetRegistryURL() string { - // TODO(jackgr): implement this method - return "" -} -// ListCharts lists the versioned chart names in this registry. -func (g *GithubPackageRegistry) ListCharts() ([]string, error) { - // TODO(jackgr): implement this method - return []string{}, nil + return GithubPackageRegistry{githubRegistry: gr}, nil } -// GetChart fetches the contents of a given chart. -func (g *GithubPackageRegistry) GetChart(chartName string) (*Chart, error) { - // TODO(jackgr): implement this method - return nil, nil -} - -// Deprecated: Use ListCharts, instead. -// List the types from the Registry. -func (g *GithubPackageRegistry) List() ([]Type, error) { +// ListTypes lists types in this registry whose string values conform to the +// supplied regular expression, or all types, if the regular expression is nil. +func (g GithubPackageRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { // Just list all the types at the top level. types, err := g.getDirs("") if err != nil { @@ -103,33 +76,41 @@ func (g *GithubPackageRegistry) List() ([]Type, error) { } } + // TODO(jackgr): Use the supplied regex to filter the results. return retTypes, nil } -// Deprecated: Use GetChart, instead. -// GetURLs fetches the download URLs for a given Type. -func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) { +// GetDownloadURLs fetches the download URLs for a given Type. +func (g GithubPackageRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { path, err := g.MakeRepositoryPath(t) if err != nil { - return []string{}, err + return nil, err } + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to list package files at path: %s: %v", path, err) - return []string{}, err + return nil, err } - downloadURLs := []string{} + + downloadURLs := []*url.URL{} for _, f := range dc { if *f.Type == "file" { if strings.HasSuffix(*f.Name, ".yaml") { - downloadURLs = append(downloadURLs, *f.DownloadURL) + u, err := url.Parse(*f.DownloadURL) + if err != nil { + return nil, fmt.Errorf("cannot parse URL from %s: %s", *f.DownloadURL, err) + } + + downloadURLs = append(downloadURLs, u) } } } + return downloadURLs, nil } -func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) { +func (g GithubPackageRegistry) getDirs(dir string) ([]string, error) { _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil) if err != nil { log.Printf("Failed to get contents at path: %s: %v", dir, err) @@ -149,7 +130,7 @@ func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) { // MakeRepositoryPath constructs a github path to a given type based on a repository, and type name. // The returned repository path will be of the form: // Type.Name/manifests -func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) { +func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) { // Construct the return path return t.Name + "/manifests", nil } diff --git a/registry/github_registry.go b/registry/github_registry.go index 970340faa..5f25aa1ae 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -19,110 +19,110 @@ package registry import ( "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" "fmt" "log" + "net/url" + "regexp" "strings" ) -// GithubRegistry implements the Registry interface that talks to github and -// implements Deployment Manager templates registry. -// A registry root must be a directory that contains all the available templates, -// one directory per template. Each template directory then contains version -// directories, each of which in turn contains all the files necessary for that -// version of the template. -// -// For example, a template registry containing two versions of redis -// (implemented in jinja), and one version of replicatedservice (implemented -// in python) would have a directory structure that looks something like this: -// qualifier [optional] prefix to a virtual root within the repository. -// /redis -// /v1 -// redis.jinja -// redis.jinja.schema -// /v2 -// redis.jinja -// redis.jinja.schema -// /replicatedservice -// /v1 -// replicatedservice.python -// replicatedservice.python.schema - -type GithubRegistry struct { +// githubRegistry implements the Registry interface and talks to github. +// The registry short URL and format determine how types are laid out in the +// registry. +type githubRegistry struct { + name string + shortURL string owner string repository string path string + format common.RegistryFormat client *github.Client } -// NewGithubRegistry creates a Registry that can be used to talk to github. -func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry { - return &GithubRegistry{ +// newGithubRegistry creates a githubRegistry. +func newGithubRegistry(name, shortURL string, format common.RegistryFormat, client *github.Client) (githubRegistry, error) { + trimmed := util.TrimURLScheme(shortURL) + owner, repository, path, err := parseGithubShortURL(trimmed) + if err != nil { + return githubRegistry{}, fmt.Errorf("cannot create Github template registry %s: %s", name, err) + } + + return githubRegistry{ + name: name, + shortURL: trimmed, owner: owner, repository: repository, path: path, + format: format, client: client, + }, nil +} + +func parseGithubShortURL(shortURL string) (string, string, string, error) { + if !strings.HasPrefix(shortURL, "github.com/") { + return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL) } + + tPath := strings.TrimPrefix(shortURL, "github.com/") + parts := strings.Split(tPath, "/") + + // Handle the case where there's no path after owner and repository. + if len(parts) == 2 { + return parts[0], parts[1], "", nil + } + + // Handle the case where there's a path after owner and repository. + if len(parts) == 3 { + return parts[0], parts[1], parts[2], nil + } + + return "", "", "", fmt.Errorf("invalid Github short URL: %s", shortURL) } // GetRegistryName returns the name of this registry -func (g *GithubRegistry) GetRegistryName() string { - // TODO(jackgr): implement this method - return "" +func (g githubRegistry) GetRegistryName() string { + return g.name } // GetRegistryType returns the type of this registry. -func (g *GithubRegistry) GetRegistryType() common.RegistryType { - // TODO(jackgr): implement this method +func (g githubRegistry) GetRegistryType() common.RegistryType { return common.GithubRegistryType } -// GetRegistryURL returns the URL for this registry. -func (g *GithubRegistry) GetRegistryURL() string { - // TODO(jackgr): implement this method - return "" +// GetRegistryShortURL returns the short URL for this registry. +func (g githubRegistry) GetRegistryShortURL() string { + return g.shortURL } -// ListCharts lists the versioned chart names in this registry. -func (g *GithubRegistry) ListCharts() ([]string, error) { - var result []string - names, err := g.getDirs("") - if err != nil { - log.Printf("Failed to fetch chart names from registry: %s/%s/%s", g.owner, g.repository, g.path) - return nil, err - } - - // Fetch the chart names - for _, name := range names { - // Then fetch the versions for each chart name - versions, err := g.getDirs("/" + name) - if err != nil { - log.Printf("Failed to fetch versions for chart name: %s/%s/%s/%s", - g.owner, g.repository, g.path, name) - return nil, err - } +// GetRegistryFormat returns the format of this registry. +func (g githubRegistry) GetRegistryFormat() common.RegistryFormat { + return g.format +} - for _, version := range versions { - result = append(result, fmt.Sprintf("%s#%s", name, version)) - } - } +// GetRegistryOwner returns the owner name for this registry +func (g githubRegistry) GetRegistryOwner() string { + return g.owner +} - return result, nil +// GetRegistryRepository returns the repository name for this registry. +func (g githubRegistry) GetRegistryRepository() string { + return g.repository } -// GetChart fetches the contents of a given chart. -func (g *GithubRegistry) GetChart(chartName string) (*Chart, error) { - // TODO(jackgr): implement this method - return nil, nil +// GetRegistryName returns the name of this registry +func (g githubRegistry) GetRegistryPath() string { + return g.path } -// Deprecated: Use ListCharts, instead. -// List the types from the Registry. -func (g *GithubRegistry) List() ([]Type, error) { +// ListTypes lists types in this registry whose string values conform to the +// supplied regular expression, or all types, if the regular expression is nil. +func (g githubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { // First list all the collections at the top level. collections, err := g.getDirs("") if err != nil { - log.Printf("Failed to list qualifiers: %v", err) + log.Printf("cannot list qualifiers: %v", err) return nil, err } @@ -131,38 +131,45 @@ func (g *GithubRegistry) List() ([]Type, error) { // Then we need to fetch the versions (directories for this type) types, err := g.getDirs(c) if err != nil { - log.Printf("Failed to fetch types for collection: %s", c) + log.Printf("cannot fetch types for collection: %s", c) return nil, err } for _, t := range types { + path := c + "/" + t // Then we need to fetch the versions (directories for this type) - versions, err := g.getDirs(c + "/" + t) + versions, err := g.getDirs(path) if err != nil { - log.Printf("Failed to fetch versions for template: %s", t) + log.Printf("cannot fetch versions at path %s", path) return nil, err } + for _, v := range versions { - retTypes = append(retTypes, Type{Name: t, Version: v, Collection: c}) + tt, err := NewType(c, t, v) + if err != nil { + return nil, fmt.Errorf("malformed type at path %s", path) + } + + retTypes = append(retTypes, tt) } } } + // TODO(jackgr): Use the supplied regex to filter the results. return retTypes, nil } -// Deprecated: Use GetChart, instead. -// GetURL fetches the download URL for a given Type and checks for existence of a schema file. -func (g *GithubRegistry) GetURLs(t Type) ([]string, error) { +// GetDownloadURLs fetches the download URLs for a given Type and checks for existence of a schema file. +func (g githubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { path, err := g.MakeRepositoryPath(t) if err != nil { - return []string{}, err + return nil, err } _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) if err != nil { - log.Printf("Failed to list versions at path: %s: %v", path, err) - return []string{}, err + return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err) } + var downloadURL, typeName, schemaName string for _, f := range dc { if *f.Type == "file" { @@ -175,16 +182,24 @@ func (g *GithubRegistry) GetURLs(t Type) ([]string, error) { } } } + if downloadURL == "" { - return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version) + return nil, fmt.Errorf("cannot find type %s", t.String()) + } + + if schemaName != typeName+".schema" { + return nil, fmt.Errorf("cannot find schema for %s, expected %s", t.String(), typeName+".schema") } - if schemaName == typeName+".schema" { - return []string{downloadURL}, nil + + result, err := url.Parse(downloadURL) + if err != nil { + return nil, fmt.Errorf("cannot parse URL from %s: %s", downloadURL, err) } - return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema") + + return []*url.URL{result}, nil } -func (g *GithubRegistry) getDirs(dir string) ([]string, error) { +func (g githubRegistry) getDirs(dir string) ([]string, error) { var path = g.path if dir != "" { path = g.path + "/" + dir @@ -206,7 +221,7 @@ func (g *GithubRegistry) getDirs(dir string) ([]string, error) { return dirs, nil } -func (g *GithubRegistry) mapCollection(collection string) (string, error) { +func (g githubRegistry) mapCollection(collection string) (string, error) { if strings.ContainsAny(collection, "/") { return "", fmt.Errorf("collection must not contain slashes, got %s", collection) } @@ -216,10 +231,10 @@ func (g *GithubRegistry) mapCollection(collection string) (string, error) { // MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version. // The returned repository path will be of the form: -// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version +// [githubRegistry.path/][Type.Collection]/Type.Name/Type.Version // Type.Collection will be mapped using mapCollection in the future, for now it's a straight // 1:1 mapping (if given) -func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) { +func (g githubRegistry) MakeRepositoryPath(t Type) (string, error) { // First map the collection collection, err := g.mapCollection(t.Collection) if err != nil { @@ -233,5 +248,5 @@ func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) { if len(collection) > 0 { p += collection + "/" } - return p + t.Name + "/" + t.Version, nil + return p + t.Name + "/" + t.GetVersion(), nil } diff --git a/registry/github_template_registry.go b/registry/github_template_registry.go new file mode 100644 index 000000000..586e22118 --- /dev/null +++ b/registry/github_template_registry.go @@ -0,0 +1,200 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" + + "fmt" + "log" + "net/url" + "regexp" + "strings" +) + +// GithubTemplateRegistry implements the Registry interface and implements a +// Deployment Manager templates registry. +// A registry root must be a directory that contains all the available templates, +// one directory per template. Each template directory then contains version +// directories, each of which in turn contains all the files necessary for that +// version of the template. +// +// For example, a template registry containing two versions of redis +// (implemented in jinja), and one version of replicatedservice (implemented +// in python) would have a directory structure that looks something like this: +// qualifier [optional] prefix to a virtual root within the repository. +// /redis +// /v1 +// redis.jinja +// redis.jinja.schema +// /v2 +// redis.jinja +// redis.jinja.schema +// /replicatedservice +// /v1 +// replicatedservice.python +// replicatedservice.python.schema +type GithubTemplateRegistry struct { + githubRegistry +} + +// NewGithubTemplateRegistry creates a GithubTemplateRegistry. +func NewGithubTemplateRegistry(name, shortURL string, client *github.Client) (GithubTemplateRegistry, error) { + format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), client) + if err != nil { + return GithubTemplateRegistry{}, err + } + + return GithubTemplateRegistry{githubRegistry: gr}, nil +} + +// ListTypes lists types in this registry whose string values conform to the +// supplied regular expression, or all types, if the regular expression is nil. +func (g GithubTemplateRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { + // First list all the collections at the top level. + collections, err := g.getDirs("") + if err != nil { + log.Printf("cannot list qualifiers: %v", err) + return nil, err + } + + var retTypes []Type + for _, c := range collections { + // Then we need to fetch the versions (directories for this type) + types, err := g.getDirs(c) + if err != nil { + log.Printf("cannot fetch types for collection: %s", c) + return nil, err + } + + for _, t := range types { + path := c + "/" + t + // Then we need to fetch the versions (directories for this type) + versions, err := g.getDirs(path) + if err != nil { + log.Printf("cannot fetch versions at path %s", path) + return nil, err + } + + for _, v := range versions { + tt, err := NewType(c, t, v) + if err != nil { + return nil, fmt.Errorf("malformed type at path %s", path) + } + + retTypes = append(retTypes, tt) + } + } + } + + // TODO(jackgr): Use the supplied regex to filter the results. + return retTypes, nil +} + +// GetDownloadURLs fetches the download URLs for a given Type and checks for existence of a schema file. +func (g GithubTemplateRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { + path, err := g.MakeRepositoryPath(t) + if err != nil { + return nil, err + } + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + if err != nil { + return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err) + } + + var downloadURL, typeName, schemaName string + for _, f := range dc { + if *f.Type == "file" { + if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" { + typeName = *f.Name + downloadURL = *f.DownloadURL + } + if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" { + schemaName = *f.Name + } + } + } + + if downloadURL == "" { + return nil, fmt.Errorf("cannot find type %s", t.String()) + } + + if schemaName != typeName+".schema" { + return nil, fmt.Errorf("cannot find schema for %s, expected %s", t.String(), typeName+".schema") + } + + result, err := url.Parse(downloadURL) + if err != nil { + return nil, fmt.Errorf("cannot parse URL from %s: %s", downloadURL, err) + } + + return []*url.URL{result}, nil +} + +func (g GithubTemplateRegistry) getDirs(dir string) ([]string, error) { + var path = g.path + if dir != "" { + path = g.path + "/" + dir + } + + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + if err != nil { + log.Printf("Failed to get contents at path: %s: %v", path, err) + return nil, err + } + + var dirs []string + for _, entry := range dc { + if *entry.Type == "dir" { + dirs = append(dirs, *entry.Name) + } + } + + return dirs, nil +} + +func (g GithubTemplateRegistry) mapCollection(collection string) (string, error) { + if strings.ContainsAny(collection, "/") { + return "", fmt.Errorf("collection must not contain slashes, got %s", collection) + } + // TODO(vaikas): Implement lookup from the root metadata file to map collection to a path + return collection, nil +} + +// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version. +// The returned repository path will be of the form: +// [GithubTemplateRegistry.path/][Type.Collection]/Type.Name/Type.Version +// Type.Collection will be mapped using mapCollection in the future, for now it's a straight +// 1:1 mapping (if given) +func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) { + // First map the collection + collection, err := g.mapCollection(t.Collection) + if err != nil { + return "", err + } + // Construct the return path + p := "" + if len(g.path) > 0 { + p += g.path + "/" + } + if len(collection) > 0 { + p += collection + "/" + } + return p + t.Name + "/" + t.GetVersion(), nil +} diff --git a/registry/inmem_registry_service.go b/registry/inmem_registry_service.go index 71de20818..8affeea14 100644 --- a/registry/inmem_registry_service.go +++ b/registry/inmem_registry_service.go @@ -27,21 +27,25 @@ type inmemRegistryService struct { registries map[string]*common.Registry } -func NewInmemRegistryService() RegistryService { +func NewInmemRegistryService() common.RegistryService { rs := &inmemRegistryService{ registries: make(map[string]*common.Registry), } + + pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) rs.Create(&common.Registry{ Name: "charts", Type: common.GithubRegistryType, URL: "github.com/helm/charts", - Format: common.UnversionedRegistry, + Format: common.RegistryFormat(pFormat), }) + + tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) rs.Create(&common.Registry{ Name: "application-dm-templates", Type: common.GithubRegistryType, URL: "github.com/kubernetes/application-dm-templates", - Format: common.VersionedRegistry, + Format: common.RegistryFormat(tFormat), }) return rs } diff --git a/registry/registry.go b/registry/registry.go index 44c037d4d..0f703f5e4 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -1,7 +1,7 @@ /* Copyright 2015 The Kubernetes Authors All rights reserved. -Licensed under the Apache License, Version 2.0 (the "License"); +Licensed under the Apache License, version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,11 +17,12 @@ limitations under the License. package registry import ( - "strings" - "github.com/kubernetes/deployment-manager/common" + "fmt" "net/url" + "regexp" + "strings" ) // Registry abstracts a registry that holds charts, which can be @@ -32,66 +33,110 @@ type Registry interface { GetRegistryName() string // GetRegistryType returns the type of this registry. GetRegistryType() common.RegistryType - // GetRegistryURL returns the URL for this registry. - GetRegistryURL() string - - // ListCharts lists the versioned chart names in this registry. - ListCharts() ([]string, error) - // GetChart fetches the contents of a given chart. - GetChart(chartName string) (*Chart, error) - - // Deprecated: Use ListCharts, instead. - List() ([]Type, error) - // Deprecated: Use GetChart, instead. - GetURLs(t Type) ([]string, error) + // GetRegistryShortURL returns the short URL for this registry. + GetRegistryShortURL() string + // GetRegistryFormat returns the format of this registry. + GetRegistryFormat() common.RegistryFormat + + // ListTypes lists types in this registry whose string values conform to the + // supplied regular expression, or all types, if the regular expression is nil. + ListTypes(regex *regexp.Regexp) ([]Type, error) + // GetDownloadURLs returns the URLs required to download the type contents. + GetDownloadURLs(t Type) ([]*url.URL, error) } -type RegistryService interface { - // List all the registries - List() ([]*common.Registry, error) - // Create a new registry - Create(registry *common.Registry) error - // Get a registry - Get(name string) (*common.Registry, error) - // Delete a registry - Delete(name string) error - // Find a registry that backs the given URL - GetByURL(URL string) (*common.Registry, error) +// GithubRegistry abstracts a registry that resides in a Github repository. +type GithubRegistry interface { + Registry // A GithubRegistry is a Registry. + // GetRegistryOwner returns the owner name for this registry + GetRegistryOwner() string + // GetRegistryRepository returns the repository name for this registry. + GetRegistryRepository() string + // GetRegistryPath returns the path to the registry in the repository. + GetRegistryPath() string } -// Deprecated: Use Chart, instead type Type struct { Collection string Name string - Version string + version SemVer } -// ParseType takes a registry name and parses it into a *registry.Type. -func ParseType(name string) *Type { - tt := &Type{} +// NewType initializes a type +func NewType(collection, name, version string) (Type, error) { + result := Type{Collection: collection, Name: name} + err := result.SetVersion(version) + return result, err +} - tList := strings.Split(name, ":") +// NewTypeOrDie initializes a type and panics if initialization fails +func NewTypeOrDie(collection, name, version string) Type { + result, err := NewType(collection, name, version) + if err != nil { + panic(err) + } + + return result +} + +// Type conforms to the Stringer interface. +func (t Type) String() string { + var result string + if t.Collection != "" { + result = t.Collection + "/" + } + + result = result + t.Name + version := t.GetVersion() + if version != "" && version != "v0" { + result = result + ":" + version + } + + return result +} + +// GetVersion returns the type version with the letter "v" prepended. +func (t Type) GetVersion() string { + var result string + version := t.version.String() + if version != "0" { + result = "v" + version + } + + return result +} + +// SetVersion strips the letter "v" from version, if present, +// and sets the the version of the type to the result. +func (t *Type) SetVersion(version string) error { + vstring := strings.TrimPrefix(version, "v") + s, err := ParseSemVer(vstring) + if err != nil { + return err + } + + t.version = s + return nil +} + +// ParseType takes a registry type string and parses it into a *registry.Type. +// TODO: needs better validation that this is actually a registry type. +func ParseType(ts string) (Type, error) { + tt := Type{} + tList := strings.Split(ts, ":") if len(tList) == 2 { - tt.Version = tList[1] + if err := tt.SetVersion(tList[1]); err != nil { + return tt, fmt.Errorf("malformed type string: %s", ts) + } } cList := strings.Split(tList[0], "/") - if len(cList) == 1 { tt.Name = tList[0] } else { tt.Collection = cList[0] tt.Name = cList[1] } - return tt -} - -type Chart struct { - Name string - Version SemVer - RegistryURL string - DownloadURLs []url.URL - // TODO(jackgr): Should the metadata be strongly typed? - Metadata map[string]interface{} + return tt, nil } diff --git a/registry/registry_provider.go b/registry/registry_provider.go deleted file mode 100644 index 9e97d62bc..000000000 --- a/registry/registry_provider.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package registry - -import ( - "fmt" - "strings" - "sync" - - "github.com/google/go-github/github" - "github.com/kubernetes/deployment-manager/common" -) - -// RegistryProvider returns factories for creating registry clients. -type RegistryProvider interface { - GetRegistryByURL(URL string) (Registry, error) - GetRegistryByName(registryName string) (Registry, error) -} - -func NewDefaultRegistryProvider() RegistryProvider { - registries := make(map[string]Registry) - rs := NewInmemRegistryService() - return &DefaultRegistryProvider{registries: registries, rs: rs} -} - -type DefaultRegistryProvider struct { - sync.RWMutex - registries map[string]Registry - rs RegistryService -} - -func (drp *DefaultRegistryProvider) GetRegistryByURL(URL string) (Registry, error) { - drp.RLock() - defer drp.RUnlock() - - ghr := drp.findRegistryByURL(URL) - if ghr == nil { - cr, err := drp.rs.GetByURL(URL) - if err != nil { - return nil, err - } - - ghr, err := drp.getGithubRegistry(cr) - if err != nil { - return nil, err - } - - drp.registries[ghr.GetRegistryName()] = ghr - } - - return ghr, nil -} - -func (drp *DefaultRegistryProvider) findRegistryByURL(URL string) Registry { - for _, ghr := range drp.registries { - if strings.HasPrefix(URL, ghr.GetRegistryURL()) { - return ghr - } - } - - return nil -} - -func (drp *DefaultRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { - drp.RLock() - defer drp.RUnlock() - - ghr, ok := drp.registries[registryName] - if !ok { - cr, err := drp.rs.Get(registryName) - if err != nil { - return nil, err - } - - ghr, err := drp.getGithubRegistry(cr) - if err != nil { - return nil, err - } - - drp.registries[ghr.GetRegistryName()] = ghr - } - - return ghr, nil -} - -func (drp *DefaultRegistryProvider) getGithubRegistry(cr *common.Registry) (Registry, error) { - // TODO(jackgr): Take owner and repository from cr instead of hard wiring - if cr.Type == common.GithubRegistryType { - switch cr.Format { - case common.UnversionedRegistry: - return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil - case common.VersionedRegistry: - return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil - default: - return nil, fmt.Errorf("unknown registry format: %s", cr.Format) - } - } - - return nil, fmt.Errorf("unknown registry type: %s", cr.Type) -} diff --git a/registry/registry_test.go b/registry/registry_test.go index 9f9ce5e06..53bf60eda 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -4,25 +4,36 @@ import ( "testing" ) -func TestParseType(t *testing.T) { - // TODO: Are there some real-world examples we want to valide here? - tests := map[string]*Type{ - "foo": &Type{Name: "foo"}, - "foo:v1": &Type{Name: "foo", Version: "v1"}, - "github.com/foo": &Type{Name: "foo", Collection: "github.com"}, - "github.com/foo:v1.2.3": &Type{Name: "foo", Collection: "github.com", Version: "v1.2.3"}, +func TestTypeConversion(t *testing.T) { + // TODO: Are there some real-world examples we want to validate here? + tests := map[string]Type{ + "foo": NewTypeOrDie("", "foo", ""), + "foo:v1": NewTypeOrDie("", "foo", "v1"), + "github.com/foo": NewTypeOrDie("github.com", "foo", ""), + "github.com/foo:v1.2.3": NewTypeOrDie("github.com", "foo", "v1.2.3"), } for in, expect := range tests { - out := ParseType(in) + out, err := ParseType(in) + if err != nil { + t.Errorf("Error parsing type string %s: $s", in, err) + } + if out.Name != expect.Name { t.Errorf("Expected name to be %q, got %q", expect.Name, out.Name) } - if out.Version != expect.Version { - t.Errorf("Expected version to be %q, got %q", expect.Version, out.Version) + + if out.GetVersion() != expect.GetVersion() { + t.Errorf("Expected version to be %q, got %q", expect.GetVersion(), out.GetVersion()) } + if out.Collection != expect.Collection { t.Errorf("Expected collection to be %q, got %q", expect.Collection, out.Collection) } + + svalue := out.String() + if svalue != in { + t.Errorf("Expected string value to be %q, got %q", in, svalue) + } } } diff --git a/registry/registryprovider.go b/registry/registryprovider.go new file mode 100644 index 000000000..3675a9401 --- /dev/null +++ b/registry/registryprovider.go @@ -0,0 +1,237 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/google/go-github/github" + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" + + "fmt" + "net/url" + "regexp" + "strings" + "sync" +) + +// RegistryProvider returns factories for creating registry clients. +type RegistryProvider interface { + GetRegistryByShortURL(URL string) (Registry, error) + GetRegistryByName(registryName string) (Registry, error) + GetGithubRegistry(cr common.Registry) (GithubRegistry, error) +} + +func NewDefaultRegistryProvider() RegistryProvider { + registries := make(map[string]Registry) + rs := NewInmemRegistryService() + return &DefaultRegistryProvider{registries: registries, rs: rs} +} + +type DefaultRegistryProvider struct { + sync.RWMutex + registries map[string]Registry + rs common.RegistryService +} + +// Deprecated: Use GetRegistryByShortURL instead. +func (drp DefaultRegistryProvider) GetRegistryByURL(URL string) (Registry, error) { + return drp.GetRegistryByShortURL(URL) +} + +func (drp DefaultRegistryProvider) GetRegistryByShortURL(URL string) (Registry, error) { + drp.RLock() + defer drp.RUnlock() + + r := drp.findRegistryByShortURL(URL) + if r == nil { + cr, err := drp.rs.GetByURL(URL) + if err != nil { + return nil, err + } + + r, err := drp.GetGithubRegistry(*cr) + if err != nil { + return nil, err + } + + drp.registries[r.GetRegistryName()] = r + } + + return r, nil +} + +func (drp DefaultRegistryProvider) findRegistryByShortURL(URL string) Registry { + for _, r := range drp.registries { + if strings.HasPrefix(URL, r.GetRegistryShortURL()) { + return r + } + } + + return nil +} + +func (drp DefaultRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { + drp.RLock() + defer drp.RUnlock() + + r, ok := drp.registries[registryName] + if !ok { + cr, err := drp.rs.Get(registryName) + if err != nil { + return nil, err + } + + r, err := drp.GetGithubRegistry(*cr) + if err != nil { + return nil, err + } + + drp.registries[r.GetRegistryName()] = r + } + + return r, nil +} + +func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]bool { + split := strings.Split(string(rf), ";") + var result map[common.RegistryFormat]bool + for _, format := range split { + result[common.RegistryFormat(format)] = true + } + + return result +} + +func (drp DefaultRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { + if cr.Type == common.GithubRegistryType { + fMap := ParseRegistryFormat(cr.Format) + if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] { + return NewGithubPackageRegistry(cr.Name, cr.URL, github.NewClient(nil)) + } + + if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] { + return NewGithubTemplateRegistry(cr.Name, cr.URL, github.NewClient(nil)) + } + + return nil, fmt.Errorf("unknown registry format: %s", cr.Format) + } + + return nil, fmt.Errorf("unknown registry type: %s", cr.Type) +} + +// RE for a registry type that does support versions and has collections. +var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") + +// RE for a registry type that does not support versions and does not have collections. +var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)") + +// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type. +// For now, this means using github types: +// github.com/owner/repo/qualifier/type:version +// for example: +// github.com/kubernetes/application-dm-templates/storage/redis:v1 +func IsGithubShortType(t string) bool { + return TemplateRegistryMatcher.MatchString(t) +} + +// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github +// package repository type. +// For now, this means using github types: +// github.com/owner/repo/type +// for example: +// github.com/helm/charts/cassandra +func IsGithubShortPackageType(t string) bool { + return PackageRegistryMatcher.MatchString(t) +} + +// GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL +// and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type +// for example), it returns an empty slice. +func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) { + if IsGithubShortType(t) { + return ShortTypeToDownloadURLs(rp, t) + } else if IsGithubShortPackageType(t) { + return ShortTypeToPackageDownloadURLs(rp, t) + } else if util.IsHttpUrl(t) { + result, err := url.Parse(t) + if err != nil { + return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err) + } + + return []string{result.String()}, nil + } + + return []string{}, nil +} + +// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github. +// Input must be of the type and is assumed to have been validated before this call: +// github.com/owner/repo/qualifier/type:version +// for example: +// github.com/kubernetes/application-dm-templates/storage/redis:v1 +func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) { + m := TemplateRegistryMatcher.FindStringSubmatch(t) + if len(m) != 6 { + return nil, fmt.Errorf("cannot parse short github url: %s", t) + } + + r, err := rp.GetRegistryByShortURL(t) + if err != nil { + return nil, err + } + + tt, err := NewType(m[3], m[4], m[5]) + if err != nil { + return nil, err + } + + urls, err := r.GetDownloadURLs(tt) + if err != nil { + return nil, err + } + + return util.ConvertURLsToStrings(urls), err +} + +// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github. +// Input must be of the type and is assumed to have been validated before this call: +// github.com/owner/repo/type +// for example: +// github.com/helm/charts/cassandra +func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) { + m := PackageRegistryMatcher.FindStringSubmatch(t) + if len(m) != 4 { + return nil, fmt.Errorf("Failed to parse short github url: %s", t) + } + + r, err := rp.GetRegistryByShortURL(t) + if err != nil { + return nil, err + } + + tt, err := NewType("", m[3], "") + if err != nil { + return nil, err + } + + urls, err := r.GetDownloadURLs(tt) + if err != nil { + return nil, err + } + + return util.ConvertURLsToStrings(urls), err +} diff --git a/registry/registryprovider_test.go b/registry/registryprovider_test.go new file mode 100644 index 000000000..708975089 --- /dev/null +++ b/registry/registryprovider_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +// TODO(jackgr): Finish implementing registry provider tests. + +import ( + "github.com/kubernetes/deployment-manager/common" + + "fmt" + "net/url" + "regexp" + "strings" + "testing" +) + +type urlAndError struct { + u string + e error +} + +type testRegistryProvider struct { + URLPrefix string + r map[string]Registry +} + +func newTestRegistryProvider(URLPrefix string, tests map[Type]urlAndError) RegistryProvider { + r := make(map[string]Registry) + r[URLPrefix] = testGithubRegistry{tests} + return testRegistryProvider{URLPrefix, r} +} + +func (trp testRegistryProvider) GetRegistryByShortURL(URL string) (Registry, error) { + for key, r := range trp.r { + if strings.HasPrefix(URL, key) { + return r, nil + } + } + + return nil, fmt.Errorf("No registry found for %s", URL) +} + +func (trp testRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { + panic(fmt.Errorf("GetRegistryByName should not be called in the test")) +} + +func (trp testRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { + panic(fmt.Errorf("GetGithubRegistry should not be called in the test")) +} + +type testGithubRegistry struct { + responses map[Type]urlAndError +} + +func (tgr testGithubRegistry) GetRegistryName() string { + panic(fmt.Errorf("GetRegistryName should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryType() common.RegistryType { + return common.GithubRegistryType +} + +func (tgr testGithubRegistry) GetRegistryShortURL() string { + panic(fmt.Errorf("GetRegistryShortURL should not be called in the test")) +} + +func (tgr testGithubRegistry) GetRegistryFormat() common.RegistryFormat { + panic(fmt.Errorf("GetRegistryFormat should not be called in the test")) +} + +func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { + ret := tgr.responses[t] + URL, err := url.Parse(ret.u) + if err != nil { + panic(err) + } + + return []*url.URL{URL}, ret.e +} + +func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { + panic(fmt.Errorf("ListTypes should not be called in the test")) +} + +func testUrlConversionDriver(rp RegistryProvider, tests map[string]urlAndError, t *testing.T) { + for in, expected := range tests { + actual, err := GetDownloadURLs(rp, in) + if err != expected.e { + t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) + } + + if actual[0] != expected.u { + t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual) + } + } +} + +func TestShortGithubUrlMapping(t *testing.T) { + githubUrlMaps := map[Type]urlAndError{ + NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + NewTypeOrDie("storage", "redis", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + } + + tests := map[string]urlAndError{ + "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + } + + test := newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) + testUrlConversionDriver(test, tests, t) +} + +func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { + githubUrlMaps := map[Type]urlAndError{ + NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + NewTypeOrDie("storage", "redis", "v1"): urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + } + + tests := map[string]urlAndError{ + "github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + } + + test := newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps) + testUrlConversionDriver(test, tests, t) +} diff --git a/registry/semver.go b/registry/semver.go index f5c1ff596..f49fd833b 100644 --- a/registry/semver.go +++ b/registry/semver.go @@ -28,45 +28,54 @@ type SemVer struct { Patch uint } -func NewSemVer(version string) (*SemVer, error) { - result := &SemVer{} - parts := strings.SplitN(version, ".", 3) - if len(parts) > 3 { - return nil, fmt.Errorf("invalid semantic version: %s", version) - } +func ParseSemVer(version string) (SemVer, error) { + var err error + major, minor, patch := uint64(0), uint64(0), uint64(0) + if len(version) > 0 { + parts := strings.SplitN(version, ".", 3) + if len(parts) > 3 { + return SemVer{}, fmt.Errorf("invalid semantic version: %s", version) + } - major, err := strconv.ParseUint(parts[0], 10, 0) - if err != nil { - return nil, fmt.Errorf("invalid semantic version: %s", version) - } + if len(parts) < 1 { + return SemVer{}, fmt.Errorf("invalid semantic version: %s", version) + } - result.Major = uint(major) - if len(parts) < 3 { - if len(parts) < 2 { - if len(parts) < 1 { - return nil, fmt.Errorf("invalid semantic version: %s", version) - } - } else { - minor, err := strconv.ParseUint(parts[1], 10, 0) + if parts[0] != "0" { + major, err = strconv.ParseUint(parts[0], 10, 0) if err != nil { - return nil, fmt.Errorf("invalid semantic version: %s", version) + return SemVer{}, fmt.Errorf("invalid semantic version: %s", version) } - - result.Minor = uint(minor) - } - } else { - patch, err := strconv.ParseUint(parts[2], 10, 0) - if err != nil { - return nil, fmt.Errorf("invalid semantic version: %s", version) } - result.Patch = uint(patch) + if len(parts) > 1 { + if parts[1] != "0" { + minor, err = strconv.ParseUint(parts[1], 10, 0) + if err != nil { + return SemVer{}, fmt.Errorf("invalid semantic version: %s", version) + } + } + + if len(parts) > 2 { + if parts[2] != "0" { + patch, err = strconv.ParseUint(parts[2], 10, 0) + if err != nil { + return SemVer{}, fmt.Errorf("invalid semantic version: %s", version) + } + } + } + } } - return result, nil + return SemVer{Major: uint(major), Minor: uint(minor), Patch: uint(patch)}, nil +} + +func (s SemVer) IsZero() bool { + return s.Major == 0 && s.Minor == 0 && s.Patch == 0 } -func (s *SemVer) String() string { +// SemVer conforms to the Stringer interface. +func (s SemVer) String() string { result := strconv.Itoa(int(s.Major)) if s.Minor != 0 || s.Patch != 0 { result = result + "." + strconv.Itoa(int(s.Minor)) diff --git a/registry/semver_test.go b/registry/semver_test.go new file mode 100644 index 000000000..b88c7455c --- /dev/null +++ b/registry/semver_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "testing" +) + +func TestParseInvalidVersionFails(t *testing.T) { + for _, test := range []string{ + ".", + "..", + "...", + "1.2.3.4", + "notAUnit", + "1.notAUint", + "1.1.notAUint", + "-1", + "1.-1", + "1.1.-1", + "1,1", + "1.1,1", + } { + _, err := ParseSemVer(test) + if err == nil { + t.Errorf("Invalid version parsed successfully: %s\n", test) + } + } +} + +func TestParseValidVersionSucceeds(t *testing.T) { + for _, test := range []struct { + String string + Version SemVer + }{ + {"", SemVer{0, 0, 0}}, + {"0", SemVer{0, 0, 0}}, + {"0.0", SemVer{0, 0, 0}}, + {"0.0.0", SemVer{0, 0, 0}}, + {"1", SemVer{1, 0, 0}}, + {"1.0", SemVer{1, 0, 0}}, + {"1.0.0", SemVer{1, 0, 0}}, + {"1.1", SemVer{1, 1, 0}}, + {"1.1.0", SemVer{1, 1, 0}}, + {"1.1.1", SemVer{1, 1, 1}}, + } { + result, err := ParseSemVer(test.String) + if err != nil { + t.Errorf("Valid version %s did not parse successfully\n", test.String) + } + + if result.Major != test.Version.Major || + result.Minor != test.Version.Minor || + result.Patch != test.Version.Patch { + t.Errorf("Valid version %s did not parse correctly: %s\n", test.String, test.Version) + } + } +} + +func TestConvertSemVerToStringSucceeds(t *testing.T) { + for _, test := range []struct { + String string + Version SemVer + }{ + {"0", SemVer{0, 0, 0}}, + {"0.1", SemVer{0, 1, 0}}, + {"0.0.1", SemVer{0, 0, 1}}, + {"1", SemVer{1, 0, 0}}, + {"1.1", SemVer{1, 1, 0}}, + {"1.1.1", SemVer{1, 1, 1}}, + } { + result := test.Version.String() + if result != test.String { + t.Errorf("Valid version %s did not format correctly: %s\n", test.Version, test.String) + } + } +} diff --git a/resourcifier/configurations.go b/resourcifier/configurations.go index 09842eb08..783f95f93 100644 --- a/resourcifier/configurations.go +++ b/resourcifier/configurations.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -238,14 +238,14 @@ func getConfigurator() *configurator.Configurator { func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) { vars := mux.Vars(r) - variable, ok := vars[variable] + escaped, ok := vars[variable] if !ok { e := errors.New(fmt.Sprintf("%s name not found in URL", variable)) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) return "", e } - unescaped, err := url.QueryUnescape(variable) + unescaped, err := url.QueryUnescape(escaped) if err != nil { e := fmt.Errorf("cannot decode name (%v)", variable) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) diff --git a/util/httputil.go b/util/httputil.go index 572f940b9..a7a21484e 100644 --- a/util/httputil.go +++ b/util/httputil.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,6 +29,26 @@ import ( "github.com/ghodss/yaml" ) +// ConvertURLsToStrings converts a slice of *url.URL to a slice of string. +func ConvertURLsToStrings(urls []*url.URL) []string { + var result []string + for _, u := range urls { + result = append(result, u.String()) + } + + return result +} + +// TrimURLScheme removes the scheme, if any, from an URL. +func TrimURLScheme(URL string) string { + parts := strings.SplitAfter(URL, "://") + if len(parts) > 1 { + return parts[1] + } + + return URL +} + // A HandlerTester is a function that takes an HTTP method, an URL path, and a // reader for a request body, creates a request from them, and serves it to the // handler to which it was bound and returns a response recorder describing the diff --git a/util/templateutil.go b/util/templateutil.go deleted file mode 100644 index 3ce844c5d..000000000 --- a/util/templateutil.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "regexp" - - "github.com/kubernetes/deployment-manager/common" -) - -var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") - -// RE for Registry that does not support versions and can have multiple files without imports. -var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)") - -// IsTemplate returns whether a given type is a template. -func IsTemplate(t string, imports []*common.ImportFile) bool { - for _, imp := range imports { - if imp.Name == t { - return true - } - } - return false -} - -// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type. -// For now, this means using github types: -// github.com/owner/repo/qualifier/type:version -// for example: -// github.com/kubernetes/application-dm-templates/storage/redis:v1 -func IsGithubShortType(t string) bool { - return TemplateRegistryMatcher.MatchString(t) -} - -// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github -// package repository type. -// For now, this means using github types: -// github.com/owner/repo/type -// for example: -// github.com/helm/charts/cassandra -func IsGithubShortPackageType(t string) bool { - return PackageRegistryMatcher.MatchString(t) -} From a6e1067a50a3245730707f7153ab3899e23448b8 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 11 Jan 2016 20:01:17 -0800 Subject: [PATCH 3/6] Make github registry provider and repository service mockable. --- registry/github_package_registry.go | 11 ++-- registry/github_registry.go | 27 ++++++++-- registry/github_template_registry.go | 9 ++-- registry/registryprovider.go | 77 +++++++++++++++++----------- 4 files changed, 77 insertions(+), 47 deletions(-) diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 681eeef18..5bea6613a 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -17,7 +17,6 @@ limitations under the License. package registry import ( - "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" "fmt" @@ -41,9 +40,9 @@ type GithubPackageRegistry struct { } // NewGithubPackageRegistry creates a GithubPackageRegistry. -func NewGithubPackageRegistry(name, shortURL string, client *github.Client) (GithubPackageRegistry, error) { +func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (GithubPackageRegistry, error) { format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) - gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), client) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) if err != nil { return GithubPackageRegistry{}, err } @@ -64,7 +63,7 @@ func (g GithubPackageRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { var retTypes []Type for _, t := range types { // Check to see if there's a Chart.yaml file in the directory - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, t, nil) if err != nil { log.Printf("Failed to list package files at path: %s: %v", t, err) return nil, err @@ -87,7 +86,7 @@ func (g GithubPackageRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { return nil, err } - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to list package files at path: %s: %v", path, err) return nil, err @@ -111,7 +110,7 @@ func (g GithubPackageRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { } func (g GithubPackageRegistry) getDirs(dir string) ([]string, error) { - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, dir, nil) if err != nil { log.Printf("Failed to get contents at path: %s: %v", dir, err) return nil, err diff --git a/registry/github_registry.go b/registry/github_registry.go index 5f25aa1ae..fed57322c 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -38,17 +38,34 @@ type githubRegistry struct { repository string path string format common.RegistryFormat - client *github.Client + service RepositoryService +} + +type RepositoryService interface { + GetContents( + owner, repo, path string, + opt *github.RepositoryContentGetOptions, + ) ( + fileContent *github.RepositoryContent, + directoryContent []*github.RepositoryContent, + resp *github.Response, + err error, + ) } // newGithubRegistry creates a githubRegistry. -func newGithubRegistry(name, shortURL string, format common.RegistryFormat, client *github.Client) (githubRegistry, error) { +func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (githubRegistry, error) { trimmed := util.TrimURLScheme(shortURL) owner, repository, path, err := parseGithubShortURL(trimmed) if err != nil { return githubRegistry{}, fmt.Errorf("cannot create Github template registry %s: %s", name, err) } + if service == nil { + client := github.NewClient(nil) + service = client.Repositories + } + return githubRegistry{ name: name, shortURL: trimmed, @@ -56,7 +73,7 @@ func newGithubRegistry(name, shortURL string, format common.RegistryFormat, clie repository: repository, path: path, format: format, - client: client, + service: service, }, nil } @@ -165,7 +182,7 @@ func (g githubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { if err != nil { return nil, err } - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) if err != nil { return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err) } @@ -205,7 +222,7 @@ func (g githubRegistry) getDirs(dir string) ([]string, error) { path = g.path + "/" + dir } - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to get contents at path: %s: %v", path, err) return nil, err diff --git a/registry/github_template_registry.go b/registry/github_template_registry.go index 586e22118..be704eae2 100644 --- a/registry/github_template_registry.go +++ b/registry/github_template_registry.go @@ -17,7 +17,6 @@ limitations under the License. package registry import ( - "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" "fmt" @@ -54,9 +53,9 @@ type GithubTemplateRegistry struct { } // NewGithubTemplateRegistry creates a GithubTemplateRegistry. -func NewGithubTemplateRegistry(name, shortURL string, client *github.Client) (GithubTemplateRegistry, error) { +func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (GithubTemplateRegistry, error) { format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) - gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), client) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) if err != nil { return GithubTemplateRegistry{}, err } @@ -113,7 +112,7 @@ func (g GithubTemplateRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { if err != nil { return nil, err } - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) if err != nil { return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err) } @@ -153,7 +152,7 @@ func (g GithubTemplateRegistry) getDirs(dir string) ([]string, error) { path = g.path + "/" + dir } - _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to get contents at path: %s: %v", path, err) return nil, err diff --git a/registry/registryprovider.go b/registry/registryprovider.go index 3675a9401..2d499fa0e 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -17,7 +17,6 @@ limitations under the License. package registry import ( - "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/util" @@ -28,55 +27,71 @@ import ( "sync" ) -// RegistryProvider returns factories for creating registry clients. +// RegistryProvider is a factory for Registry instances. type RegistryProvider interface { GetRegistryByShortURL(URL string) (Registry, error) GetRegistryByName(registryName string) (Registry, error) +} + +// GithubRegistryProvider is a factory for GithubRegistry instances. +type GithubRegistryProvider interface { GetGithubRegistry(cr common.Registry) (GithubRegistry, error) } func NewDefaultRegistryProvider() RegistryProvider { + return NewRegistryProvider(nil, nil) +} + +func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider { + if rs == nil { + rs = NewInmemRegistryService() + } + registries := make(map[string]Registry) - rs := NewInmemRegistryService() - return &DefaultRegistryProvider{registries: registries, rs: rs} + rp := registryProvider{rs: rs, registries: registries} + if grp == nil { + grp = rp + } + + rp.grp = grp + return rp } -type DefaultRegistryProvider struct { +type registryProvider struct { sync.RWMutex - registries map[string]Registry rs common.RegistryService + grp GithubRegistryProvider + registries map[string]Registry } -// Deprecated: Use GetRegistryByShortURL instead. -func (drp DefaultRegistryProvider) GetRegistryByURL(URL string) (Registry, error) { - return drp.GetRegistryByShortURL(URL) -} - -func (drp DefaultRegistryProvider) GetRegistryByShortURL(URL string) (Registry, error) { - drp.RLock() - defer drp.RUnlock() +func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { + rp.RLock() + defer rp.RUnlock() - r := drp.findRegistryByShortURL(URL) + r := rp.findRegistryByShortURL(URL) if r == nil { - cr, err := drp.rs.GetByURL(URL) + cr, err := rp.rs.GetByURL(URL) if err != nil { return nil, err } - r, err := drp.GetGithubRegistry(*cr) + r, err := rp.GetGithubRegistry(*cr) if err != nil { return nil, err } - drp.registries[r.GetRegistryName()] = r + rp.registries[r.GetRegistryName()] = r } return r, nil } -func (drp DefaultRegistryProvider) findRegistryByShortURL(URL string) Registry { - for _, r := range drp.registries { - if strings.HasPrefix(URL, r.GetRegistryShortURL()) { +// findRegistryByShortURL trims the scheme from both the supplied URL +// and the short URL returned by GetRegistryShortURL. +func (rp registryProvider) findRegistryByShortURL(URL string) Registry { + trimmed := util.TrimURLScheme(URL) + for _, r := range rp.registries { + if strings.HasPrefix(trimmed, util.TrimURLScheme(r.GetRegistryShortURL())) { return r } } @@ -84,23 +99,23 @@ func (drp DefaultRegistryProvider) findRegistryByShortURL(URL string) Registry { return nil } -func (drp DefaultRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { - drp.RLock() - defer drp.RUnlock() +func (rp registryProvider) GetRegistryByName(registryName string) (Registry, error) { + rp.RLock() + defer rp.RUnlock() - r, ok := drp.registries[registryName] + r, ok := rp.registries[registryName] if !ok { - cr, err := drp.rs.Get(registryName) + cr, err := rp.rs.Get(registryName) if err != nil { return nil, err } - r, err := drp.GetGithubRegistry(*cr) + r, err := rp.GetGithubRegistry(*cr) if err != nil { return nil, err } - drp.registries[r.GetRegistryName()] = r + rp.registries[r.GetRegistryName()] = r } return r, nil @@ -116,15 +131,15 @@ func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]boo return result } -func (drp DefaultRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { +func (rp registryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { if cr.Type == common.GithubRegistryType { fMap := ParseRegistryFormat(cr.Format) if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] { - return NewGithubPackageRegistry(cr.Name, cr.URL, github.NewClient(nil)) + return NewGithubPackageRegistry(cr.Name, cr.URL, nil) } if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] { - return NewGithubTemplateRegistry(cr.Name, cr.URL, github.NewClient(nil)) + return NewGithubTemplateRegistry(cr.Name, cr.URL, nil) } return nil, fmt.Errorf("unknown registry format: %s", cr.Format) From 8b67285c529403a470f650fb576a4cefcbcd999c Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 11 Jan 2016 22:16:46 -0800 Subject: [PATCH 4/6] Additional test clean ups. --- manager/manager/manager_test.go | 2 +- manager/manager/registryprovider_test.go | 106 ---------------- manager/manager/typeresolver_test.go | 12 +- registry/github_package_registry.go | 12 +- registry/github_registry.go | 149 +---------------------- registry/github_template_registry.go | 12 +- registry/registryprovider.go | 24 ++-- registry/registryprovider_test.go | 122 ++++--------------- registry/testhelper.go | 82 +++++++++++++ 9 files changed, 150 insertions(+), 371 deletions(-) delete mode 100644 manager/manager/registryprovider_test.go create mode 100644 registry/testhelper.go diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index 9d80a438c..810c0cb14 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -254,7 +254,7 @@ var testExpander = &expanderStub{} var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() var testRegistryService = registry.NewInmemRegistryService() -var testProvider = newTestRegistryProvider("", nil) +var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil)) var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService) func TestListDeployments(t *testing.T) { diff --git a/manager/manager/registryprovider_test.go b/manager/manager/registryprovider_test.go deleted file mode 100644 index 4a4cab423..000000000 --- a/manager/manager/registryprovider_test.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manager - -import ( - "fmt" - "net/url" - "regexp" - "strings" - - "github.com/kubernetes/deployment-manager/common" - "github.com/kubernetes/deployment-manager/registry" -) - -type urlAndError struct { - u string - e error -} - -type testRegistryProvider struct { - r map[string]registry.Registry -} - -func newTestRegistryProvider(shortURL string, tests map[registry.Type]urlAndError) registry.RegistryProvider { - r := make(map[string]registry.Registry) - r[shortURL] = &testGithubRegistry{tests} - return testRegistryProvider{r} -} - -func (trp testRegistryProvider) GetRegistryByShortURL(URL string) (registry.Registry, error) { - for key, r := range trp.r { - if strings.HasPrefix(URL, key) { - return r, nil - } - } - - return nil, fmt.Errorf("No registry found for %s", URL) -} - -func (trp testRegistryProvider) GetRegistryByName(registryName string) (registry.Registry, error) { - panic(fmt.Errorf("GetRegistryByName should not be called in the test")) -} - -func (trp testRegistryProvider) GetGithubRegistry(cr common.Registry) (registry.GithubRegistry, error) { - panic(fmt.Errorf("GetGithubRegistry should not be called in the test")) -} - -type testGithubRegistry struct { - responses map[registry.Type]urlAndError -} - -func (tgr testGithubRegistry) GetRegistryName() string { - panic(fmt.Errorf("GetRegistryName should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryType() common.RegistryType { - return common.GithubRegistryType -} - -func (tgr testGithubRegistry) GetRegistryShortURL() string { - panic(fmt.Errorf("GetRegistryShortURL should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryFormat() common.RegistryFormat { - panic(fmt.Errorf("GetRegistryFormat should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryOwner() string { - panic(fmt.Errorf("GetRegistryOwner should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryRepository() string { - panic(fmt.Errorf("GetRegistryRepository should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryPath() string { - panic(fmt.Errorf("GetRegistryPath should not be called in the test")) -} - -func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]registry.Type, error) { - panic(fmt.Errorf("ListTypes should not be called in the test")) -} - -func (tgr testGithubRegistry) GetDownloadURLs(t registry.Type) ([]*url.URL, error) { - ret := tgr.responses[t] - URL, err := url.Parse(ret.u) - if err != nil { - panic(err) - } - - return []*url.URL{URL}, ret.e -} diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index 321564c23..37b58396e 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -228,6 +228,7 @@ func TestTooManyImports(t *testing.T) { responses: responses, expectedErr: errors.New("Number of imports exceeds maximum of 5"), } + testDriver(test, t) } @@ -270,6 +271,7 @@ func TestSharedImport(t *testing.T) { responses: responses, importOut: finalImports, } + testDriver(test, t) } @@ -300,17 +302,19 @@ func TestShortGithubUrl(t *testing.T) { "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""}, } - githubUrlMaps := map[registry.Type]urlAndError{ - registry.NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.NewTypeOrDie("common", "replicatedservice", "v2"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, + githubUrlMaps := map[registry.Type]registry.TestURLAndError{ + registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, } + grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) test := resolverTestCase{ config: templateShortGithubTemplate, importOut: finalImports, urlcount: 4, responses: responses, - registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps), + registryProvider: registry.NewRegistryProvider(nil, grp), } + testDriver(test, t) } diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 5bea6613a..a7de76e16 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -17,6 +17,7 @@ limitations under the License. package registry import ( + "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" "fmt" @@ -40,14 +41,19 @@ type GithubPackageRegistry struct { } // NewGithubPackageRegistry creates a GithubPackageRegistry. -func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (GithubPackageRegistry, error) { +func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (*GithubPackageRegistry, error) { format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) + if service == nil { + client := github.NewClient(nil) + service = client.Repositories + } + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) if err != nil { - return GithubPackageRegistry{}, err + return nil, err } - return GithubPackageRegistry{githubRegistry: gr}, nil + return &GithubPackageRegistry{githubRegistry: *gr}, nil } // ListTypes lists types in this registry whose string values conform to the diff --git a/registry/github_registry.go b/registry/github_registry.go index fed57322c..e40f20d3d 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -22,9 +22,6 @@ import ( "github.com/kubernetes/deployment-manager/util" "fmt" - "log" - "net/url" - "regexp" "strings" ) @@ -54,19 +51,14 @@ type RepositoryService interface { } // newGithubRegistry creates a githubRegistry. -func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (githubRegistry, error) { +func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (*githubRegistry, error) { trimmed := util.TrimURLScheme(shortURL) owner, repository, path, err := parseGithubShortURL(trimmed) if err != nil { - return githubRegistry{}, fmt.Errorf("cannot create Github template registry %s: %s", name, err) + return nil, fmt.Errorf("cannot create Github template registry %s: %s", name, err) } - if service == nil { - client := github.NewClient(nil) - service = client.Repositories - } - - return githubRegistry{ + return &githubRegistry{ name: name, shortURL: trimmed, owner: owner, @@ -132,138 +124,3 @@ func (g githubRegistry) GetRegistryRepository() string { func (g githubRegistry) GetRegistryPath() string { return g.path } - -// ListTypes lists types in this registry whose string values conform to the -// supplied regular expression, or all types, if the regular expression is nil. -func (g githubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { - // First list all the collections at the top level. - collections, err := g.getDirs("") - if err != nil { - log.Printf("cannot list qualifiers: %v", err) - return nil, err - } - - var retTypes []Type - for _, c := range collections { - // Then we need to fetch the versions (directories for this type) - types, err := g.getDirs(c) - if err != nil { - log.Printf("cannot fetch types for collection: %s", c) - return nil, err - } - - for _, t := range types { - path := c + "/" + t - // Then we need to fetch the versions (directories for this type) - versions, err := g.getDirs(path) - if err != nil { - log.Printf("cannot fetch versions at path %s", path) - return nil, err - } - - for _, v := range versions { - tt, err := NewType(c, t, v) - if err != nil { - return nil, fmt.Errorf("malformed type at path %s", path) - } - - retTypes = append(retTypes, tt) - } - } - } - - // TODO(jackgr): Use the supplied regex to filter the results. - return retTypes, nil -} - -// GetDownloadURLs fetches the download URLs for a given Type and checks for existence of a schema file. -func (g githubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { - path, err := g.MakeRepositoryPath(t) - if err != nil { - return nil, err - } - _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) - if err != nil { - return nil, fmt.Errorf("cannot list versions at path %s: %v", path, err) - } - - var downloadURL, typeName, schemaName string - for _, f := range dc { - if *f.Type == "file" { - if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" { - typeName = *f.Name - downloadURL = *f.DownloadURL - } - if *f.Name == t.Name+".jinja.schema" || *f.Name == t.Name+".py.schema" { - schemaName = *f.Name - } - } - } - - if downloadURL == "" { - return nil, fmt.Errorf("cannot find type %s", t.String()) - } - - if schemaName != typeName+".schema" { - return nil, fmt.Errorf("cannot find schema for %s, expected %s", t.String(), typeName+".schema") - } - - result, err := url.Parse(downloadURL) - if err != nil { - return nil, fmt.Errorf("cannot parse URL from %s: %s", downloadURL, err) - } - - return []*url.URL{result}, nil -} - -func (g githubRegistry) getDirs(dir string) ([]string, error) { - var path = g.path - if dir != "" { - path = g.path + "/" + dir - } - - _, dc, _, err := g.service.GetContents(g.owner, g.repository, path, nil) - if err != nil { - log.Printf("Failed to get contents at path: %s: %v", path, err) - return nil, err - } - - var dirs []string - for _, entry := range dc { - if *entry.Type == "dir" { - dirs = append(dirs, *entry.Name) - } - } - - return dirs, nil -} - -func (g githubRegistry) mapCollection(collection string) (string, error) { - if strings.ContainsAny(collection, "/") { - return "", fmt.Errorf("collection must not contain slashes, got %s", collection) - } - // TODO(vaikas): Implement lookup from the root metadata file to map collection to a path - return collection, nil -} - -// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version. -// The returned repository path will be of the form: -// [githubRegistry.path/][Type.Collection]/Type.Name/Type.Version -// Type.Collection will be mapped using mapCollection in the future, for now it's a straight -// 1:1 mapping (if given) -func (g githubRegistry) MakeRepositoryPath(t Type) (string, error) { - // First map the collection - collection, err := g.mapCollection(t.Collection) - if err != nil { - return "", err - } - // Construct the return path - p := "" - if len(g.path) > 0 { - p += g.path + "/" - } - if len(collection) > 0 { - p += collection + "/" - } - return p + t.Name + "/" + t.GetVersion(), nil -} diff --git a/registry/github_template_registry.go b/registry/github_template_registry.go index be704eae2..40f34968b 100644 --- a/registry/github_template_registry.go +++ b/registry/github_template_registry.go @@ -17,6 +17,7 @@ limitations under the License. package registry import ( + "github.com/google/go-github/github" "github.com/kubernetes/deployment-manager/common" "fmt" @@ -53,14 +54,19 @@ type GithubTemplateRegistry struct { } // NewGithubTemplateRegistry creates a GithubTemplateRegistry. -func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (GithubTemplateRegistry, error) { +func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (*GithubTemplateRegistry, error) { format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) + if service == nil { + client := github.NewClient(nil) + service = client.Repositories + } + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) if err != nil { - return GithubTemplateRegistry{}, err + return nil, err } - return GithubTemplateRegistry{githubRegistry: gr}, nil + return &GithubTemplateRegistry{githubRegistry: *gr}, nil } // ListTypes lists types in this registry whose string values conform to the diff --git a/registry/registryprovider.go b/registry/registryprovider.go index 2d499fa0e..b25102e84 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -48,7 +48,7 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) } registries := make(map[string]Registry) - rp := registryProvider{rs: rs, registries: registries} + rp := ®istryProvider{rs: rs, registries: registries} if grp == nil { grp = rp } @@ -68,22 +68,23 @@ func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { rp.RLock() defer rp.RUnlock() - r := rp.findRegistryByShortURL(URL) - if r == nil { + result := rp.findRegistryByShortURL(URL) + if result == nil { cr, err := rp.rs.GetByURL(URL) if err != nil { return nil, err } - r, err := rp.GetGithubRegistry(*cr) + r, err := rp.grp.GetGithubRegistry(*cr) if err != nil { return nil, err } rp.registries[r.GetRegistryName()] = r + result = r } - return r, nil + return result, nil } // findRegistryByShortURL trims the scheme from both the supplied URL @@ -103,27 +104,28 @@ func (rp registryProvider) GetRegistryByName(registryName string) (Registry, err rp.RLock() defer rp.RUnlock() - r, ok := rp.registries[registryName] + result, ok := rp.registries[registryName] if !ok { cr, err := rp.rs.Get(registryName) if err != nil { return nil, err } - r, err := rp.GetGithubRegistry(*cr) + r, err := rp.grp.GetGithubRegistry(*cr) if err != nil { return nil, err } rp.registries[r.GetRegistryName()] = r + result = r } - return r, nil + return result, nil } func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]bool { split := strings.Split(string(rf), ";") - var result map[common.RegistryFormat]bool + var result = map[common.RegistryFormat]bool{} for _, format := range split { result[common.RegistryFormat(format)] = true } @@ -209,6 +211,10 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) { return nil, err } + if r == nil { + panic(fmt.Errorf("cannot get github registry for %s", t)) + } + tt, err := NewType(m[3], m[4], m[5]) if err != nil { return nil, err diff --git a/registry/registryprovider_test.go b/registry/registryprovider_test.go index 708975089..28e24c45e 100644 --- a/registry/registryprovider_test.go +++ b/registry/registryprovider_test.go @@ -16,125 +16,49 @@ limitations under the License. package registry -// TODO(jackgr): Finish implementing registry provider tests. - import ( - "github.com/kubernetes/deployment-manager/common" - - "fmt" - "net/url" - "regexp" - "strings" "testing" ) -type urlAndError struct { - u string - e error -} - -type testRegistryProvider struct { - URLPrefix string - r map[string]Registry -} - -func newTestRegistryProvider(URLPrefix string, tests map[Type]urlAndError) RegistryProvider { - r := make(map[string]Registry) - r[URLPrefix] = testGithubRegistry{tests} - return testRegistryProvider{URLPrefix, r} -} - -func (trp testRegistryProvider) GetRegistryByShortURL(URL string) (Registry, error) { - for key, r := range trp.r { - if strings.HasPrefix(URL, key) { - return r, nil - } - } - - return nil, fmt.Errorf("No registry found for %s", URL) -} - -func (trp testRegistryProvider) GetRegistryByName(registryName string) (Registry, error) { - panic(fmt.Errorf("GetRegistryByName should not be called in the test")) -} - -func (trp testRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { - panic(fmt.Errorf("GetGithubRegistry should not be called in the test")) -} - -type testGithubRegistry struct { - responses map[Type]urlAndError -} - -func (tgr testGithubRegistry) GetRegistryName() string { - panic(fmt.Errorf("GetRegistryName should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryType() common.RegistryType { - return common.GithubRegistryType -} - -func (tgr testGithubRegistry) GetRegistryShortURL() string { - panic(fmt.Errorf("GetRegistryShortURL should not be called in the test")) -} - -func (tgr testGithubRegistry) GetRegistryFormat() common.RegistryFormat { - panic(fmt.Errorf("GetRegistryFormat should not be called in the test")) -} - -func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { - ret := tgr.responses[t] - URL, err := url.Parse(ret.u) - if err != nil { - panic(err) - } - - return []*url.URL{URL}, ret.e -} - -func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { - panic(fmt.Errorf("ListTypes should not be called in the test")) -} - -func testUrlConversionDriver(rp RegistryProvider, tests map[string]urlAndError, t *testing.T) { +func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) { for in, expected := range tests { actual, err := GetDownloadURLs(rp, in) - if err != expected.e { - t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) + if err != expected.Err { + t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err) } - if actual[0] != expected.u { - t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual) + if actual[0] != expected.URL { + t.Fatalf("failed on: %s : expected %s but got %v", in, expected.URL, actual) } } } -func TestShortGithubUrlMapping(t *testing.T) { - githubUrlMaps := map[Type]urlAndError{ - NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - NewTypeOrDie("storage", "redis", "v1"): urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, +func TestShortGithubUrlTemplateMapping(t *testing.T) { + githubUrlMaps := map[Type]TestURLAndError{ + NewTypeOrDie("common", "replicatedservice", "v1"): TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + NewTypeOrDie("storage", "redis", "v1"): TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, } - tests := map[string]urlAndError{ - "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + tests := map[string]TestURLAndError{ + "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + "github.com/kubernetes/application-dm-templates/storage/redis:v1": TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, } - test := newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) - testUrlConversionDriver(test, tests, t) + grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) + testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t) } -func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { - githubUrlMaps := map[Type]urlAndError{ - NewTypeOrDie("common", "replicatedservice", "v1"): urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - NewTypeOrDie("storage", "redis", "v1"): urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, +func TestShortGithubUrlPackageMapping(t *testing.T) { + githubUrlMaps := map[Type]TestURLAndError{ + NewTypeOrDie("", "mongodb", ""): TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil}, + NewTypeOrDie("", "redis", ""): TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil}, } - tests := map[string]urlAndError{ - "github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + tests := map[string]TestURLAndError{ + "github.com/helm/charts/mongodb": TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/mongodb/manifests/mongodb.yaml", nil}, + "github.com/helm/charts/redis": TestURLAndError{"https://raw.githubusercontent.com/helm/charts/master/redis/manifests/redis.yaml", nil}, } - test := newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps) - testUrlConversionDriver(test, tests, t) + grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps) + testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t) } diff --git a/registry/testhelper.go b/registry/testhelper.go new file mode 100644 index 000000000..7b7dfe937 --- /dev/null +++ b/registry/testhelper.go @@ -0,0 +1,82 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +// TODO(jackgr): Mock github repository service to test package and template registry implementations. + +import ( + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" + + "fmt" + "net/url" + "regexp" + "strings" +) + +type TestURLAndError struct { + URL string + Err error +} + +type testGithubRegistryProvider struct { + shortURL string + responses map[Type]TestURLAndError +} + +type testGithubRegistry struct { + githubRegistry + responses map[Type]TestURLAndError +} + +func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GithubRegistryProvider { + return testGithubRegistryProvider{ + shortURL: util.TrimURLScheme(shortURL), + responses: responses, + } +} + +func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { + trimmed := util.TrimURLScheme(cr.URL) + if strings.HasPrefix(trimmed, tgrp.shortURL) { + ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil) + if err != nil { + panic(fmt.Errorf("cannot create a github registry: %s", err)) + } + + return &testGithubRegistry{ + githubRegistry: *ghr, + responses: tgrp.responses, + }, nil + } + + panic(fmt.Errorf("unknown registry: %v", cr)) +} + +func (tgr testGithubRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { + panic(fmt.Errorf("ListTypes should not be called in the test")) +} + +func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { + result := tgr.responses[t] + URL, err := url.Parse(result.URL) + if err != nil { + panic(err) + } + + return []*url.URL{URL}, result.Err +} From 42f894d882c113f0bce18744fa0eb13857ef84d1 Mon Sep 17 00:00:00 2001 From: jackgr Date: Mon, 11 Jan 2016 22:24:55 -0800 Subject: [PATCH 5/6] Implemented regex matching for type names. --- registry/github_package_registry.go | 12 +++++++++++- registry/github_template_registry.go | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index a7de76e16..10873c140 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -81,7 +81,17 @@ func (g GithubPackageRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { } } - // TODO(jackgr): Use the supplied regex to filter the results. + if regex != nil { + var matchTypes []Type + for _, retType := range retTypes { + if regex.MatchString(retType.String()) { + matchTypes = append(matchTypes, retType) + } + } + + return matchTypes, nil + } + return retTypes, nil } diff --git a/registry/github_template_registry.go b/registry/github_template_registry.go index 40f34968b..42f69da77 100644 --- a/registry/github_template_registry.go +++ b/registry/github_template_registry.go @@ -108,7 +108,17 @@ func (g GithubTemplateRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) } } - // TODO(jackgr): Use the supplied regex to filter the results. + if regex != nil { + var matchTypes []Type + for _, retType := range retTypes { + if regex.MatchString(retType.String()) { + matchTypes = append(matchTypes, retType) + } + } + + return matchTypes, nil + } + return retTypes, nil } From 2c59a07bfdd52b4dbbbb3f5d95b3340d46ab9c23 Mon Sep 17 00:00:00 2001 From: jackgr Date: Tue, 12 Jan 2016 15:44:56 -0800 Subject: [PATCH 6/6] Regex search in dm and Github credential setting to facilitate testing. --- common/types.go | 16 +++++- dm/dm.go | 64 +++++++++++++++++++++-- manager/manager/manager.go | 11 ++++ registry/inmem_registry_service.go | 81 ++++++++++++++++++++++++++---- registry/registryprovider.go | 39 ++++++++------ 5 files changed, 180 insertions(+), 31 deletions(-) diff --git a/common/types.go b/common/types.go index fc6a3a909..f2a1d62f2 100644 --- a/common/types.go +++ b/common/types.go @@ -173,8 +173,11 @@ type BasicAuthCredential struct { Password string `json:"password"` } -// Credentials used to access the repository +type APITokenCredential string + +// Credential used to access the repository type RegistryCredential struct { + APIToken APITokenCredential `json:"apitoken,omitempty"` BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` } @@ -187,7 +190,7 @@ type Registry struct { Format RegistryFormat `json:"format,omitempty"` // Format of the registry } -// AuthenticatedRegistry describes a type registry with credentials. +// AuthenticatedRegistry describes a type registry with credential. // Broke this out of Registry, so that we can pass around instances of Registry // without worrying about secrets. type AuthenticatedRegistry struct { @@ -229,8 +232,17 @@ type RegistryService interface { Create(registry *Registry) error // Get a registry Get(name string) (*Registry, error) + // Get a registry with credential. + GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error) // Delete a registry Delete(name string) error // Find a registry that backs the given URL GetByURL(URL string) (*Registry, error) + // GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL. + GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error) + // Set the credential for a registry. + // May not be supported by some registry services. + SetCredential(name string, credential RegistryCredential) error + // Get the credential for a registry. + GetCredential(name string) (RegistryCredential, error) } diff --git a/dm/dm.go b/dm/dm.go index 677e8e9bc..5f0757137 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -22,6 +22,7 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/registry" + "github.com/kubernetes/deployment-manager/util" "archive/tar" "bytes" @@ -35,6 +36,7 @@ import ( "net/url" "os" "path" + "regexp" "strconv" "strings" "time" @@ -48,6 +50,8 @@ var ( service = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager") binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") timeout = flag.Int("timeout", 10, "Time in seconds to wait for response") + regex_string = flag.String("regex", "", "Regular expression to filter the templates listed in a template registry") + apitoken = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable") ) var commands = []string{ @@ -81,9 +85,52 @@ var usage = func() { panic("\n") } -var provider = registry.NewDefaultRegistryProvider() +// TODO(jackgr): Move all registry related operations to the server side. +var registryProvider registry.RegistryProvider + +func getRegistryProvider() registry.RegistryProvider { + if registryProvider == nil { + rs := registry.NewInmemRegistryService() + r, err := rs.GetByURL(*template_registry) + if err != nil { + r := newRegistry(*template_registry) + if err := rs.Create(r); err != nil { + panic(fmt.Errorf("cannot configure registry at %s: %s", r.URL, err)) + } + } + + if *apitoken == "" { + *apitoken = os.Getenv("DM_GITHUB_API_TOKEN") + } + + if *apitoken != "" { + credential := common.RegistryCredential{ + APIToken: common.APITokenCredential(*apitoken), + } + + if err := rs.SetCredential(r.Name, credential); err != nil { + panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err)) + } + } + + registryProvider = registry.NewRegistryProvider(rs, nil) + } + + return registryProvider +} + +func newRegistry(URL string) *common.Registry { + tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) + return &common.Registry{ + Name: util.TrimURLScheme(URL), + Type: common.GithubRegistryType, + URL: URL, + Format: common.RegistryFormat(tFormat), + } +} func getGithubRegistry() registry.Registry { + provider := getRegistryProvider() git, err := provider.GetRegistryByShortURL(*template_registry) if err != nil { panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err)) @@ -114,13 +161,21 @@ func execute() { switch args[0] { case "templates": + var regex *regexp.Regexp + if *regex_string != "" { + var err error + regex, err = regexp.Compile(*regex_string) + if err != nil { + panic(fmt.Errorf("cannot compile regular expression %s: %s", *regex_string, err)) + } + } + git := getGithubRegistry() - types, err := git.ListTypes(nil) + types, err := git.ListTypes(regex) if err != nil { - panic(fmt.Errorf("cannot list types in registry %s: %s", *template_registry, err)) + panic(fmt.Errorf("cannot list templates in registry %s: %s", *template_registry, err)) } - fmt.Printf("Templates:\n") for _, t := range types { fmt.Printf("%s\n", t.String()) urls, err := git.GetDownloadURLs(t) @@ -278,6 +333,7 @@ func describeType(args []string) { // getDownloadURLs returns URLs or empty list if a primitive type. func getDownloadURLs(tName string) []string { qName := path.Join(*template_registry, tName) + provider := getRegistryProvider() result, err := registry.GetDownloadURLs(provider, qName) if err != nil { panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err)) diff --git a/manager/manager/manager.go b/manager/manager/manager.go index 6ea47c19e..2e82c378c 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -343,6 +343,17 @@ func (m *manager) DeleteRegistry(name string) error { return m.service.Delete(name) } +// Set the credential for a registry. +// May not be supported by some registry services. +func (m *manager) SetCredential(name string, credential common.RegistryCredential) error { + return m.service.SetCredential(name, credential) +} + +// Get the credential for a registry. +func (m *manager) GetCredential(name string) (common.RegistryCredential, error) { + return m.service.GetCredential(name) +} + func generateManifestName() string { return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano()) } diff --git a/registry/inmem_registry_service.go b/registry/inmem_registry_service.go index 8affeea14..63f26fefc 100644 --- a/registry/inmem_registry_service.go +++ b/registry/inmem_registry_service.go @@ -17,19 +17,20 @@ limitations under the License. package registry import ( + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" + "fmt" "strings" - - "github.com/kubernetes/deployment-manager/common" ) type inmemRegistryService struct { - registries map[string]*common.Registry + registries map[string]*common.AuthenticatedRegistry } func NewInmemRegistryService() common.RegistryService { rs := &inmemRegistryService{ - registries: make(map[string]*common.Registry), + registries: make(map[string]*common.AuthenticatedRegistry), } pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) @@ -47,36 +48,98 @@ func NewInmemRegistryService() common.RegistryService { URL: "github.com/kubernetes/application-dm-templates", Format: common.RegistryFormat(tFormat), }) + return rs } +// List returns the list of known registries. func (rs *inmemRegistryService) List() ([]*common.Registry, error) { ret := []*common.Registry{} for _, r := range rs.registries { - ret = append(ret, r) + ret = append(ret, &r.Registry) } + return ret, nil } +// Create creates an authenticated registry. func (rs *inmemRegistryService) Create(registry *common.Registry) error { - rs.registries[registry.URL] = registry + rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry} return nil } +// Get returns a registry with a given name. func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) { - return &common.Registry{}, nil + r, ok := rs.registries[name] + if !ok { + return nil, fmt.Errorf("Failed to find registry named %s", name) + } + + return &r.Registry, nil } +// GetAuthenticatedRegistry returns an authenticated registry with a given name. +func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) { + r, ok := rs.registries[name] + if !ok { + return nil, fmt.Errorf("Failed to find registry named %s", name) + } + + return r, nil +} + +// Create deletes the authenticated registry with a given name. func (rs *inmemRegistryService) Delete(name string) error { + _, ok := rs.registries[name] + if !ok { + return fmt.Errorf("Failed to find registry named %s", name) + } + + delete(rs.registries, name) return nil } // GetByURL returns a registry that handles the types for a given URL. func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) { + trimmed := util.TrimURLScheme(URL) + for _, r := range rs.registries { + if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) { + return &r.Registry, nil + } + } + + return nil, fmt.Errorf("Failed to find registry for url: %s", URL) +} + +// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL. +func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) { + trimmed := util.TrimURLScheme(URL) for _, r := range rs.registries { - if strings.HasPrefix(URL, r.URL) { + if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) { return r, nil } } - return nil, fmt.Errorf("Failed to find registry for github url: %s", URL) + + return nil, fmt.Errorf("Failed to find registry for url: %s", URL) +} + +// Set the credential for a registry. +func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error { + r, ok := rs.registries[name] + if !ok { + return fmt.Errorf("Failed to find registry named %s", name) + } + + r.Credential = credential + return nil +} + +// Get the credential for a registry. +func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) { + r, ok := rs.registries[name] + if !ok { + return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name) + } + + return r.Credential, nil } diff --git a/registry/registryprovider.go b/registry/registryprovider.go index b25102e84..42a6e4e35 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -33,13 +33,15 @@ type RegistryProvider interface { GetRegistryByName(registryName string) (Registry, error) } -// GithubRegistryProvider is a factory for GithubRegistry instances. -type GithubRegistryProvider interface { - GetGithubRegistry(cr common.Registry) (GithubRegistry, error) +type registryProvider struct { + sync.RWMutex + rs common.RegistryService + grp GithubRegistryProvider + registries map[string]Registry } func NewDefaultRegistryProvider() RegistryProvider { - return NewRegistryProvider(nil, nil) + return NewRegistryProvider(nil, NewGithubRegistryProvider()) } func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider { @@ -47,23 +49,15 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) rs = NewInmemRegistryService() } - registries := make(map[string]Registry) - rp := ®istryProvider{rs: rs, registries: registries} if grp == nil { - grp = rp + grp = NewGithubRegistryProvider() } - rp.grp = grp + registries := make(map[string]Registry) + rp := ®istryProvider{rs: rs, grp: grp, registries: registries} return rp } -type registryProvider struct { - sync.RWMutex - rs common.RegistryService - grp GithubRegistryProvider - registries map[string]Registry -} - func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { rp.RLock() defer rp.RUnlock() @@ -133,7 +127,20 @@ func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]boo return result } -func (rp registryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { +// GithubRegistryProvider is a factory for GithubRegistry instances. +type GithubRegistryProvider interface { + GetGithubRegistry(cr common.Registry) (GithubRegistry, error) +} + +type githubRegistryProvider struct { +} + +// NewGithubRegistryProvider creates a GithubRegistryProvider. +func NewGithubRegistryProvider() GithubRegistryProvider { + return &githubRegistryProvider{} +} + +func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { if cr.Type == common.GithubRegistryType { fMap := ParseRegistryFormat(cr.Format) if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {