Merge pull request #172 from vaikas-google/github_auth

First cut of handling multiple registries with auth.

common/types.go

@@ -166,3 +166,37 @@ type KubernetesObject struct {
 	Metadata   map[string]interface{} `json:"metadata"`
 	Spec       map[string]interface{} `json:"spec"`
 }
+
+// Repository related types
+type BasicAuthCredential struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// Credentials used to access the repository
+type RegistryCredential struct {
+	BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
+}
+
+type Registry struct {
+	Name       string             `json:"name,omitempty"` // Friendly name for the repo
+	Type       RegistryType       `json:"type,omitempty"` // Technology implementing the registry
+	URL        string             `json:"name,omitempty"` // URL to the root of the repo, for example: github.com/helm/charts
+	Credential RegistryCredential `json:"credential,omitempty"`
+	Format     RegistryFormat     `json:"format,omitempty"`
+}
+
+// RegistryType defines the technology that implements the registry
+type RegistryType string
+
+const (
+	Github RegistryType = "github"
+)
+
+// RegistryFormat defines the format of the registry
+type RegistryFormat string
+
+const (
+	VersionedRegistry   RegistryFormat = "versioned"
+	UnversionedRegistry RegistryFormat = "unversioned"
+)

dm/dm.go

@@ -44,7 +44,7 @@ var (
 	deployment_name   = flag.String("name", "", "Name of deployment, used for deploy and update commands (defaults to template name)")
 	stdin             = flag.Bool("stdin", false, "Reads a configuration from the standard input")
 	properties        = flag.String("properties", "", "Properties to use when deploying a template (e.g., --properties k1=v1,k2=v2)")
-	template_registry = flag.String("registry", "kubernetes/application-dm-templates", "Github based template registry (owner/repo[/path])")
+	template_registry = flag.String("registry", "github.com/kubernetes/application-dm-templates", "Registry (github.com/owner/repo)")
 	service           = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
 	binary            = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
 	timeout           = flag.Int("timeout", 10, "Time in seconds to wait for response")
@@ -81,22 +81,8 @@ var usage = func() {
 	panic("\n")
 }
 
-func getGitRegistry() registry.Registry {
+func getGitRegistry() (registry.Registry, error) {
-	s := strings.Split(*template_registry, "/")
+	return registry.NewDefaultRegistryProvider().GetRegistry(*template_registry)
-	if len(s) < 2 {
-		panic(fmt.Errorf("invalid template registry: %s", *template_registry))
-	}
-
-	var path = ""
-	if len(s) > 2 {
-		path = strings.Join(s[2:], "/")
-	}
-
-	if s[0] == "helm" {
-		return registry.NewGithubPackageRegistry(s[0], s[1])
-	} else {
-		return registry.NewGithubRegistry(s[0], s[1], path)
-	}
 }
 
 func main() {
@@ -121,7 +107,10 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
-		git := getGitRegistry()
+		git, err := getGitRegistry()
+		if err != nil {
+			panic(fmt.Errorf("Cannot get registry %v", err))
+		}
 		templates, err := git.List()
 		if err != nil {
 			panic(fmt.Errorf("Cannot list %v", err))
@@ -305,7 +294,10 @@ func getTypeURLs(tName string) []string {
 }
 
 func getDownloadURLs(t registry.Type) []string {
-	git := getGitRegistry()
+	git, err := getGitRegistry()
+	if err != nil {
+		panic(fmt.Errorf("Failed to get registry"))
+	}
 	urls, err := git.GetURLs(t)
 	if err != nil {
 		panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))

manager/deployments.go

@@ -31,9 +31,10 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/gorilla/mux"
 
-	"github.com/kubernetes/deployment-manager/manager/manager"
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/manager/manager"
 	"github.com/kubernetes/deployment-manager/manager/repository"
+	"github.com/kubernetes/deployment-manager/registry"
 	"github.com/kubernetes/deployment-manager/util"
 )
 
@@ -48,6 +49,7 @@ var deployments = []Route{
 	{"Expand", "/expand", "POST", expandHandlerFunc, ""},
 	{"ListTypes", "/types", "GET", listTypesHandlerFunc, ""},
 	{"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""},
+	{"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""},
 }
 
 var (
@@ -70,10 +72,11 @@ func init() {
 }
 
 func newManager() manager.Manager {
-	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver())
+	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), manager.NewTypeResolver(registry.NewDefaultRegistryProvider()))
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
-	return manager.NewManager(expander, deployer, r)
+	registryService := registry.NewInmemRepositoryService()
+	return manager.NewManager(expander, deployer, r, registryService)
 }
 
 func getServiceURL(serviceURL, serviceName string) string {
@@ -329,3 +332,15 @@ func listTypeInstancesHandlerFunc(w http.ResponseWriter, r *http.Request) {
 
 	util.LogHandlerExitWithJSON(handler, w, backend.ListInstances(typeName), http.StatusOK)
 }
+
+// Putting Registry handlers here for now because deployments.go
+// currently owns its own Manager backend and doesn't like to share.
+func listRegistriesHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: list registries"
+	util.LogHandlerEntry(handler, r)
+	registries, err := backend.ListRegistries()
+	if err != nil {
+		return
+	}
+	util.LogHandlerExitWithJSON(handler, w, registries, http.StatusOK)
+}

manager/manager/manager.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/manager/repository"
+	"github.com/kubernetes/deployment-manager/registry"
 )
 
 // Manager manages a persistent set of Deployments.
@@ -37,17 +38,23 @@ type Manager interface {
 	Expand(t *common.Template) (*common.Manifest, error)
 	ListTypes() []string
 	ListInstances(typeName string) []*common.TypeInstance
+	// Registry related functions
+	ListRegistries() ([]*common.Registry, error)
+	CreateRegistry(pr *common.Registry) error
+	GetRegistry(name string) (*common.Registry, error)
+	DeleteRegistry(name string) error
 }
 
 type manager struct {
-	expander   Expander
+	expander        Expander
-	deployer   Deployer
+	deployer        Deployer
-	repository repository.Repository
+	repository      repository.Repository
+	registryService registry.RegistryService
 }
 
 // NewManager returns a new initialized Manager.
-func NewManager(expander Expander, deployer Deployer, repository repository.Repository) Manager {
+func NewManager(expander Expander, deployer Deployer, repository repository.Repository, registryService registry.RegistryService) Manager {
-	return &manager{expander, deployer, repository}
+	return &manager{expander, deployer, repository, registryService}
 }
 
 // ListDeployments returns the list of deployments
@@ -302,6 +309,22 @@ func (m *manager) ListInstances(typeName string) []*common.TypeInstance {
 	return m.repository.GetTypeInstances(typeName)
 }
 
+func (m *manager) ListRegistries() ([]*common.Registry, error) {
+	return m.registryService.List()
+}
+
+func (m *manager) CreateRegistry(pr *common.Registry) error {
+	return m.registryService.Create(pr)
+}
+
+func (m *manager) GetRegistry(name string) (*common.Registry, error) {
+	return m.registryService.Get(name)
+}
+
+func (m *manager) DeleteRegistry(name string) error {
+	return m.registryService.Delete(name)
+}
+
 func generateManifestName() string {
 	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
 }

manager/manager/manager_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/registry"
 )
 
 var template = common.Template{Name: "test", Content: "test"}
@@ -251,7 +252,8 @@ func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.Type
 var testExpander = &expanderStub{}
 var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
-var testManager = NewManager(testExpander, testDeployer, testRepository)
+var testRegistryService = registry.NewInmemRepositoryService()
+var testManager = NewManager(testExpander, testDeployer, testRepository, testRegistryService)
 
 func TestListDeployments(t *testing.T) {
 	testRepository.reset()

manager/manager/typeresolver.go

@@ -50,7 +50,7 @@ type fetchUnit struct {
 }
 
 // NewTypeResolver returns a new initialized TypeResolver.
-func NewTypeResolver() TypeResolver {
+func NewTypeResolver(rp registry.RegistryProvider) TypeResolver {
 	ret := &typeResolver{}
 	client := http.DefaultClient
 	//TODO (iantw): Make this a flag
@@ -58,7 +58,7 @@ func NewTypeResolver() TypeResolver {
 	client.Timeout = timeout
 	ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
 	ret.maxUrls = maxURLImports
-	ret.rp = &registry.DefaultRegistryProvider{}
+	ret.rp = rp
 	return ret
 }
 
@@ -244,7 +244,10 @@ func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, erro
 	if len(m) != 6 {
 		return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
 	}
-	r := tr.rp.GetGithubRegistry(m[1], m[2])
+	r, err := tr.rp.GetRegistry(template)
+	if err != nil {
+		return []string{}, err
+	}
 	t := registry.Type{m[3], m[4], m[5]}
 	return r.GetURLs(t)
 }
@@ -259,7 +262,10 @@ func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]strin
 	if len(m) != 4 {
 		return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
 	}
-	r := tr.rp.GetGithubPackageRegistry(m[1], m[2])
+	r, err := tr.rp.GetRegistry(template)
+	if err != nil {
+		return []string{}, err
+	}
 	t := registry.Type{Name: m[3]}
 	return r.GetURLs(t)
 }

manager/manager/typeresolver_test.go

@@ -64,23 +64,23 @@ type urlAndError struct {
 }
 
 type testRegistryProvider struct {
-	owner string
+	URLPrefix string
-	repo  string
+	r         map[string]registry.Registry
-	r     map[string]registry.Registry
 }
 
-func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
+func newTestRegistryProvider(URLPrefix string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
 	r := make(map[string]registry.Registry)
-	r[owner+repository] = &testGithubRegistry{tests, count}
+	r[URLPrefix] = &testGithubRegistry{tests, count}
-	return &testRegistryProvider{owner, repository, r}
+	return &testRegistryProvider{URLPrefix, r}
 }
 
-func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry {
+func (trp *testRegistryProvider) GetRegistry(URL string) (registry.Registry, error) {
-	return trp.r[owner+repository]
+	for key, r := range trp.r {
-}
+		if strings.HasPrefix(URL, key) {
-
+			return r, nil
-func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry {
+		}
-	return trp.r[owner+repository]
+	}
+	return nil, fmt.Errorf("No registry found for %s", URL)
 }
 
 type testGithubRegistry struct {
@@ -356,7 +356,7 @@ func TestShortGithubUrlMapping(t *testing.T) {
 	}
 
 	test := resolverTestCase{
-		registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
+		registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
 	}
 	testUrlConversionDriver(test, tests, t)
 }
@@ -373,7 +373,7 @@ func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
 	}
 
 	test := resolverTestCase{
-		registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2),
+		registryProvider: newTestRegistryProvider("github.com/example/mytemplates", githubUrlMaps, 2),
 	}
 	testUrlConversionDriver(test, tests, t)
 }
@@ -415,7 +415,7 @@ func TestShortGithubUrl(t *testing.T) {
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
+		registryProvider: newTestRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps, 2),
 	}
 	testDriver(test, t)
 }

registry/github_package_registry.go

@@ -39,16 +39,15 @@ type GithubPackageRegistry struct {
 }
 
 // NewGithubRegistry creates a Registry that can be used to talk to github.
-func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry {
+func NewGithubPackageRegistry(owner, repository string, client *github.Client) *GithubPackageRegistry {
 	return &GithubPackageRegistry{
 		owner:      owner,
 		repository: repository,
-		client:     github.NewClient(nil),
+		client:     client,
 	}
 }
 
 // List the types from the Registry.
-// TODO(vaikas): Figure out how the versions work here.
 func (g *GithubPackageRegistry) List() ([]Type, error) {
 	// Just list all the types at the top level.
 	types, err := g.getDirs("")

registry/github_registry.go

@@ -55,12 +55,12 @@ type GithubRegistry struct {
 }
 
 // NewGithubRegistry creates a Registry that can be used to talk to github.
-func NewGithubRegistry(owner, repository, path string) *GithubRegistry {
+func NewGithubRegistry(owner, repository, path string, client *github.Client) *GithubRegistry {
 	return &GithubRegistry{
 		owner:      owner,
 		repository: repository,
 		path:       path,
-		client:     github.NewClient(nil),
+		client:     client,
 	}
 }
 

registry/inmem_repository_service.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kubernetes/deployment-manager/common"
+)
+
+type inmemRepositoryService struct {
+	repositories map[string]*common.Registry
+}
+
+func NewInmemRepositoryService() RegistryService {
+	rs := &inmemRepositoryService{
+		repositories: make(map[string]*common.Registry),
+	}
+	rs.Create(&common.Registry{
+		Name:   "charts",
+		Type:   common.Github,
+		URL:    "github.com/helm/charts",
+		Format: common.UnversionedRegistry,
+	})
+	rs.Create(&common.Registry{
+		Name:   "application-dm-templates",
+		Type:   common.Github,
+		URL:    "github.com/kubernetes/application-dm-templates",
+		Format: common.VersionedRegistry,
+	})
+	return rs
+}
+
+func (rs *inmemRepositoryService) List() ([]*common.Registry, error) {
+	ret := []*common.Registry{}
+	for _, r := range rs.repositories {
+		ret = append(ret, r)
+	}
+	return ret, nil
+}
+
+func (rs *inmemRepositoryService) Create(repository *common.Registry) error {
+	rs.repositories[repository.URL] = repository
+	return nil
+}
+
+func (rs *inmemRepositoryService) Get(name string) (*common.Registry, error) {
+	return &common.Registry{}, nil
+}
+
+func (rs *inmemRepositoryService) Delete(name string) error {
+	return nil
+}
+
+// GetByURL returns a registry that handles the types for a given URL.
+func (rs *inmemRepositoryService) GetByURL(URL string) (*common.Registry, error) {
+	for _, r := range rs.repositories {
+		if strings.HasPrefix(URL, r.URL) {
+			return r, nil
+		}
+	}
+	return nil, fmt.Errorf("Failed to find registry for github url: %s", URL)
+}

registry/registry.go

@@ -16,6 +16,23 @@ limitations under the License.
 
 package registry
 
+import (
+	"github.com/kubernetes/deployment-manager/common"
+)
+
+type RegistryService interface {
+	// List all the registries
+	List() ([]*common.Registry, error)
+	// Create a new registry
+	Create(repository *common.Registry) error
+	// Get a registry
+	Get(name string) (*common.Registry, error)
+	// Delete a registry
+	Delete(name string) error
+	// Find a registry that backs the given URL
+	GetByURL(URL string) (*common.Registry, error)
+}
+
 // Registry abstracts a registry that holds templates, which can be
 // used in a Deployment Manager configurations. There can be multiple
 // implementations of a registry. Currently we support Deployment Manager

registry/registryprovider.go

@@ -16,19 +16,40 @@ limitations under the License.
 
 package registry
 
+import (
+	"fmt"
+
+	"github.com/google/go-github/github"
+	"github.com/kubernetes/deployment-manager/common"
+)
+
 // RegistryProvider returns factories for creating registries for a given RegistryType.
 type RegistryProvider interface {
-	GetGithubRegistry(owner string, repository string) Registry
+	GetRegistry(prefix string) (Registry, error)
-	GetGithubPackageRegistry(owner string, repository string) Registry
 }
 
-type DefaultRegistryProvider struct {
+func NewDefaultRegistryProvider() RegistryProvider {
+	rs := NewInmemRepositoryService()
+	return &DefaultRegistryProvider{rs: rs}
+
 }
 
-func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
+type DefaultRegistryProvider struct {
-	return NewGithubRegistry(owner, repository, "")
+	rs RegistryService
 }
 
-func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry {
+func (drp *DefaultRegistryProvider) GetRegistry(URL string) (Registry, error) {
-	return NewGithubPackageRegistry(owner, repository)
+	r, err := drp.rs.GetByURL(URL)
+	if err != nil {
+		return nil, err
+	}
+	if r.Type == common.Github {
+		if r.Format == common.UnversionedRegistry {
+			return NewGithubPackageRegistry("helm", "charts", github.NewClient(nil)), nil
+		}
+		if r.Format == common.VersionedRegistry {
+			return NewGithubRegistry("kubernetes", "application-dm-templates", "", github.NewClient(nil)), nil
+		}
+	}
+	return nil, fmt.Errorf("cannot find registry backing url %s", URL)
 }