Merge pull request #182 from vaikas-google/master

Add support for CredentialProvider

common/types.go

@@ -184,18 +184,11 @@ type RegistryCredential struct {
 // Registry describes a template registry
 // TODO(jackr): Fix ambiguity re: whether or not URL has a scheme.
 type Registry struct {
-	Name   string         `json:"name,omitempty"`   // Friendly name for the registry
+	Name           string         `json:"name,omitempty"`           // Friendly name for the registry
-	Type   RegistryType   `json:"type,omitempty"`   // Technology implementing the registry
+	Type           RegistryType   `json:"type,omitempty"`           // Technology implementing the registry
-	URL    string         `json:"name,omitempty"`   // URL to the root of the registry
+	URL            string         `json:"name,omitempty"`           // URL to the root of the registry
-	Format RegistryFormat `json:"format,omitempty"` // Format of the registry
+	Format         RegistryFormat `json:"format,omitempty"`         // Format of the registry
-}
+	CredentialName string         `json:"credentialname,omitempty"` // Name of the credential to use
-
-// AuthenticatedRegistry describes a type registry with credential.
-// Broke this out of Registry, so that we can pass around instances of Registry
-// without worrying about secrets.
-type AuthenticatedRegistry struct {
-	Registry
-	Credential RegistryCredential `json:"credential,omitempty"`
 }
 
 // RegistryType defines the technology that implements the registry
@@ -233,16 +226,22 @@ type RegistryService interface {
 	// Get a registry
 	Get(name string) (*Registry, error)
 	// Get a registry with credential.
-	GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error)
+	GetRegistry(name string) (*Registry, error)
 	// Delete a registry
 	Delete(name string) error
 	// Find a registry that backs the given URL
 	GetByURL(URL string) (*Registry, error)
-	// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+	// GetRegistryByURL returns a registry that handles the types for a given URL.
-	GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error)
+	GetRegistryByURL(URL string) (*Registry, error)
+}
+
+// CredentialProvider provides credentials for registries.
+type CredentialProvider interface {
 	// Set the credential for a registry.
 	// May not be supported by some registry services.
-	SetCredential(name string, credential RegistryCredential) error
+	SetCredential(name string, credential *RegistryCredential) error
-	// Get the credential for a registry.
+
-	GetCredential(name string) (RegistryCredential, error)
+	// GetCredential returns the specified credential or nil if there's no credential.
+	// Error is non-nil if fetching the credential failed.
+	GetCredential(name string) (*RegistryCredential, error)
 }

dm/dm.go

@@ -91,7 +91,7 @@ var registryProvider registry.RegistryProvider
 func getRegistryProvider() registry.RegistryProvider {
 	if registryProvider == nil {
 		rs := registry.NewInmemRegistryService()
-		r, err := rs.GetByURL(*template_registry)
+		_, err := rs.GetByURL(*template_registry)
 		if err != nil {
 			r := newRegistry(*template_registry)
 			if err := rs.Create(r); err != nil {
@@ -99,6 +99,7 @@ func getRegistryProvider() registry.RegistryProvider {
 			}
 		}
 
+		cp := registry.NewInmemCredentialProvider()
 		if *apitoken == "" {
 			*apitoken = os.Getenv("DM_GITHUB_API_TOKEN")
 		}
@@ -107,13 +108,12 @@ func getRegistryProvider() registry.RegistryProvider {
 			credential := common.RegistryCredential{
 				APIToken: common.APITokenCredential(*apitoken),
 			}
-
+			if err := cp.SetCredential("default", &credential); err != nil {
-			if err := rs.SetCredential(r.Name, credential); err != nil {
+				panic(fmt.Errorf("cannot set credential at %s: %s", "default", err))
-				panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err))
 			}
 		}
 
-		registryProvider = registry.NewRegistryProvider(rs, nil)
+		registryProvider = registry.NewRegistryProvider(rs, nil, cp)
 	}
 
 	return registryProvider
@@ -122,10 +122,11 @@ func getRegistryProvider() registry.RegistryProvider {
 func newRegistry(URL string) *common.Registry {
 	tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	return &common.Registry{
-		Name:   util.TrimURLScheme(URL),
+		Name:           util.TrimURLScheme(URL),
-		Type:   common.GithubRegistryType,
+		Type:           common.GithubRegistryType,
-		URL:    URL,
+		URL:            URL,
-		Format: common.RegistryFormat(tFormat),
+		Format:         common.RegistryFormat(tFormat),
+		CredentialName: "default",
 	}
 }
 

manager/deployments.go

@@ -56,14 +56,17 @@ var deployments = []Route{
 	{"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""},
 	{"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""},
 	{"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""},
+	{"CreateCredential", "/credentials/{credential}", "POST", createCredentialHandlerFunc, "JSON"},
+	{"GetCredential", "/credentials/{credential}", "GET", getCredentialHandlerFunc, ""},
 }
 
 var (
-	maxLength    = flag.Int64("maxLength", 1024, "The maximum length (KB) of a template.")
+	maxLength      = flag.Int64("maxLength", 1024, "The maximum length (KB) of a template.")
-	expanderName = flag.String("expander", "expandybird-service", "The DNS name of the expander service.")
+	expanderName   = flag.String("expander", "expandybird-service", "The DNS name of the expander service.")
-	expanderURL  = flag.String("expanderURL", "", "The URL for the expander service.")
+	expanderURL    = flag.String("expanderURL", "", "The URL for the expander service.")
-	deployerName = flag.String("deployer", "resourcifier-service", "The DNS name of the deployer service.")
+	deployerName   = flag.String("deployer", "resourcifier-service", "The DNS name of the deployer service.")
-	deployerURL  = flag.String("deployerURL", "", "The URL for the deployer service.")
+	deployerURL    = flag.String("deployerURL", "", "The URL for the deployer service.")
+	credentialFile = flag.String("credentialFile", "", "Local file to use for credentials.")
 )
 
 var backend manager.Manager
@@ -74,17 +77,28 @@ func init() {
 	}
 
 	routes = append(routes, deployments...)
-	backend = newManager()
+	var credentialProvider common.CredentialProvider
+	if *credentialFile != "" {
+		var err error
+		credentialProvider, err = registry.NewFilebasedCredentialProvider(*credentialFile)
+		if err != nil {
+			panic(fmt.Errorf("cannot create credential provider %s: %s", *credentialFile, err))
+		}
+	} else {
+		credentialProvider = registry.NewInmemCredentialProvider()
+	}
+	backend = newManager(credentialProvider)
 }
 
-func newManager() manager.Manager {
+func newManager(cp common.CredentialProvider) manager.Manager {
-	provider := registry.NewDefaultRegistryProvider()
+	registryProvider := registry.NewDefaultRegistryProvider(cp)
-	resolver := manager.NewTypeResolver(provider)
+	resolver := manager.NewTypeResolver(registryProvider)
 	expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver)
 	deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName))
 	r := repository.NewMapBasedRepository()
 	service := registry.NewInmemRegistryService()
-	return manager.NewManager(expander, deployer, r, provider, service)
+	credentialProvider := cp
+	return manager.NewManager(expander, deployer, r, registryProvider, service, credentialProvider)
 }
 
 func getServiceURL(serviceURL, serviceName string) string {
@@ -232,30 +246,9 @@ func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler s
 
 func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
 	util.LogHandlerEntry(handler, r)
-	b := io.LimitReader(r.Body, *maxLength*1024)
+	j, err := getJsonFromRequest(w, r, handler)
-	y, err := ioutil.ReadAll(b)
-	if err != nil {
-		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
-		return nil
-	}
-
-	// Reject the input if it exceeded the length limit,
-	// since we may not have read all of it into the buffer.
-	if _, err = b.Read(make([]byte, 0, 1)); err != io.EOF {
-		e := fmt.Errorf("template exceeds maximum length of %d KB", *maxLength)
-		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
-		return nil
-	}
-
-	if err := r.Body.Close(); err != nil {
-		util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
-		return nil
-	}
 
-	j, err := yaml.YAMLToJSON(y)
 	if err != nil {
-		e := fmt.Errorf("%v\n%v", err, string(y))
-		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
 		return nil
 	}
 
@@ -433,3 +426,83 @@ func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) {
 
 	util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK)
 }
+
+func getCredential(w http.ResponseWriter, r *http.Request, handler string) *common.RegistryCredential {
+	util.LogHandlerEntry(handler, r)
+	j, err := getJsonFromRequest(w, r, handler)
+	if err != nil {
+		return nil
+	}
+
+	t := &common.RegistryCredential{}
+	if err := json.Unmarshal(j, t); err != nil {
+		e := fmt.Errorf("%v\n%v", err, string(j))
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return nil
+	}
+
+	return t
+}
+
+func createCredentialHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: create credential"
+	util.LogHandlerEntry(handler, r)
+	defer r.Body.Close()
+	credentialName, err := getPathVariable(w, r, "credential", handler)
+	if err != nil {
+		return
+	}
+
+	c := getCredential(w, r, handler)
+	if c != nil {
+		err = backend.CreateCredential(credentialName, c)
+		if err != nil {
+			util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+			return
+		}
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK)
+}
+
+func getCredentialHandlerFunc(w http.ResponseWriter, r *http.Request) {
+	handler := "manager: get credential"
+	util.LogHandlerEntry(handler, r)
+	credentialName, err := getPathVariable(w, r, "credential", handler)
+	if err != nil {
+		return
+	}
+
+	c, err := backend.GetCredential(credentialName)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return
+	}
+
+	util.LogHandlerExitWithJSON(handler, w, c, http.StatusOK)
+}
+
+func getJsonFromRequest(w http.ResponseWriter, r *http.Request, handler string) ([]byte, error) {
+	util.LogHandlerEntry(handler, r)
+	b := io.LimitReader(r.Body, *maxLength*1024)
+	y, err := ioutil.ReadAll(b)
+	if err != nil {
+		util.LogAndReturnError(handler, http.StatusBadRequest, err, w)
+		return []byte{}, err
+	}
+
+	// Reject the input if it exceeded the length limit,
+	// since we may not have read all of it into the buffer.
+	if _, err = b.Read(make([]byte, 0, 1)); err != io.EOF {
+		e := fmt.Errorf("template exceeds maximum length of %d KB", *maxLength)
+		util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
+		return []byte{}, err
+	}
+
+	if err := r.Body.Close(); err != nil {
+		util.LogAndReturnError(handler, http.StatusInternalServerError, err, w)
+		return []byte{}, err
+	}
+
+	return yaml.YAMLToJSON(y)
+}

manager/manager/manager.go

@@ -55,23 +55,29 @@ type Manager interface {
 	// Registry Types
 	ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error)
 	GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error)
+
+	// Credentials
+	CreateCredential(name string, c *common.RegistryCredential) error
+	GetCredential(name string) (*common.RegistryCredential, error)
 }
 
 type manager struct {
-	expander   Expander
+	expander           Expander
-	deployer   Deployer
+	deployer           Deployer
-	repository repository.Repository
+	repository         repository.Repository
-	provider   registry.RegistryProvider
+	registryProvider   registry.RegistryProvider
-	service    common.RegistryService
+	service            common.RegistryService
+	credentialProvider common.CredentialProvider
 }
 
 // NewManager returns a new initialized Manager.
 func NewManager(expander Expander,
 	deployer Deployer,
 	repository repository.Repository,
-	provider registry.RegistryProvider,
+	registryProvider registry.RegistryProvider,
-	service common.RegistryService) Manager {
+	service common.RegistryService,
-	return &manager{expander, deployer, repository, provider, service}
+	credentialProvider common.CredentialProvider) Manager {
+	return &manager{expander, deployer, repository, registryProvider, service, credentialProvider}
 }
 
 // ListDeployments returns the list of deployments
@@ -343,17 +349,6 @@ func (m *manager) DeleteRegistry(name string) error {
 	return m.service.Delete(name)
 }
 
-// Set the credential for a registry.
-// May not be supported by some registry services.
-func (m *manager) SetCredential(name string, credential common.RegistryCredential) error {
-	return m.service.SetCredential(name, credential)
-}
-
-// Get the credential for a registry.
-func (m *manager) GetCredential(name string) (common.RegistryCredential, error) {
-	return m.service.GetCredential(name)
-}
-
 func generateManifestName() string {
 	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
 }
@@ -380,7 +375,7 @@ func getResourceErrors(c *common.Configuration) []string {
 // conform to the supplied regular expression, or all types, if the regular
 // expression is nil.
 func (m *manager) ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) {
-	r, err := m.provider.GetRegistryByName(registryName)
+	r, err := m.registryProvider.GetRegistryByName(registryName)
 	if err != nil {
 		return nil, err
 	}
@@ -391,10 +386,19 @@ func (m *manager) ListRegistryTypes(registryName string, regex *regexp.Regexp) (
 // GetDownloadURLs returns the URLs required to download the contents
 // of a given type in a given registry.
 func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) {
-	r, err := m.provider.GetRegistryByName(registryName)
+	r, err := m.registryProvider.GetRegistryByName(registryName)
 	if err != nil {
 		return nil, err
 	}
 
 	return r.GetDownloadURLs(t)
 }
+
+// CreateCredential creates a credential that can be used to authenticate to registry
+func (m *manager) CreateCredential(name string, c *common.RegistryCredential) error {
+	return m.credentialProvider.SetCredential(name, c)
+}
+
+func (m *manager) GetCredential(name string) (*common.RegistryCredential, error) {
+	return m.credentialProvider.GetCredential(name)
+}

manager/manager/manager_test.go

@@ -254,8 +254,9 @@ var testExpander = &expanderStub{}
 var testRepository = newRepositoryStub()
 var testDeployer = newDeployerStub()
 var testRegistryService = registry.NewInmemRegistryService()
-var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil))
+var testCredentialProvider = registry.NewInmemCredentialProvider()
-var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService)
+var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), testCredentialProvider)
+var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService, testCredentialProvider)
 
 func TestListDeployments(t *testing.T) {
 	testRepository.reset()

manager/manager/typeresolver_test.go

@@ -313,7 +313,7 @@ func TestShortGithubUrl(t *testing.T) {
 		importOut:        finalImports,
 		urlcount:         4,
 		responses:        responses,
-		registryProvider: registry.NewRegistryProvider(nil, grp),
+		registryProvider: registry.NewRegistryProvider(nil, grp, registry.NewInmemCredentialProvider()),
 	}
 
 	testDriver(test, t)

registry/filebased_credential_provider.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+
+	"github.com/ghodss/yaml"
+	"github.com/kubernetes/deployment-manager/common"
+
+	/*
+		"net/url"
+		"regexp"
+		"strings"
+		"sync"
+	*/)
+
+// CredentialProvider provides credentials for registries.
+type FilebasedCredentialProvider struct {
+	// Actual backing store
+	backingCredentialProvider common.CredentialProvider
+}
+
+type NamedRegistryCredential struct {
+	Name string `json:"name,omitempty"`
+	common.RegistryCredential
+}
+
+func NewFilebasedCredentialProvider(filename string) (common.CredentialProvider, error) {
+	icp := NewInmemCredentialProvider()
+	c, err := readCredentialsFile(filename)
+	if err != nil {
+		return &FilebasedCredentialProvider{}, err
+	}
+	for _, nc := range c {
+		log.Printf("Adding credential %s", nc.Name)
+		icp.SetCredential(nc.Name, &nc.RegistryCredential)
+	}
+
+	return &FilebasedCredentialProvider{icp}, nil
+}
+
+func readCredentialsFile(filename string) ([]NamedRegistryCredential, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return []NamedRegistryCredential{}, err
+	}
+	return parseCredentials(bytes)
+}
+
+func parseCredentials(bytes []byte) ([]NamedRegistryCredential, error) {
+	r := []NamedRegistryCredential{}
+	if err := yaml.Unmarshal(bytes, &r); err != nil {
+		return []NamedRegistryCredential{}, fmt.Errorf("cannot unmarshal credentials file (%#v)", err)
+	}
+	return r, nil
+}
+
+func (fcp *FilebasedCredentialProvider) GetCredential(name string) (*common.RegistryCredential, error) {
+	return fcp.backingCredentialProvider.GetCredential(name)
+}
+
+func (fcp *FilebasedCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error {
+	return fmt.Errorf("SetCredential operation not supported with FilebasedCredentialProvider")
+}

registry/filebased_credential_provider_test.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"testing"
+
+	"github.com/kubernetes/deployment-manager/common"
+)
+
+var filename = "./test/test_credentials_file.yaml"
+
+type filebasedTestCase struct {
+	name   string
+	exp    *common.RegistryCredential
+	expErr error
+}
+
+func TestNotExistFilebased(t *testing.T) {
+	cp, err := NewFilebasedCredentialProvider(filename)
+	if err != nil {
+		t.Fatalf("Failed to create a new FilebasedCredentialProvider %s : %v", filename, err)
+	}
+	tc := &testCase{"nonexistent", nil, createMissingError("nonexistent")}
+	testGetCredential(t, cp, tc)
+}
+
+func TestGetApiTokenFilebased(t *testing.T) {
+	cp, err := NewFilebasedCredentialProvider(filename)
+	if err != nil {
+		t.Fatalf("Failed to create a new FilebasedCredentialProvider %s : %v", filename, err)
+	}
+	tc := &testCase{"test1", &common.RegistryCredential{APIToken: "token"}, nil}
+	testGetCredential(t, cp, tc)
+}
+
+func TestSetAndGetBasicAuthFilebased(t *testing.T) {
+	cp, err := NewFilebasedCredentialProvider(filename)
+	if err != nil {
+		t.Fatalf("Failed to create a new FilebasedCredentialProvider %s : %v", filename, err)
+	}
+	tc := &testCase{"test2",
+		&common.RegistryCredential{
+			BasicAuth: common.BasicAuthCredential{"user", "password"}}, nil}
+	testGetCredential(t, cp, tc)
+}

registry/github_package_registry.go

@@ -41,11 +41,14 @@ type GithubPackageRegistry struct {
 }
 
 // NewGithubPackageRegistry creates a GithubPackageRegistry.
-func NewGithubPackageRegistry(name, shortURL string, service RepositoryService) (*GithubPackageRegistry, error) {
+func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubPackageRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
 	if service == nil {
-		client := github.NewClient(nil)
+		if client == nil {
-		service = client.Repositories
+			service = github.NewClient(nil).Repositories
+		} else {
+			service = client.Repositories
+		}
 	}
 
 	gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service)

registry/github_registry.go

@@ -25,20 +25,23 @@ import (
 	"strings"
 )
 
-// githubRegistry implements the Registry interface and talks to github.
+// githubRegistry is the base class for the Registry interface and talks to github. Actual implementations are
+// in GithubPackageRegistry and GithubTemplateRegistry.
 // The registry short URL and format determine how types are laid out in the
 // registry.
 type githubRegistry struct {
-	name       string
+	name           string
-	shortURL   string
+	shortURL       string
-	owner      string
+	owner          string
-	repository string
+	repository     string
-	path       string
+	path           string
-	format     common.RegistryFormat
+	format         common.RegistryFormat
-	service    RepositoryService
+	credentialName string
+	service        GithubRepositoryService
 }
 
-type RepositoryService interface {
+// GithubRepositoryService defines the interface that's defined in github.com/go-github/repos_contents.go GetContents method.
+type GithubRepositoryService interface {
 	GetContents(
 		owner, repo, path string,
 		opt *github.RepositoryContentGetOptions,
@@ -51,7 +54,7 @@ type RepositoryService interface {
 }
 
 // newGithubRegistry creates a githubRegistry.
-func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service RepositoryService) (*githubRegistry, error) {
+func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service GithubRepositoryService) (*githubRegistry, error) {
 	trimmed := util.TrimURLScheme(shortURL)
 	owner, repository, path, err := parseGithubShortURL(trimmed)
 	if err != nil {

registry/github_template_registry.go

@@ -54,10 +54,9 @@ type GithubTemplateRegistry struct {
 }
 
 // NewGithubTemplateRegistry creates a GithubTemplateRegistry.
-func NewGithubTemplateRegistry(name, shortURL string, service RepositoryService) (*GithubTemplateRegistry, error) {
+func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubTemplateRegistry, error) {
 	format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	if service == nil {
-		client := github.NewClient(nil)
 		service = client.Repositories
 	}
 

registry/inmem_credential_provider.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"github.com/kubernetes/deployment-manager/common"
+
+	"fmt"
+)
+
+type InmemCredentialProvider struct {
+	credentials map[string]*common.RegistryCredential
+}
+
+func NewInmemCredentialProvider() common.CredentialProvider {
+	return &InmemCredentialProvider{credentials: make(map[string]*common.RegistryCredential)}
+}
+
+func (fcp *InmemCredentialProvider) GetCredential(name string) (*common.RegistryCredential, error) {
+	if val, ok := fcp.credentials[name]; ok {
+		return val, nil
+	}
+	return nil, fmt.Errorf("no such credential : %s", name)
+}
+
+func (fcp *InmemCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error {
+	fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth}
+	return nil
+}

registry/inmem_credential_provider_test.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package registry
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/kubernetes/deployment-manager/common"
+)
+
+type testCase struct {
+	name   string
+	exp    *common.RegistryCredential
+	expErr error
+}
+
+func createMissingError(name string) error {
+	return fmt.Errorf("no such credential : %s", name)
+}
+
+func testGetCredential(t *testing.T, cp common.CredentialProvider, tc *testCase) {
+	actual, actualErr := cp.GetCredential(tc.name)
+	if !reflect.DeepEqual(actual, tc.exp) {
+		t.Fatalf("failed on: %s : expected %#v but got %#v", tc.name, tc.exp, actual)
+	}
+	if !reflect.DeepEqual(actualErr, tc.expErr) {
+		t.Fatalf("failed on: %s : expected error %#v but got %#v", tc.name, tc.expErr, actualErr)
+	}
+}
+
+func verifySetAndGetCredential(t *testing.T, cp common.CredentialProvider, tc *testCase) {
+	err := cp.SetCredential(tc.name, tc.exp)
+	if err != nil {
+		t.Fatalf("Failed to SetCredential on %s : %v", tc.name, err)
+	}
+	testGetCredential(t, cp, tc)
+}
+
+func TestNotExist(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	tc := &testCase{"nonexistent", nil, createMissingError("nonexistent")}
+	testGetCredential(t, cp, tc)
+}
+
+func TestSetAndGetApiToken(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	tc := &testCase{"testcredential", &common.RegistryCredential{APIToken: "some token here"}, nil}
+	verifySetAndGetCredential(t, cp, tc)
+}
+
+func TestSetAndGetBasicAuth(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	tc := &testCase{"testcredential",
+		&common.RegistryCredential{
+			BasicAuth: common.BasicAuthCredential{"user", "pass"}}, nil}
+	verifySetAndGetCredential(t, cp, tc)
+}

registry/inmem_registry_service.go

@@ -25,28 +25,30 @@ import (
 )
 
 type inmemRegistryService struct {
-	registries map[string]*common.AuthenticatedRegistry
+	registries map[string]*common.Registry
 }
 
 func NewInmemRegistryService() common.RegistryService {
 	rs := &inmemRegistryService{
-		registries: make(map[string]*common.AuthenticatedRegistry),
+		registries: make(map[string]*common.Registry),
 	}
 
 	pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
 	rs.Create(&common.Registry{
-		Name:   "charts",
+		Name:           "charts",
-		Type:   common.GithubRegistryType,
+		Type:           common.GithubRegistryType,
-		URL:    "github.com/helm/charts",
+		URL:            "github.com/helm/charts",
-		Format: common.RegistryFormat(pFormat),
+		Format:         common.RegistryFormat(pFormat),
+		CredentialName: "default",
 	})
 
 	tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
 	rs.Create(&common.Registry{
-		Name:   "application-dm-templates",
+		Name:           "application-dm-templates",
-		Type:   common.GithubRegistryType,
+		Type:           common.GithubRegistryType,
-		URL:    "github.com/kubernetes/application-dm-templates",
+		URL:            "github.com/kubernetes/application-dm-templates",
-		Format: common.RegistryFormat(tFormat),
+		Format:         common.RegistryFormat(tFormat),
+		CredentialName: "default",
 	})
 
 	return rs
@@ -56,15 +58,15 @@ func NewInmemRegistryService() common.RegistryService {
 func (rs *inmemRegistryService) List() ([]*common.Registry, error) {
 	ret := []*common.Registry{}
 	for _, r := range rs.registries {
-		ret = append(ret, &r.Registry)
+		ret = append(ret, r)
 	}
 
 	return ret, nil
 }
 
-// Create creates an authenticated registry.
+// Create creates a registry.
 func (rs *inmemRegistryService) Create(registry *common.Registry) error {
-	rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry}
+	rs.registries[registry.Name] = registry
 	return nil
 }
 
@@ -75,11 +77,11 @@ func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) {
 		return nil, fmt.Errorf("Failed to find registry named %s", name)
 	}
 
-	return &r.Registry, nil
+	return r, nil
 }
 
-// GetAuthenticatedRegistry returns an authenticated registry with a given name.
+// GetRegistry returns a registry with a given name.
-func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) {
+func (rs *inmemRegistryService) GetRegistry(name string) (*common.Registry, error) {
 	r, ok := rs.registries[name]
 	if !ok {
 		return nil, fmt.Errorf("Failed to find registry named %s", name)
@@ -88,7 +90,7 @@ func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.A
 	return r, nil
 }
 
-// Create deletes the authenticated registry with a given name.
+// Delete deletes the registry with a given name.
 func (rs *inmemRegistryService) Delete(name string) error {
 	_, ok := rs.registries[name]
 	if !ok {
@@ -104,15 +106,15 @@ func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) {
 	trimmed := util.TrimURLScheme(URL)
 	for _, r := range rs.registries {
 		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
-			return &r.Registry, nil
+			return r, nil
 		}
 	}
 
 	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
 }
 
-// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+// GetRegistryByURL returns a registry that handles the types for a given URL.
-func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) {
+func (rs *inmemRegistryService) GetRegistryByURL(URL string) (*common.Registry, error) {
 	trimmed := util.TrimURLScheme(URL)
 	for _, r := range rs.registries {
 		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
@@ -122,24 +124,3 @@ func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*comm
 
 	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
 }
-
-// Set the credential for a registry.
-func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error {
-	r, ok := rs.registries[name]
-	if !ok {
-		return fmt.Errorf("Failed to find registry named %s", name)
-	}
-
-	r.Credential = credential
-	return nil
-}
-
-// Get the credential for a registry.
-func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) {
-	r, ok := rs.registries[name]
-	if !ok {
-		return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name)
-	}
-
-	return r.Credential, nil
-}

registry/registryprovider.go

@@ -17,10 +17,13 @@ limitations under the License.
 package registry
 
 import (
+	"github.com/google/go-github/github"
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/util"
+	"golang.org/x/oauth2"
 
 	"fmt"
+	"log"
 	"net/url"
 	"regexp"
 	"strings"
@@ -37,24 +40,29 @@ type registryProvider struct {
 	sync.RWMutex
 	rs         common.RegistryService
 	grp        GithubRegistryProvider
+	cp         common.CredentialProvider
 	registries map[string]Registry
 }
 
-func NewDefaultRegistryProvider() RegistryProvider {
+func NewDefaultRegistryProvider(cp common.CredentialProvider) RegistryProvider {
-	return NewRegistryProvider(nil, NewGithubRegistryProvider())
+	return NewRegistryProvider(nil, NewGithubRegistryProvider(cp), cp)
 }
 
-func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider {
+func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, cp common.CredentialProvider) RegistryProvider {
 	if rs == nil {
 		rs = NewInmemRegistryService()
 	}
 
+	if cp == nil {
+		cp = NewInmemCredentialProvider()
+	}
+
 	if grp == nil {
-		grp = NewGithubRegistryProvider()
+		grp = NewGithubRegistryProvider(cp)
 	}
 
 	registries := make(map[string]Registry)
-	rp := &registryProvider{rs: rs, grp: grp, registries: registries}
+	rp := &registryProvider{rs: rs, grp: grp, cp: cp, registries: registries}
 	return rp
 }
 
@@ -98,23 +106,19 @@ func (rp registryProvider) GetRegistryByName(registryName string) (Registry, err
 	rp.RLock()
 	defer rp.RUnlock()
 
-	result, ok := rp.registries[registryName]
+	cr, err := rp.rs.Get(registryName)
-	if !ok {
+	if err != nil {
-		cr, err := rp.rs.Get(registryName)
+		return nil, err
-		if err != nil {
+	}
-			return nil, err
-		}
-
-		r, err := rp.grp.GetGithubRegistry(*cr)
-		if err != nil {
-			return nil, err
-		}
 
-		rp.registries[r.GetRegistryName()] = r
+	r, err := rp.grp.GetGithubRegistry(*cr)
-		result = r
+	if err != nil {
+		return nil, err
 	}
 
-	return result, nil
+	rp.registries[r.GetRegistryName()] = r
+
+	return r, nil
 }
 
 func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]bool {
@@ -133,22 +137,67 @@ type GithubRegistryProvider interface {
 }
 
 type githubRegistryProvider struct {
+	cp common.CredentialProvider
 }
 
 // NewGithubRegistryProvider creates a GithubRegistryProvider.
-func NewGithubRegistryProvider() GithubRegistryProvider {
+func NewGithubRegistryProvider(cp common.CredentialProvider) GithubRegistryProvider {
-	return &githubRegistryProvider{}
+	if cp == nil {
+		panic(fmt.Errorf("CP IS NIL: %v", cp))
+	}
+	return &githubRegistryProvider{cp: cp}
+}
+
+func (grp githubRegistryProvider) createGithubClient(credentialName string) (*github.Client, error) {
+	if credentialName == "" {
+		return github.NewClient(nil), nil
+	}
+	c, err := grp.cp.GetCredential(credentialName)
+
+	if err != nil {
+		log.Printf("Failed to fetch credential %s: %v", credentialName, err)
+		log.Print("Trying to use unauthenticated client")
+		return github.NewClient(nil), nil
+	}
+
+	if c != nil {
+		if c.APIToken != "" {
+			ts := oauth2.StaticTokenSource(
+				&oauth2.Token{AccessToken: string(c.APIToken)},
+			)
+			tc := oauth2.NewClient(oauth2.NoContext, ts)
+			return github.NewClient(tc), nil
+		}
+		if c.BasicAuth.Username != "" && c.BasicAuth.Password != "" {
+			tp := github.BasicAuthTransport{
+				Username: c.BasicAuth.Username,
+				Password: c.BasicAuth.Password,
+			}
+			return github.NewClient(tp.Client()), nil
+		}
+
+	}
+	return nil, fmt.Errorf("No suitable credential found for %s", credentialName)
+
 }
 
+// GetGithubRegistry returns a new GithubRegistry. If there's a credential that is specified, will try
+// to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client.
 func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
 	if cr.Type == common.GithubRegistryType {
+		// If there's a credential that we need to use, fetch it and create a client for it.
+		client, err := grp.createGithubClient(cr.CredentialName)
+		if err != nil {
+			return nil, err
+		}
+
 		fMap := ParseRegistryFormat(cr.Format)
 		if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {
-			return NewGithubPackageRegistry(cr.Name, cr.URL, nil)
+			return NewGithubPackageRegistry(cr.Name, cr.URL, nil, client)
 		}
 
 		if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] {
-			return NewGithubTemplateRegistry(cr.Name, cr.URL, nil)
+			return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, client)
 		}
 
 		return nil, fmt.Errorf("unknown registry format: %s", cr.Format)

registry/registryprovider_test.go

@@ -45,7 +45,7 @@ func TestShortGithubUrlTemplateMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
 }
 
 func TestShortGithubUrlPackageMapping(t *testing.T) {
@@ -60,5 +60,5 @@ func TestShortGithubUrlPackageMapping(t *testing.T) {
 	}
 
 	grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps)
-	testUrlConversionDriver(NewRegistryProvider(nil, grp), tests, t)
+	testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t)
 }

registry/test/test_credentials_file.yaml

@@ -0,0 +1,6 @@
+- name: test1
+  apitoken: token
+- name: test2
+  basicauth:
+    username: user
+    password: password