Regex search in dm and Github credential setting to facilitate testing.

common/types.go

@@ -173,8 +173,11 @@ type BasicAuthCredential struct {
 	Password string `json:"password"`
 }
 
-// Credentials used to access the repository
+type APITokenCredential string
+
+// Credential used to access the repository
 type RegistryCredential struct {
+	APIToken  APITokenCredential  `json:"apitoken,omitempty"`
 	BasicAuth BasicAuthCredential `json:"basicauth,omitempty"`
 }
 
@@ -187,7 +190,7 @@ type Registry struct {
 	Format RegistryFormat `json:"format,omitempty"` // Format of the registry
 }
 
-// AuthenticatedRegistry describes a type registry with credentials.
+// AuthenticatedRegistry describes a type registry with credential.
 // Broke this out of Registry, so that we can pass around instances of Registry
 // without worrying about secrets.
 type AuthenticatedRegistry struct {
@@ -229,8 +232,17 @@ type RegistryService interface {
 	Create(registry *Registry) error
 	// Get a registry
 	Get(name string) (*Registry, error)
+	// Get a registry with credential.
+	GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error)
 	// Delete a registry
 	Delete(name string) error
 	// Find a registry that backs the given URL
 	GetByURL(URL string) (*Registry, error)
+	// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+	GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error)
+	// Set the credential for a registry.
+	// May not be supported by some registry services.
+	SetCredential(name string, credential RegistryCredential) error
+	// Get the credential for a registry.
+	GetCredential(name string) (RegistryCredential, error)
 }

dm/dm.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kubernetes/deployment-manager/common"
 	"github.com/kubernetes/deployment-manager/expandybird/expander"
 	"github.com/kubernetes/deployment-manager/registry"
+	"github.com/kubernetes/deployment-manager/util"
 
 	"archive/tar"
 	"bytes"
@@ -35,6 +36,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -48,6 +50,8 @@ var (
 	service           = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
 	binary            = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
 	timeout           = flag.Int("timeout", 10, "Time in seconds to wait for response")
+	regex_string      = flag.String("regex", "", "Regular expression to filter the templates listed in a template registry")
+	apitoken          = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable")
 )
 
 var commands = []string{
@@ -81,9 +85,52 @@ var usage = func() {
 	panic("\n")
 }
 
-var provider = registry.NewDefaultRegistryProvider()
+// TODO(jackgr): Move all registry related operations to the server side.
+var registryProvider registry.RegistryProvider
+
+func getRegistryProvider() registry.RegistryProvider {
+	if registryProvider == nil {
+		rs := registry.NewInmemRegistryService()
+		r, err := rs.GetByURL(*template_registry)
+		if err != nil {
+			r := newRegistry(*template_registry)
+			if err := rs.Create(r); err != nil {
+				panic(fmt.Errorf("cannot configure registry at %s: %s", r.URL, err))
+			}
+		}
+
+		if *apitoken == "" {
+			*apitoken = os.Getenv("DM_GITHUB_API_TOKEN")
+		}
+
+		if *apitoken != "" {
+			credential := common.RegistryCredential{
+				APIToken: common.APITokenCredential(*apitoken),
+			}
+
+			if err := rs.SetCredential(r.Name, credential); err != nil {
+				panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err))
+			}
+		}
+
+		registryProvider = registry.NewRegistryProvider(rs, nil)
+	}
+
+	return registryProvider
+}
+
+func newRegistry(URL string) *common.Registry {
+	tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry)
+	return &common.Registry{
+		Name:   util.TrimURLScheme(URL),
+		Type:   common.GithubRegistryType,
+		URL:    URL,
+		Format: common.RegistryFormat(tFormat),
+	}
+}
 
 func getGithubRegistry() registry.Registry {
+	provider := getRegistryProvider()
 	git, err := provider.GetRegistryByShortURL(*template_registry)
 	if err != nil {
 		panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err))
@@ -114,13 +161,21 @@ func execute() {
 
 	switch args[0] {
 	case "templates":
+		var regex *regexp.Regexp
+		if *regex_string != "" {
+			var err error
+			regex, err = regexp.Compile(*regex_string)
+			if err != nil {
+				panic(fmt.Errorf("cannot compile regular expression %s: %s", *regex_string, err))
+			}
+		}
+
 		git := getGithubRegistry()
-		types, err := git.ListTypes(nil)
+		types, err := git.ListTypes(regex)
 		if err != nil {
-			panic(fmt.Errorf("cannot list types in registry %s: %s", *template_registry, err))
+			panic(fmt.Errorf("cannot list templates in registry %s: %s", *template_registry, err))
 		}
 
-		fmt.Printf("Templates:\n")
 		for _, t := range types {
 			fmt.Printf("%s\n", t.String())
 			urls, err := git.GetDownloadURLs(t)
@@ -278,6 +333,7 @@ func describeType(args []string) {
 // getDownloadURLs returns URLs or empty list if a primitive type.
 func getDownloadURLs(tName string) []string {
 	qName := path.Join(*template_registry, tName)
+	provider := getRegistryProvider()
 	result, err := registry.GetDownloadURLs(provider, qName)
 	if err != nil {
 		panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err))

manager/manager/manager.go

@@ -343,6 +343,17 @@ func (m *manager) DeleteRegistry(name string) error {
 	return m.service.Delete(name)
 }
 
+// Set the credential for a registry.
+// May not be supported by some registry services.
+func (m *manager) SetCredential(name string, credential common.RegistryCredential) error {
+	return m.service.SetCredential(name, credential)
+}
+
+// Get the credential for a registry.
+func (m *manager) GetCredential(name string) (common.RegistryCredential, error) {
+	return m.service.GetCredential(name)
+}
+
 func generateManifestName() string {
 	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
 }

registry/inmem_registry_service.go

@@ -17,19 +17,20 @@ limitations under the License.
 package registry
 
 import (
+	"github.com/kubernetes/deployment-manager/common"
+	"github.com/kubernetes/deployment-manager/util"
+
 	"fmt"
 	"strings"
-
-	"github.com/kubernetes/deployment-manager/common"
 )
 
 type inmemRegistryService struct {
-	registries map[string]*common.Registry
+	registries map[string]*common.AuthenticatedRegistry
 }
 
 func NewInmemRegistryService() common.RegistryService {
 	rs := &inmemRegistryService{
-		registries: make(map[string]*common.Registry),
+		registries: make(map[string]*common.AuthenticatedRegistry),
 	}
 
 	pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry)
@@ -47,36 +48,98 @@ func NewInmemRegistryService() common.RegistryService {
 		URL:    "github.com/kubernetes/application-dm-templates",
 		Format: common.RegistryFormat(tFormat),
 	})
+
 	return rs
 }
 
+// List returns the list of known registries.
 func (rs *inmemRegistryService) List() ([]*common.Registry, error) {
 	ret := []*common.Registry{}
 	for _, r := range rs.registries {
-		ret = append(ret, r)
+		ret = append(ret, &r.Registry)
 	}
+
 	return ret, nil
 }
 
+// Create creates an authenticated registry.
 func (rs *inmemRegistryService) Create(registry *common.Registry) error {
-	rs.registries[registry.URL] = registry
+	rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry}
 	return nil
 }
 
+// Get returns a registry with a given name.
 func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) {
-	return &common.Registry{}, nil
+	r, ok := rs.registries[name]
+	if !ok {
+		return nil, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return &r.Registry, nil
 }
 
+// GetAuthenticatedRegistry returns an authenticated registry with a given name.
+func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) {
+	r, ok := rs.registries[name]
+	if !ok {
+		return nil, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return r, nil
+}
+
+// Create deletes the authenticated registry with a given name.
 func (rs *inmemRegistryService) Delete(name string) error {
+	_, ok := rs.registries[name]
+	if !ok {
+		return fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	delete(rs.registries, name)
 	return nil
 }
 
 // GetByURL returns a registry that handles the types for a given URL.
 func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) {
+	trimmed := util.TrimURLScheme(URL)
+	for _, r := range rs.registries {
+		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
+			return &r.Registry, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
+}
+
+// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL.
+func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) {
+	trimmed := util.TrimURLScheme(URL)
 	for _, r := range rs.registries {
-		if strings.HasPrefix(URL, r.URL) {
+		if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) {
 			return r, nil
 		}
 	}
-	return nil, fmt.Errorf("Failed to find registry for github url: %s", URL)
+
+	return nil, fmt.Errorf("Failed to find registry for url: %s", URL)
+}
+
+// Set the credential for a registry.
+func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error {
+	r, ok := rs.registries[name]
+	if !ok {
+		return fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	r.Credential = credential
+	return nil
+}
+
+// Get the credential for a registry.
+func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) {
+	r, ok := rs.registries[name]
+	if !ok {
+		return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name)
+	}
+
+	return r.Credential, nil
 }

registry/registryprovider.go

@@ -33,13 +33,15 @@ type RegistryProvider interface {
 	GetRegistryByName(registryName string) (Registry, error)
 }
 
-// GithubRegistryProvider is a factory for GithubRegistry instances.
-type GithubRegistryProvider interface {
-	GetGithubRegistry(cr common.Registry) (GithubRegistry, error)
+type registryProvider struct {
+	sync.RWMutex
+	rs         common.RegistryService
+	grp        GithubRegistryProvider
+	registries map[string]Registry
 }
 
 func NewDefaultRegistryProvider() RegistryProvider {
-	return NewRegistryProvider(nil, nil)
+	return NewRegistryProvider(nil, NewGithubRegistryProvider())
 }
 
 func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider {
@@ -47,23 +49,15 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider)
 		rs = NewInmemRegistryService()
 	}
 
-	registries := make(map[string]Registry)
-	rp := &registryProvider{rs: rs, registries: registries}
 	if grp == nil {
-		grp = rp
+		grp = NewGithubRegistryProvider()
 	}
 
-	rp.grp = grp
+	registries := make(map[string]Registry)
+	rp := &registryProvider{rs: rs, grp: grp, registries: registries}
 	return rp
 }
 
-type registryProvider struct {
-	sync.RWMutex
-	rs         common.RegistryService
-	grp        GithubRegistryProvider
-	registries map[string]Registry
-}
-
 func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) {
 	rp.RLock()
 	defer rp.RUnlock()
@@ -133,7 +127,20 @@ func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]boo
 	return result
 }
 
-func (rp registryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
+// GithubRegistryProvider is a factory for GithubRegistry instances.
+type GithubRegistryProvider interface {
+	GetGithubRegistry(cr common.Registry) (GithubRegistry, error)
+}
+
+type githubRegistryProvider struct {
+}
+
+// NewGithubRegistryProvider creates a GithubRegistryProvider.
+func NewGithubRegistryProvider() GithubRegistryProvider {
+	return &githubRegistryProvider{}
+}
+
+func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) {
 	if cr.Type == common.GithubRegistryType {
 		fMap := ParseRegistryFormat(cr.Format)
 		if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {