From 2c59a07bfdd52b4dbbbb3f5d95b3340d46ab9c23 Mon Sep 17 00:00:00 2001 From: jackgr Date: Tue, 12 Jan 2016 15:44:56 -0800 Subject: [PATCH] Regex search in dm and Github credential setting to facilitate testing. --- common/types.go | 16 +++++- dm/dm.go | 64 +++++++++++++++++++++-- manager/manager/manager.go | 11 ++++ registry/inmem_registry_service.go | 81 ++++++++++++++++++++++++++---- registry/registryprovider.go | 39 ++++++++------ 5 files changed, 180 insertions(+), 31 deletions(-) diff --git a/common/types.go b/common/types.go index fc6a3a909..f2a1d62f2 100644 --- a/common/types.go +++ b/common/types.go @@ -173,8 +173,11 @@ type BasicAuthCredential struct { Password string `json:"password"` } -// Credentials used to access the repository +type APITokenCredential string + +// Credential used to access the repository type RegistryCredential struct { + APIToken APITokenCredential `json:"apitoken,omitempty"` BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` } @@ -187,7 +190,7 @@ type Registry struct { Format RegistryFormat `json:"format,omitempty"` // Format of the registry } -// AuthenticatedRegistry describes a type registry with credentials. +// AuthenticatedRegistry describes a type registry with credential. // Broke this out of Registry, so that we can pass around instances of Registry // without worrying about secrets. type AuthenticatedRegistry struct { @@ -229,8 +232,17 @@ type RegistryService interface { Create(registry *Registry) error // Get a registry Get(name string) (*Registry, error) + // Get a registry with credential. + GetAuthenticatedRegistry(name string) (*AuthenticatedRegistry, error) // Delete a registry Delete(name string) error // Find a registry that backs the given URL GetByURL(URL string) (*Registry, error) + // GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL. + GetAuthenticatedRegistryByURL(URL string) (*AuthenticatedRegistry, error) + // Set the credential for a registry. + // May not be supported by some registry services. + SetCredential(name string, credential RegistryCredential) error + // Get the credential for a registry. + GetCredential(name string) (RegistryCredential, error) } diff --git a/dm/dm.go b/dm/dm.go index 677e8e9bc..5f0757137 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -22,6 +22,7 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/registry" + "github.com/kubernetes/deployment-manager/util" "archive/tar" "bytes" @@ -35,6 +36,7 @@ import ( "net/url" "os" "path" + "regexp" "strconv" "strings" "time" @@ -48,6 +50,8 @@ var ( service = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager") binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") timeout = flag.Int("timeout", 10, "Time in seconds to wait for response") + regex_string = flag.String("regex", "", "Regular expression to filter the templates listed in a template registry") + apitoken = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable") ) var commands = []string{ @@ -81,9 +85,52 @@ var usage = func() { panic("\n") } -var provider = registry.NewDefaultRegistryProvider() +// TODO(jackgr): Move all registry related operations to the server side. +var registryProvider registry.RegistryProvider + +func getRegistryProvider() registry.RegistryProvider { + if registryProvider == nil { + rs := registry.NewInmemRegistryService() + r, err := rs.GetByURL(*template_registry) + if err != nil { + r := newRegistry(*template_registry) + if err := rs.Create(r); err != nil { + panic(fmt.Errorf("cannot configure registry at %s: %s", r.URL, err)) + } + } + + if *apitoken == "" { + *apitoken = os.Getenv("DM_GITHUB_API_TOKEN") + } + + if *apitoken != "" { + credential := common.RegistryCredential{ + APIToken: common.APITokenCredential(*apitoken), + } + + if err := rs.SetCredential(r.Name, credential); err != nil { + panic(fmt.Errorf("cannot configure registry at %s: %s", r.Name, err)) + } + } + + registryProvider = registry.NewRegistryProvider(rs, nil) + } + + return registryProvider +} + +func newRegistry(URL string) *common.Registry { + tFormat := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) + return &common.Registry{ + Name: util.TrimURLScheme(URL), + Type: common.GithubRegistryType, + URL: URL, + Format: common.RegistryFormat(tFormat), + } +} func getGithubRegistry() registry.Registry { + provider := getRegistryProvider() git, err := provider.GetRegistryByShortURL(*template_registry) if err != nil { panic(fmt.Errorf("cannot open registry %s: %s", *template_registry, err)) @@ -114,13 +161,21 @@ func execute() { switch args[0] { case "templates": + var regex *regexp.Regexp + if *regex_string != "" { + var err error + regex, err = regexp.Compile(*regex_string) + if err != nil { + panic(fmt.Errorf("cannot compile regular expression %s: %s", *regex_string, err)) + } + } + git := getGithubRegistry() - types, err := git.ListTypes(nil) + types, err := git.ListTypes(regex) if err != nil { - panic(fmt.Errorf("cannot list types in registry %s: %s", *template_registry, err)) + panic(fmt.Errorf("cannot list templates in registry %s: %s", *template_registry, err)) } - fmt.Printf("Templates:\n") for _, t := range types { fmt.Printf("%s\n", t.String()) urls, err := git.GetDownloadURLs(t) @@ -278,6 +333,7 @@ func describeType(args []string) { // getDownloadURLs returns URLs or empty list if a primitive type. func getDownloadURLs(tName string) []string { qName := path.Join(*template_registry, tName) + provider := getRegistryProvider() result, err := registry.GetDownloadURLs(provider, qName) if err != nil { panic(fmt.Errorf("cannot get URLs for %s: %s\n", tName, err)) diff --git a/manager/manager/manager.go b/manager/manager/manager.go index 6ea47c19e..2e82c378c 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -343,6 +343,17 @@ func (m *manager) DeleteRegistry(name string) error { return m.service.Delete(name) } +// Set the credential for a registry. +// May not be supported by some registry services. +func (m *manager) SetCredential(name string, credential common.RegistryCredential) error { + return m.service.SetCredential(name, credential) +} + +// Get the credential for a registry. +func (m *manager) GetCredential(name string) (common.RegistryCredential, error) { + return m.service.GetCredential(name) +} + func generateManifestName() string { return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano()) } diff --git a/registry/inmem_registry_service.go b/registry/inmem_registry_service.go index 8affeea14..63f26fefc 100644 --- a/registry/inmem_registry_service.go +++ b/registry/inmem_registry_service.go @@ -17,19 +17,20 @@ limitations under the License. package registry import ( + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" + "fmt" "strings" - - "github.com/kubernetes/deployment-manager/common" ) type inmemRegistryService struct { - registries map[string]*common.Registry + registries map[string]*common.AuthenticatedRegistry } func NewInmemRegistryService() common.RegistryService { rs := &inmemRegistryService{ - registries: make(map[string]*common.Registry), + registries: make(map[string]*common.AuthenticatedRegistry), } pFormat := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) @@ -47,36 +48,98 @@ func NewInmemRegistryService() common.RegistryService { URL: "github.com/kubernetes/application-dm-templates", Format: common.RegistryFormat(tFormat), }) + return rs } +// List returns the list of known registries. func (rs *inmemRegistryService) List() ([]*common.Registry, error) { ret := []*common.Registry{} for _, r := range rs.registries { - ret = append(ret, r) + ret = append(ret, &r.Registry) } + return ret, nil } +// Create creates an authenticated registry. func (rs *inmemRegistryService) Create(registry *common.Registry) error { - rs.registries[registry.URL] = registry + rs.registries[registry.Name] = &common.AuthenticatedRegistry{Registry: *registry} return nil } +// Get returns a registry with a given name. func (rs *inmemRegistryService) Get(name string) (*common.Registry, error) { - return &common.Registry{}, nil + r, ok := rs.registries[name] + if !ok { + return nil, fmt.Errorf("Failed to find registry named %s", name) + } + + return &r.Registry, nil } +// GetAuthenticatedRegistry returns an authenticated registry with a given name. +func (rs *inmemRegistryService) GetAuthenticatedRegistry(name string) (*common.AuthenticatedRegistry, error) { + r, ok := rs.registries[name] + if !ok { + return nil, fmt.Errorf("Failed to find registry named %s", name) + } + + return r, nil +} + +// Create deletes the authenticated registry with a given name. func (rs *inmemRegistryService) Delete(name string) error { + _, ok := rs.registries[name] + if !ok { + return fmt.Errorf("Failed to find registry named %s", name) + } + + delete(rs.registries, name) return nil } // GetByURL returns a registry that handles the types for a given URL. func (rs *inmemRegistryService) GetByURL(URL string) (*common.Registry, error) { + trimmed := util.TrimURLScheme(URL) + for _, r := range rs.registries { + if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) { + return &r.Registry, nil + } + } + + return nil, fmt.Errorf("Failed to find registry for url: %s", URL) +} + +// GetAuthenticatedRegistryByURL returns an authenticated registry that handles the types for a given URL. +func (rs *inmemRegistryService) GetAuthenticatedRegistryByURL(URL string) (*common.AuthenticatedRegistry, error) { + trimmed := util.TrimURLScheme(URL) for _, r := range rs.registries { - if strings.HasPrefix(URL, r.URL) { + if strings.HasPrefix(trimmed, util.TrimURLScheme(r.URL)) { return r, nil } } - return nil, fmt.Errorf("Failed to find registry for github url: %s", URL) + + return nil, fmt.Errorf("Failed to find registry for url: %s", URL) +} + +// Set the credential for a registry. +func (rs *inmemRegistryService) SetCredential(name string, credential common.RegistryCredential) error { + r, ok := rs.registries[name] + if !ok { + return fmt.Errorf("Failed to find registry named %s", name) + } + + r.Credential = credential + return nil +} + +// Get the credential for a registry. +func (rs *inmemRegistryService) GetCredential(name string) (common.RegistryCredential, error) { + r, ok := rs.registries[name] + if !ok { + return common.RegistryCredential{}, fmt.Errorf("Failed to find registry named %s", name) + } + + return r.Credential, nil } diff --git a/registry/registryprovider.go b/registry/registryprovider.go index b25102e84..42a6e4e35 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -33,13 +33,15 @@ type RegistryProvider interface { GetRegistryByName(registryName string) (Registry, error) } -// GithubRegistryProvider is a factory for GithubRegistry instances. -type GithubRegistryProvider interface { - GetGithubRegistry(cr common.Registry) (GithubRegistry, error) +type registryProvider struct { + sync.RWMutex + rs common.RegistryService + grp GithubRegistryProvider + registries map[string]Registry } func NewDefaultRegistryProvider() RegistryProvider { - return NewRegistryProvider(nil, nil) + return NewRegistryProvider(nil, NewGithubRegistryProvider()) } func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) RegistryProvider { @@ -47,23 +49,15 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider) rs = NewInmemRegistryService() } - registries := make(map[string]Registry) - rp := ®istryProvider{rs: rs, registries: registries} if grp == nil { - grp = rp + grp = NewGithubRegistryProvider() } - rp.grp = grp + registries := make(map[string]Registry) + rp := ®istryProvider{rs: rs, grp: grp, registries: registries} return rp } -type registryProvider struct { - sync.RWMutex - rs common.RegistryService - grp GithubRegistryProvider - registries map[string]Registry -} - func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { rp.RLock() defer rp.RUnlock() @@ -133,7 +127,20 @@ func ParseRegistryFormat(rf common.RegistryFormat) map[common.RegistryFormat]boo return result } -func (rp registryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { +// GithubRegistryProvider is a factory for GithubRegistry instances. +type GithubRegistryProvider interface { + GetGithubRegistry(cr common.Registry) (GithubRegistry, error) +} + +type githubRegistryProvider struct { +} + +// NewGithubRegistryProvider creates a GithubRegistryProvider. +func NewGithubRegistryProvider() GithubRegistryProvider { + return &githubRegistryProvider{} +} + +func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { if cr.Type == common.GithubRegistryType { fMap := ParseRegistryFormat(cr.Format) if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] {