From 8465cb1a1b67c5c409d4b5426aaf94bd7162e191 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 2 Feb 2016 11:21:15 -0800 Subject: [PATCH] Add GCS as a registry provider. Add a way to get a file using credentials from a registry to support private repos. Add ability to create / update a registry through manager --- common/types.go | 7 +- dm/dm.go | 49 ++++++- manager/deployments.go | 73 +++++++++- manager/manager/manager.go | 17 +++ manager/manager/manager_test.go | 2 +- manager/manager/typeresolver.go | 50 ++++--- manager/manager/typeresolver_test.go | 18 ++- registry/gcs_registry.go | 157 ++++++++++++++++++++ registry/github_package_registry.go | 9 +- registry/github_registry.go | 5 +- registry/github_template_registry.go | 9 +- registry/inmem_credential_provider.go | 2 +- registry/inmem_registry_service.go | 8 + registry/registry.go | 11 ++ registry/registryprovider.go | 186 ++++++++++++++++++------ registry/registryprovider_test.go | 9 +- registry/secrets_credential_provider.go | 4 +- registry/testhelper.go | 41 +++++- util/httpclient.go | 6 +- util/kubernetes_kubectl.go | 14 +- 20 files changed, 572 insertions(+), 105 deletions(-) create mode 100644 registry/gcs_registry.go diff --git a/common/types.go b/common/types.go index b0495592f..90f1a43a0 100644 --- a/common/types.go +++ b/common/types.go @@ -182,11 +182,13 @@ type BasicAuthCredential struct { } type APITokenCredential string +type JWTTokenCredential string // Credential used to access the repository type RegistryCredential struct { - APIToken APITokenCredential `json:"apitoken,omitempty"` - BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` + APIToken APITokenCredential `json:"apitoken,omitempty"` + BasicAuth BasicAuthCredential `json:"basicauth,omitempty"` + ServiceAccount JWTTokenCredential `json:"serviceaccount,omitempty"` } // Registry describes a template registry @@ -204,6 +206,7 @@ type RegistryType string const ( GithubRegistryType RegistryType = "github" + GCSRegistryType RegistryType = "gcs" ) // RegistryFormat is a semi-colon delimited string that describes the format diff --git a/dm/dm.go b/dm/dm.go index ebed53b6f..099e108a2 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -50,6 +50,8 @@ var ( username = flag.String("username", "", "Github user name that overrides GITHUB_USERNAME environment variable") password = flag.String("password", "", "Github password that overrides GITHUB_PASSWORD environment variable") apitoken = flag.String("apitoken", "", "Github api token that overrides GITHUB_API_TOKEN environment variable") + serviceaccount = flag.String("serviceaccount", "", "Service account file containing JWT token") + registryfile = flag.String("registryfile", "", "File containing registry specification") ) var commands = []string{ @@ -67,6 +69,7 @@ var commands = []string{ "describe \t\t Describes the named template in a given template registry", "getcredential \t\t Gets the named credential used by a registry", "setcredential \t\t Sets a credential used by a registry", + "createregistry \t\t Creates a registry that holds charts", } var usage = func() { @@ -86,7 +89,7 @@ var usage = func() { os.Exit(0) } -func getGithubCredential() *common.RegistryCredential { +func getCredential() *common.RegistryCredential { *apitoken = strings.TrimSpace(*apitoken) if *apitoken == "" { *apitoken = strings.TrimSpace(os.Getenv("GITHUB_API_TOKEN")) @@ -117,6 +120,15 @@ func getGithubCredential() *common.RegistryCredential { } } + if *serviceaccount != "" { + b, err := ioutil.ReadFile(*serviceaccount) + if err != nil { + log.Fatalf("Unable to read service account file: %v", err) + } + return &common.RegistryCredential{ + ServiceAccount: common.JWTTokenCredential(string(b)), + } + } return nil } @@ -124,6 +136,13 @@ func init() { flag.Usage = usage } +func getRegistry() ([]byte, error) { + if *registryfile == "" { + log.Fatalf("No registryfile specified (-registryfile)") + } + return ioutil.ReadFile(*registryfile) +} + func main() { defer func() { result := recover() @@ -145,6 +164,10 @@ func execute() { switch args[0] { case "templates": + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "No registry name supplied") + usage() + } path := fmt.Sprintf("registries/%s/types", args[1]) callService(path, "GET", "list templates", nil) case "describe": @@ -162,7 +185,7 @@ func execute() { path := fmt.Sprintf("credentials/%s", args[1]) callService(path, "GET", "get credential", nil) case "setcredential": - c := getGithubCredential() + c := getCredential() if c == nil { panic(fmt.Errorf("Failed to create a credential from flags/arguments")) } @@ -172,7 +195,14 @@ func execute() { } path := fmt.Sprintf("credentials/%s", args[1]) - callService(path, "POST", "get credential", ioutil.NopCloser(bytes.NewReader(y))) + callService(path, "POST", "set credential", ioutil.NopCloser(bytes.NewReader(y))) + case "createregistry": + reg, err := getRegistry() + if err != nil { + panic(fmt.Errorf("Failed to create a registry from arguments: %#v", err)) + } + path := fmt.Sprintf("registries/%s", args[1]) + callService(path, "POST", "set registry", ioutil.NopCloser(bytes.NewReader(reg))) case "get": if len(args) < 2 { fmt.Fprintln(os.Stderr, "No deployment name supplied") @@ -300,13 +330,20 @@ func describeType(args []string) { os.Exit(1) } - tUrls := getDownloadURLs(args[1]) + tUrls := getDownloadURLs(url.QueryEscape(args[1])) if len(tUrls) == 0 { panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \":\": %s", args[1])) } - schemaUrl := tUrls[0] + ".schema" - fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil)) + if !strings.Contains(tUrls[0], ".prov") { + // It's not a chart, so grab the schema + path := fmt.Sprintf("registries/%s/download?file=%s.schema", *template_registry, url.QueryEscape(tUrls[0])) + callService(path, "GET", "get schema for type ("+tUrls[0]+")", nil) + } else { + // It's a chart, so grab the provenance file + path := fmt.Sprintf("registries/%s/download?file=%s", *template_registry, url.QueryEscape(tUrls[0])) + callService(path, "GET", "get file", nil) + } } // getDownloadURLs returns URLs for a type in the given registry diff --git a/manager/deployments.go b/manager/deployments.go index ff6accb6c..a6c26c72c 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -54,8 +54,10 @@ var deployments = []Route{ {"ListTypeInstances", "/types/{type}/instances", "GET", listTypeInstancesHandlerFunc, ""}, {"ListRegistries", "/registries", "GET", listRegistriesHandlerFunc, ""}, {"GetRegistry", "/registries/{registry}", "GET", getRegistryHandlerFunc, ""}, + {"CreateRegistry", "/registries/{registry}", "POST", createRegistryHandlerFunc, "JSON"}, {"ListRegistryTypes", "/registries/{registry}/types", "GET", listRegistryTypesHandlerFunc, ""}, {"GetDownloadURLs", "/registries/{registry}/types/{type}", "GET", getDownloadURLsHandlerFunc, ""}, + {"GetFile", "/registries/{registry}/download", "GET", getFileHandlerFunc, ""}, {"CreateCredential", "/credentials/{credential}", "POST", createCredentialHandlerFunc, "JSON"}, {"GetCredential", "/credentials/{credential}", "GET", getCredentialHandlerFunc, ""}, } @@ -97,12 +99,12 @@ func init() { } func newManager(cp common.CredentialProvider) manager.Manager { - registryProvider := registry.NewDefaultRegistryProvider(cp) - resolver := manager.NewTypeResolver(registryProvider) + service := registry.NewInmemRegistryService() + registryProvider := registry.NewDefaultRegistryProvider(cp, service) + resolver := manager.NewTypeResolver(registryProvider, util.DefaultHTTPClient()) expander := manager.NewExpander(getServiceURL(*expanderURL, *expanderName), resolver) deployer := manager.NewDeployer(getServiceURL(*deployerURL, *deployerName)) r := repository.NewMapBasedRepository() - service := registry.NewInmemRegistryService() credentialProvider := cp return manager.NewManager(expander, deployer, r, registryProvider, service, credentialProvider) } @@ -378,6 +380,49 @@ func getRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, cr, http.StatusOK) } +func getRegistry(w http.ResponseWriter, r *http.Request, handler string) *common.Registry { + util.LogHandlerEntry(handler, r) + j, err := getJsonFromRequest(w, r, handler) + if err != nil { + return nil + } + + t := &common.Registry{} + if err := json.Unmarshal(j, t); err != nil { + e := fmt.Errorf("%v\n%v", err, string(j)) + util.LogAndReturnError(handler, http.StatusBadRequest, e, w) + return nil + } + + return t +} + +func createRegistryHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: create registry" + util.LogHandlerEntry(handler, r) + defer r.Body.Close() + registryName, err := getPathVariable(w, r, "registry", handler) + if err != nil { + return + } + + reg := getRegistry(w, r, handler) + if reg.Name != registryName { + e := fmt.Errorf("Registry name does not match %s != %s", reg.Name, registryName) + util.LogAndReturnError(handler, http.StatusBadRequest, e, w) + return + } + if reg != nil { + err = backend.CreateRegistry(reg) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + } + + util.LogHandlerExitWithJSON(handler, w, reg, http.StatusOK) +} + func listRegistryTypesHandlerFunc(w http.ResponseWriter, r *http.Request) { handler := "manager: list registry types" util.LogHandlerEntry(handler, r) @@ -437,6 +482,28 @@ func getDownloadURLsHandlerFunc(w http.ResponseWriter, r *http.Request) { util.LogHandlerExitWithJSON(handler, w, urls, http.StatusOK) } +func getFileHandlerFunc(w http.ResponseWriter, r *http.Request) { + handler := "manager: get file" + util.LogHandlerEntry(handler, r) + registryName, err := getPathVariable(w, r, "registry", handler) + if err != nil { + return + } + + file := r.FormValue("file") + if file == "" { + return + } + + b, err := backend.GetFile(registryName, file) + if err != nil { + util.LogAndReturnError(handler, http.StatusBadRequest, err, w) + return + } + + util.LogHandlerExitWithJSON(handler, w, b, http.StatusOK) +} + func getCredential(w http.ResponseWriter, r *http.Request, handler string) *common.RegistryCredential { util.LogHandlerEntry(handler, r) j, err := getJsonFromRequest(w, r, handler) diff --git a/manager/manager/manager.go b/manager/manager/manager.go index c5116499f..da1b448c0 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -26,6 +26,7 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/manager/repository" "github.com/kubernetes/deployment-manager/registry" + "github.com/kubernetes/deployment-manager/util" ) // Manager manages a persistent set of Deployments. @@ -55,6 +56,7 @@ type Manager interface { // Registry Types ListRegistryTypes(registryName string, regex *regexp.Regexp) ([]registry.Type, error) GetDownloadURLs(registryName string, t registry.Type) ([]*url.URL, error) + GetFile(registryName string, url string) (string, error) // Credentials CreateCredential(name string, c *common.RegistryCredential) error @@ -394,6 +396,21 @@ func (m *manager) GetDownloadURLs(registryName string, t registry.Type) ([]*url. return r.GetDownloadURLs(t) } +// GetFile returns a file from the backing registry +func (m *manager) GetFile(registryName string, url string) (string, error) { + r, err := m.registryProvider.GetRegistryByName(registryName) + if err != nil { + return "", err + } + + getter := util.NewHTTPClient(3, r, util.NewSleeper()) + body, _, err := getter.Get(url) + if err != nil { + return "", err + } + return body, nil +} + // CreateCredential creates a credential that can be used to authenticate to registry func (m *manager) CreateCredential(name string, c *common.RegistryCredential) error { return m.credentialProvider.SetCredential(name, c) diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index 403af4689..2c4d0ca15 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -255,7 +255,7 @@ var testRepository = newRepositoryStub() var testDeployer = newDeployerStub() var testRegistryService = registry.NewInmemRegistryService() var testCredentialProvider = registry.NewInmemCredentialProvider() -var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), testCredentialProvider) +var testProvider = registry.NewRegistryProvider(nil, registry.NewTestGithubRegistryProvider("", nil), registry.NewTestGCSRegistryProvider("", nil), testCredentialProvider) var testManager = NewManager(testExpander, testDeployer, testRepository, testProvider, testRegistryService, testCredentialProvider) func TestListDeployments(t *testing.T) { diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 5869558c3..d7a2a8155 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -19,7 +19,6 @@ package manager import ( "fmt" "net/http" - "time" "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/registry" @@ -40,26 +39,23 @@ type TypeResolver interface { } type typeResolver struct { - getter util.HTTPClient maxUrls int rp registry.RegistryProvider + c util.HTTPClient +} + +type fetchableURL struct { + registry registry.Registry + url string } type fetchUnit struct { - urls []string + urls []fetchableURL } // NewTypeResolver returns a new initialized TypeResolver. -func NewTypeResolver(rp registry.RegistryProvider) TypeResolver { - ret := &typeResolver{} - client := http.DefaultClient - //TODO (iantw): Make this a flag - timeout, _ := time.ParseDuration("10s") - client.Timeout = timeout - ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) - ret.maxUrls = maxURLImports - ret.rp = rp - return ret +func NewTypeResolver(rp registry.RegistryProvider, c util.HTTPClient) TypeResolver { + return &typeResolver{maxUrls: maxURLImports, rp: rp, c: c} } func resolverError(c *common.Configuration, err error) error { @@ -67,7 +63,13 @@ func resolverError(c *common.Configuration, err error) error { c, err) } -func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content string, err error) { +func (tr *typeResolver) performHTTPGet(d util.HTTPDoer, u string, allowMissing bool) (content string, err error) { + var g util.HTTPClient + if d == nil { + g = tr.c + } else { + g = util.NewHTTPClient(3, d, util.NewSleeper()) + } r, code, err := g.Get(u) if err != nil { return "", err @@ -100,7 +102,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co toFetch := make([]*fetchUnit, 0, tr.maxUrls) for _, r := range config.Resources { // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). - urls, err := registry.GetDownloadURLs(tr.rp, r.Type) + urls, registry, err := registry.GetDownloadURLs(tr.rp, r.Type) if err != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err)) } @@ -108,14 +110,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co f := &fetchUnit{} for _, u := range urls { if len(u) > 0 { - f.urls = append(f.urls, u) + f.urls = append(f.urls, fetchableURL{registry, u}) // Add to existing map so it is not fetched multiple times. existing[r.Type] = true } } if len(f.urls) > 0 { toFetch = append(toFetch, f) - fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]}) + fetched[f.urls[0].url] = append(fetched[f.urls[0].url], &common.ImportFile{Name: r.Type, Path: f.urls[0].url}) } } } @@ -138,14 +140,14 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co templates := []string{} url := toFetch[0].urls[0] for _, u := range toFetch[0].urls { - template, err := performHTTPGet(tr.getter, u, false) + template, err := tr.performHTTPGet(u.registry, u.url, false) if err != nil { return nil, resolverError(config, err) } templates = append(templates, template) } - for _, i := range fetched[url] { + for _, i := range fetched[url.url] { template, err := parseContent(templates) if err != nil { return nil, resolverError(config, err) @@ -153,8 +155,8 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co i.Content = template } - schemaURL := url + schemaSuffix - sch, err := performHTTPGet(tr.getter, schemaURL, true) + schemaURL := url.url + schemaSuffix + sch, err := tr.performHTTPGet(url.registry, schemaURL, true) if err != nil { return nil, resolverError(config, err) } @@ -168,7 +170,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co for _, v := range s.Imports { i := &common.ImportFile{Name: v.Name} var existingSchema string - urls, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path) + urls, registry, conversionErr := registry.GetDownloadURLs(tr.rp, v.Path) if conversionErr != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr)) } @@ -180,7 +182,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co for _, u := range urls { if len(fetched[u]) == 0 { // If this import URL is new to us, add it to the URLs to fetch. - toFetch = append(toFetch, &fetchUnit{[]string{u}}) + toFetch = append(toFetch, &fetchUnit{[]fetchableURL{fetchableURL{registry, u}}}) } else { // If this is not a new import URL and we've already fetched its contents, // reuse them. Also, check if we also found a schema for that import URL and @@ -201,7 +203,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co } // Add the schema we've fetched as the schema for any templates which used this URL. - for _, i := range fetched[url] { + for _, i := range fetched[url.url] { schemaImportName := i.Name + schemaSuffix fetched[schemaURL] = append(fetched[schemaURL], &common.ImportFile{Name: schemaImportName, Content: sch}) diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index a0302109a..f76362a8d 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -18,6 +18,7 @@ package manager import ( "errors" + "log" "net/http" "reflect" "strings" @@ -50,19 +51,20 @@ type testGetter struct { test *testing.T } -func (tg *testGetter) Get(url string) (body string, code int, err error) { +func (tg testGetter) Get(url string) (body string, code int, err error) { tg.count = tg.count + 1 ret := tg.responses[url] - + log.Printf("GET RETURNING: '%s' '%d'", ret.resp, tg.count) return ret.resp, ret.code, ret.err } func testDriver(c resolverTestCase, t *testing.T) { g := &testGetter{test: t, responses: c.responses} + log.Printf("getter: %#v", g) r := &typeResolver{ - getter: g, maxUrls: 5, rp: c.registryProvider, + c: g, } conf := &common.Configuration{} @@ -74,7 +76,7 @@ func testDriver(c resolverTestCase, t *testing.T) { result, err := r.ResolveTypes(conf, c.imports) if g.count != c.urlcount { - t.Errorf("Expected %d url GETs but only %d found", c.urlcount, g.count) + t.Errorf("Expected %d url GETs but only %d found %#v", c.urlcount, g.count, g) } if (err != nil && c.expectedErr == nil) || (err == nil && c.expectedErr != nil) { @@ -307,13 +309,19 @@ func TestShortGithubUrl(t *testing.T) { registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, } + gcsUrlMaps := map[registry.Type]registry.TestURLAndError{ + registry.NewTypeOrDie("common", "replicatedservice", "v1"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.NewTypeOrDie("common", "replicatedservice", "v2"): registry.TestURLAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, + } + grp := registry.NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) + gcsrp := registry.NewTestGCSRegistryProvider("gs://charts", gcsUrlMaps) test := resolverTestCase{ config: templateShortGithubTemplate, importOut: finalImports, urlcount: 4, responses: responses, - registryProvider: registry.NewRegistryProvider(nil, grp, registry.NewInmemCredentialProvider()), + registryProvider: registry.NewRegistryProvider(nil, grp, gcsrp, registry.NewInmemCredentialProvider()), } testDriver(test, t) diff --git a/registry/gcs_registry.go b/registry/gcs_registry.go new file mode 100644 index 000000000..659571444 --- /dev/null +++ b/registry/gcs_registry.go @@ -0,0 +1,157 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" + + // "golang.org/x/net/context" + // "golang.org/x/oauth2/google" + storage "google.golang.org/api/storage/v1" + + "fmt" + "log" + "net/http" + "net/url" + "regexp" +) + +// GCSRegistry implements the ObbectStorageRegistry interface and implements a +// Deployment Manager templates registry. +// +// A registry root must be a directory that contains all the available charts, +// one or two files per template. +// name-version.tgz +// name-version.prov +type GCSRegistry struct { + name string + shortURL string + bucket string + format common.RegistryFormat + credentialName string + + httpClient *http.Client + service *storage.Service +} + +// RE for GCS storage +var ChartFormatMatcher = regexp.MustCompile("(.*)-(.*).tgz") +var URLFormatMatcher = regexp.MustCompile("gs://(.*)") + +// NewGithubTemplateRegistry creates a GithubTemplateRegistry. +func NewGCSRegistry(name, shortURL string, httpClient *http.Client, gcsService *storage.Service) (*GCSRegistry, error) { + format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.OneLevelRegistry) + trimmed := util.TrimURLScheme(shortURL) + m := URLFormatMatcher.FindStringSubmatch(shortURL) + if len(m) != 2 { + return nil, fmt.Errorf("URL must be of the form gs:// was: %s", shortURL) + } + + return &GCSRegistry{ + name: name, + shortURL: trimmed, + format: common.RegistryFormat(format), + httpClient: httpClient, + service: gcsService, + bucket: m[1], + }, + nil +} + +func (g GCSRegistry) GetRegistryName() string { + return g.name +} + +func (g GCSRegistry) GetBucket() string { + return g.bucket +} + +func (g GCSRegistry) GetRegistryType() common.RegistryType { + return common.GCSRegistryType +} + +// ListTypes lists types in this registry whose string values conform to the +// supplied regular expression, or all types, if the regular expression is nil. +func (g GCSRegistry) ListTypes(regex *regexp.Regexp) ([]Type, error) { + // List all files in the bucket/prefix that contain the + types := []Type{} + + // List all objects in a bucket using pagination + pageToken := "" + for { + call := g.service.Objects.List(g.bucket) + call.Delimiter("/") + if pageToken != "" { + call = call.PageToken(pageToken) + } + res, err := call.Do() + if err != nil { + return []Type{}, err + } + for _, object := range res.Items { + // Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here + m := ChartFormatMatcher.FindStringSubmatch(object.Name) + if len(m) != 3 { + continue + } + + t, err := NewType("", m[1], m[2]) + if err != nil { + return []Type{}, fmt.Errorf("can't create a type type at path %#v", err) + } + types = append(types, t) + } + if pageToken = res.NextPageToken; pageToken == "" { + break + } + } + return types, nil +} + +func (g GCSRegistry) GetRegistryFormat() common.RegistryFormat { + return common.CollectionRegistry +} + +func (g GCSRegistry) GetRegistryShortURL() string { + return g.shortURL +} + +// GetDownloadURLs fetches the download URLs for a given Chart +func (g GCSRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { + call := g.service.Objects.List(g.bucket) + call.Delimiter("/") + call.Prefix(t.String()) + res, err := call.Do() + ret := []*url.URL{} + if err != nil { + return ret, err + } + for _, object := range res.Items { + log.Printf("Found: %s", object.Name) + u, err := url.Parse(object.MediaLink) + if err != nil { + return nil, fmt.Errorf("cannot parse URL from %s: %s", object.MediaLink, err) + } + ret = append(ret, u) + } + return ret, err +} + +func (g GCSRegistry) Do(req *http.Request) (resp *http.Response, err error) { + return g.httpClient.Do(req) +} diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go index 7f457021b..bc2ede948 100644 --- a/registry/github_package_registry.go +++ b/registry/github_package_registry.go @@ -22,6 +22,7 @@ import ( "fmt" "log" + "net/http" "net/url" "regexp" "strings" @@ -41,7 +42,7 @@ type GithubPackageRegistry struct { } // NewGithubPackageRegistry creates a GithubPackageRegistry. -func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubPackageRegistry, error) { +func NewGithubPackageRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubPackageRegistry, error) { format := fmt.Sprintf("%s;%s", common.UnversionedRegistry, common.OneLevelRegistry) if service == nil { if client == nil { @@ -51,7 +52,7 @@ func NewGithubPackageRegistry(name, shortURL string, service GithubRepositorySer } } - gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service) if err != nil { return nil, err } @@ -151,3 +152,7 @@ func (g GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) { // Construct the return path return t.Name + "/manifests", nil } + +func (g GithubPackageRegistry) Do(req *http.Request) (resp *http.Response, err error) { + return g.httpClient.Do(req) +} diff --git a/registry/github_registry.go b/registry/github_registry.go index 19d81ff5b..bb40df620 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -22,6 +22,7 @@ import ( "github.com/kubernetes/deployment-manager/util" "fmt" + "net/http" "strings" ) @@ -38,6 +39,7 @@ type githubRegistry struct { format common.RegistryFormat credentialName string service GithubRepositoryService + httpClient *http.Client } // GithubRepositoryService defines the interface that's defined in github.com/go-github/repos_contents.go GetContents method. @@ -54,7 +56,7 @@ type GithubRepositoryService interface { } // newGithubRegistry creates a githubRegistry. -func newGithubRegistry(name, shortURL string, format common.RegistryFormat, service GithubRepositoryService) (*githubRegistry, error) { +func newGithubRegistry(name, shortURL string, format common.RegistryFormat, httpClient *http.Client, service GithubRepositoryService) (*githubRegistry, error) { trimmed := util.TrimURLScheme(shortURL) owner, repository, path, err := parseGithubShortURL(trimmed) if err != nil { @@ -69,6 +71,7 @@ func newGithubRegistry(name, shortURL string, format common.RegistryFormat, serv path: path, format: format, service: service, + httpClient: httpClient, }, nil } diff --git a/registry/github_template_registry.go b/registry/github_template_registry.go index b971dabae..4a9d3be48 100644 --- a/registry/github_template_registry.go +++ b/registry/github_template_registry.go @@ -22,6 +22,7 @@ import ( "fmt" "log" + "net/http" "net/url" "regexp" "strings" @@ -54,13 +55,13 @@ type GithubTemplateRegistry struct { } // NewGithubTemplateRegistry creates a GithubTemplateRegistry. -func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, client *github.Client) (*GithubTemplateRegistry, error) { +func NewGithubTemplateRegistry(name, shortURL string, service GithubRepositoryService, httpClient *http.Client, client *github.Client) (*GithubTemplateRegistry, error) { format := fmt.Sprintf("%s;%s", common.VersionedRegistry, common.CollectionRegistry) if service == nil { service = client.Repositories } - gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), service) + gr, err := newGithubRegistry(name, shortURL, common.RegistryFormat(format), httpClient, service) if err != nil { return nil, err } @@ -212,3 +213,7 @@ func (g GithubTemplateRegistry) MakeRepositoryPath(t Type) (string, error) { } return p + t.Name + "/" + t.GetVersion(), nil } + +func (g GithubTemplateRegistry) Do(req *http.Request) (resp *http.Response, err error) { + return g.httpClient.Do(req) +} diff --git a/registry/inmem_credential_provider.go b/registry/inmem_credential_provider.go index f3d064f67..f9938aba2 100644 --- a/registry/inmem_credential_provider.go +++ b/registry/inmem_credential_provider.go @@ -38,6 +38,6 @@ func (fcp *InmemCredentialProvider) GetCredential(name string) (*common.Registry } func (fcp *InmemCredentialProvider) SetCredential(name string, credential *common.RegistryCredential) error { - fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth} + fcp.credentials[name] = &common.RegistryCredential{credential.APIToken, credential.BasicAuth, credential.ServiceAccount} return nil } diff --git a/registry/inmem_registry_service.go b/registry/inmem_registry_service.go index ed1077974..ba33e42f5 100644 --- a/registry/inmem_registry_service.go +++ b/registry/inmem_registry_service.go @@ -51,6 +51,14 @@ func NewInmemRegistryService() common.RegistryService { CredentialName: "default", }) + gFormat := fmt.Sprintf("%s", common.CollectionRegistry) + rs.Create(&common.Registry{ + Name: "charts_gcs", + Type: common.GCSRegistryType, + URL: "gs://helm-charts-test", + Format: common.RegistryFormat(gFormat), + }) + return rs } diff --git a/registry/registry.go b/registry/registry.go index fa690d019..9fcd9818b 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -18,6 +18,7 @@ package registry import ( "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" "fmt" "net/url" @@ -29,6 +30,9 @@ import ( // used in a Deployment Manager configuration. There can be multiple // registry implementations. type Registry interface { + // Also handles http.Client.Do method for authenticated File accesses + util.HTTPDoer + // GetRegistryName returns the name of this registry GetRegistryName() string // GetRegistryType returns the type of this registry. @@ -56,6 +60,13 @@ type GithubRegistry interface { GetRegistryPath() string } +// ObjectStorageRegistry abstracts a registry that resides in an Object Storage, for +// example Google Cloud Storage or AWS S3, etc. +type ObjectStorageRegistry interface { + Registry // An ObjectStorageRegistry is a Registry. + GetBucket() string +} + type Type struct { Collection string Name string diff --git a/registry/registryprovider.go b/registry/registryprovider.go index bb625839d..53cb0165d 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -21,9 +21,12 @@ import ( "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/util" "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + storage "google.golang.org/api/storage/v1" "fmt" "log" + "net/http" "net/url" "regexp" "strings" @@ -40,15 +43,16 @@ type registryProvider struct { sync.RWMutex rs common.RegistryService grp GithubRegistryProvider + gcsrp GCSRegistryProvider cp common.CredentialProvider registries map[string]Registry } -func NewDefaultRegistryProvider(cp common.CredentialProvider) RegistryProvider { - return NewRegistryProvider(nil, NewGithubRegistryProvider(cp), cp) +func NewDefaultRegistryProvider(cp common.CredentialProvider, rs common.RegistryService) RegistryProvider { + return NewRegistryProvider(rs, NewGithubRegistryProvider(cp), NewGCSRegistryProvider(cp), cp) } -func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, cp common.CredentialProvider) RegistryProvider { +func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, gcsrp GCSRegistryProvider, cp common.CredentialProvider) RegistryProvider { if rs == nil { rs = NewInmemRegistryService() } @@ -61,11 +65,27 @@ func NewRegistryProvider(rs common.RegistryService, grp GithubRegistryProvider, grp = NewGithubRegistryProvider(cp) } + if gcsrp == nil { + gcsrp = NewGCSRegistryProvider(cp) + } + registries := make(map[string]Registry) - rp := ®istryProvider{rs: rs, grp: grp, cp: cp, registries: registries} + rp := ®istryProvider{rs: rs, grp: grp, gcsrp: gcsrp, cp: cp, registries: registries} return rp } +func (rp registryProvider) getRegistry(cr common.Registry) (Registry, error) { + switch cr.Type { + case common.GithubRegistryType: + return rp.grp.GetGithubRegistry(cr) + case common.GCSRegistryType: + log.Printf("Creating a bigstore client using %#v", rp.gcsrp) + return rp.gcsrp.GetGCSRegistry(cr) + default: + return nil, fmt.Errorf("unknown registry type: %s", cr.Type) + } +} + func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { rp.RLock() defer rp.RUnlock() @@ -77,7 +97,7 @@ func (rp registryProvider) GetRegistryByShortURL(URL string) (Registry, error) { return nil, err } - r, err := rp.grp.GetGithubRegistry(*cr) + r, err := rp.getRegistry(*cr) if err != nil { return nil, err } @@ -111,7 +131,7 @@ func (rp registryProvider) GetRegistryByName(registryName string) (Registry, err return nil, err } - r, err := rp.grp.GetGithubRegistry(*cr) + r, err := rp.getRegistry(*cr) if err != nil { return nil, err } @@ -143,21 +163,21 @@ type githubRegistryProvider struct { // NewGithubRegistryProvider creates a GithubRegistryProvider. func NewGithubRegistryProvider(cp common.CredentialProvider) GithubRegistryProvider { if cp == nil { - panic(fmt.Errorf("CP IS NIL: %v", cp)) + panic(fmt.Errorf("cp is nil: %v", cp)) } return &githubRegistryProvider{cp: cp} } -func (grp githubRegistryProvider) createGithubClient(credentialName string) (*github.Client, error) { +func (grp githubRegistryProvider) createGithubClient(credentialName string) (*http.Client, *github.Client, error) { if credentialName == "" { - return github.NewClient(nil), nil + return http.DefaultClient, github.NewClient(nil), nil } c, err := grp.cp.GetCredential(credentialName) if err != nil { log.Printf("Failed to fetch credential %s: %v", credentialName, err) log.Print("Trying to use unauthenticated client") - return github.NewClient(nil), nil + return http.DefaultClient, github.NewClient(nil), nil } if c != nil { @@ -166,44 +186,90 @@ func (grp githubRegistryProvider) createGithubClient(credentialName string) (*gi &oauth2.Token{AccessToken: string(c.APIToken)}, ) tc := oauth2.NewClient(oauth2.NoContext, ts) - return github.NewClient(tc), nil + return tc, github.NewClient(tc), nil } if c.BasicAuth.Username != "" && c.BasicAuth.Password != "" { tp := github.BasicAuthTransport{ Username: c.BasicAuth.Username, Password: c.BasicAuth.Password, } - return github.NewClient(tp.Client()), nil + return tp.Client(), github.NewClient(tp.Client()), nil } } - return nil, fmt.Errorf("No suitable credential found for %s", credentialName) + return nil, nil, fmt.Errorf("No suitable credential found for %s", credentialName) } // GetGithubRegistry returns a new GithubRegistry. If there's a credential that is specified, will try // to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client. func (grp githubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { - if cr.Type == common.GithubRegistryType { - // If there's a credential that we need to use, fetch it and create a client for it. - client, err := grp.createGithubClient(cr.CredentialName) - if err != nil { - return nil, err - } + // If there's a credential that we need to use, fetch it and create a client for it. + httpClient, client, err := grp.createGithubClient(cr.CredentialName) + if err != nil { + return nil, err + } - fMap := ParseRegistryFormat(cr.Format) - if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] { - return NewGithubPackageRegistry(cr.Name, cr.URL, nil, client) - } + fMap := ParseRegistryFormat(cr.Format) + if fMap[common.UnversionedRegistry] && fMap[common.OneLevelRegistry] { + return NewGithubPackageRegistry(cr.Name, cr.URL, nil, httpClient, client) + } - if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] { - return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, client) - } + if fMap[common.VersionedRegistry] && fMap[common.CollectionRegistry] { + return NewGithubTemplateRegistry(cr.Name, cr.URL, nil, httpClient, client) + } + + return nil, fmt.Errorf("unknown registry format: %s", cr.Format) +} + +// GCSRegistryProvider is a factory for GCS Registry instances. +type GCSRegistryProvider interface { + GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) +} + +type gcsRegistryProvider struct { + cp common.CredentialProvider +} + +// NewGCSRegistryProvider creates a GCSRegistryProvider. +func NewGCSRegistryProvider(cp common.CredentialProvider) GCSRegistryProvider { + if cp == nil { + panic(fmt.Errorf("cp is nil: %v", cp)) + } + return &gcsRegistryProvider{cp: cp} +} + +// GetGCSRegistry returns a new Google Cloud Storage . If there's a credential that is specified, will try +// to fetch it and use it, and if there's no credential found, will fall back to unauthenticated client. +func (gcsrp gcsRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) { + // If there's a credential that we need to use, fetch it and create a client for it. + if cr.CredentialName == "" { + return nil, fmt.Errorf("No CredentialName specified for %s", cr.Name) + } + client, err := gcsrp.createGCSClient(cr.CredentialName) + if err != nil { + return nil, err + } + service, err := storage.New(client) + if err != nil { + log.Fatalf("Unable to create storage service: %v", err) + } + return NewGCSRegistry(cr.Name, cr.URL, client, service) +} - return nil, fmt.Errorf("unknown registry format: %s", cr.Format) +func (gcsrp gcsRegistryProvider) createGCSClient(credentialName string) (*http.Client, error) { + c, err := gcsrp.cp.GetCredential(credentialName) + if err != nil { + log.Printf("Failed to fetch credential %s: %v", credentialName, err) + return nil, fmt.Errorf("Failed to fetch Credential %s: %s", credentialName, err) } - return nil, fmt.Errorf("unknown registry type: %s", cr.Type) + config, err := google.JWTConfigFromJSON([]byte(c.ServiceAccount), storage.DevstorageReadOnlyScope) + + if err != nil { + log.Fatalf("Unable to parse client secret file to config: %v", err) + } + return config.Client(oauth2.NoContext), nil } // RE for a registry type that does support versions and has collections. @@ -212,6 +278,9 @@ var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*) // RE for a registry type that does not support versions and does not have collections. var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)") +// RE for GCS storage +var GCSRegistryMatcher = regexp.MustCompile("gs://(.*)/(.*)") + // IsGithubShortType returns whether a given type is a type description in a short format to a github repository type. // For now, this means using github types: // github.com/owner/repo/qualifier/type:version @@ -231,24 +300,31 @@ func IsGithubShortPackageType(t string) bool { return PackageRegistryMatcher.MatchString(t) } +// IsGCSShortType returns whether a given type is a type description in a short format to GCS +func IsGCSShortType(t string) bool { + return strings.HasPrefix(t, "gs://") +} + // GetDownloadURLs checks a type to see if it is either a short git hub url or a fully specified URL // and returns the URLs that should be used to fetch it. If the url is not fetchable (primitive type // for example), it returns an empty slice. -func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) { +func GetDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) { if IsGithubShortType(t) { return ShortTypeToDownloadURLs(rp, t) } else if IsGithubShortPackageType(t) { return ShortTypeToPackageDownloadURLs(rp, t) + } else if IsGCSShortType(t) { + return ShortTypeToGCSDownloadUrls(rp, t) } else if util.IsHttpUrl(t) { result, err := url.Parse(t) if err != nil { - return nil, fmt.Errorf("cannot parse download URL %s: %s", t, err) + return nil, nil, fmt.Errorf("cannot parse download URL %s: %s", t, err) } - return []string{result.String()}, nil + return []string{result.String()}, nil, nil } - return []string{}, nil + return []string{}, nil, nil } // ShortTypeToDownloadURLs converts a github URL into downloadable URL from github. @@ -256,15 +332,15 @@ func GetDownloadURLs(rp RegistryProvider, t string) ([]string, error) { // github.com/owner/repo/qualifier/type:version // for example: // github.com/kubernetes/application-dm-templates/storage/redis:v1 -func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) { +func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) { m := TemplateRegistryMatcher.FindStringSubmatch(t) if len(m) != 6 { - return nil, fmt.Errorf("cannot parse short github url: %s", t) + return nil, nil, fmt.Errorf("cannot parse short github url: %s", t) } r, err := rp.GetRegistryByShortURL(t) if err != nil { - return nil, err + return nil, nil, err } if r == nil { @@ -273,15 +349,15 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) { tt, err := NewType(m[3], m[4], m[5]) if err != nil { - return nil, err + return nil, r, err } urls, err := r.GetDownloadURLs(tt) if err != nil { - return nil, err + return nil, r, err } - return util.ConvertURLsToStrings(urls), err + return util.ConvertURLsToStrings(urls), r, err } // ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github. @@ -289,26 +365,48 @@ func ShortTypeToDownloadURLs(rp RegistryProvider, t string) ([]string, error) { // github.com/owner/repo/type // for example: // github.com/helm/charts/cassandra -func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, error) { +func ShortTypeToPackageDownloadURLs(rp RegistryProvider, t string) ([]string, Registry, error) { m := PackageRegistryMatcher.FindStringSubmatch(t) if len(m) != 4 { - return nil, fmt.Errorf("Failed to parse short github url: %s", t) + return nil, nil, fmt.Errorf("Failed to parse short github url: %s", t) } r, err := rp.GetRegistryByShortURL(t) if err != nil { - return nil, err + return nil, nil, err } tt, err := NewType("", m[3], "") if err != nil { - return nil, err + return nil, r, err } urls, err := r.GetDownloadURLs(tt) if err != nil { - return nil, err + return nil, r, err + } + + return util.ConvertURLsToStrings(urls), r, err +} + +func ShortTypeToGCSDownloadUrls(rp RegistryProvider, t string) ([]string, Registry, error) { + m := GCSRegistryMatcher.FindStringSubmatch(t) + if len(m) != 3 { + return nil, nil, fmt.Errorf("Failed to parse short gs url: %s", t) } - return util.ConvertURLsToStrings(urls), err + r, err := rp.GetRegistryByShortURL(t) + if err != nil { + return nil, nil, err + } + tt, err := NewType(m[1], m[2], "") + if err != nil { + return nil, r, err + } + + urls, err := r.GetDownloadURLs(tt) + if err != nil { + return nil, r, err + } + return util.ConvertURLsToStrings(urls), r, err } diff --git a/registry/registryprovider_test.go b/registry/registryprovider_test.go index 89bf87547..ad12af5ca 100644 --- a/registry/registryprovider_test.go +++ b/registry/registryprovider_test.go @@ -22,7 +22,8 @@ import ( func testUrlConversionDriver(rp RegistryProvider, tests map[string]TestURLAndError, t *testing.T) { for in, expected := range tests { - actual, err := GetDownloadURLs(rp, in) + // TODO(vaikas): Test to make sure it's the right registry. + actual, _, err := GetDownloadURLs(rp, in) if err != expected.Err { t.Fatalf("failed on: %s : expected error %v but got %v", in, expected.Err, err) } @@ -45,7 +46,8 @@ func TestShortGithubUrlTemplateMapping(t *testing.T) { } grp := NewTestGithubRegistryProvider("github.com/kubernetes/application-dm-templates", githubUrlMaps) - testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t) + // TODO(vaikas): XXXX FIXME Add gcsrp + testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t) } func TestShortGithubUrlPackageMapping(t *testing.T) { @@ -60,5 +62,6 @@ func TestShortGithubUrlPackageMapping(t *testing.T) { } grp := NewTestGithubRegistryProvider("github.com/helm/charts", githubUrlMaps) - testUrlConversionDriver(NewRegistryProvider(nil, grp, NewInmemCredentialProvider()), tests, t) + // TODO(vaikas): XXXX FIXME Add gcsrp + testUrlConversionDriver(NewRegistryProvider(nil, grp, nil, NewInmemCredentialProvider()), tests, t) } diff --git a/registry/secrets_credential_provider.go b/registry/secrets_credential_provider.go index 93c76c68b..ca31c8a08 100644 --- a/registry/secrets_credential_provider.go +++ b/registry/secrets_credential_provider.go @@ -120,8 +120,6 @@ func (scp *SecretsCredentialProvider) SetCredential(name string, credential *com log.Printf("yaml marshal failed for kubernetes object: %s: %v", name, err) return err } - log.Printf("Calling with: %s", string(ko)) - o, err := scp.k.Create(string(ko)) - log.Printf("Create returned: %s", o) + _, err = scp.k.Create(string(ko)) return err } diff --git a/registry/testhelper.go b/registry/testhelper.go index 7b7dfe937..b6a6938e5 100644 --- a/registry/testhelper.go +++ b/registry/testhelper.go @@ -23,6 +23,7 @@ import ( "github.com/kubernetes/deployment-manager/util" "fmt" + "net/http" "net/url" "regexp" "strings" @@ -53,7 +54,7 @@ func NewTestGithubRegistryProvider(shortURL string, responses map[Type]TestURLAn func (tgrp testGithubRegistryProvider) GetGithubRegistry(cr common.Registry) (GithubRegistry, error) { trimmed := util.TrimURLScheme(cr.URL) if strings.HasPrefix(trimmed, tgrp.shortURL) { - ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, nil) + ghr, err := newGithubRegistry(cr.Name, trimmed, cr.Format, http.DefaultClient, nil) if err != nil { panic(fmt.Errorf("cannot create a github registry: %s", err)) } @@ -80,3 +81,41 @@ func (tgr testGithubRegistry) GetDownloadURLs(t Type) ([]*url.URL, error) { return []*url.URL{URL}, result.Err } + +func (g testGithubRegistry) Do(req *http.Request) (resp *http.Response, err error) { + return nil, fmt.Errorf("Not implemented yet") +} + +type testGCSRegistryProvider struct { + shortURL string + responses map[Type]TestURLAndError +} + +type testGCSRegistry struct { + GCSRegistry + responses map[Type]TestURLAndError +} + +func NewTestGCSRegistryProvider(shortURL string, responses map[Type]TestURLAndError) GCSRegistryProvider { + return testGCSRegistryProvider{ + shortURL: util.TrimURLScheme(shortURL), + responses: responses, + } +} + +func (tgrp testGCSRegistryProvider) GetGCSRegistry(cr common.Registry) (ObjectStorageRegistry, error) { + trimmed := util.TrimURLScheme(cr.URL) + if strings.HasPrefix(trimmed, tgrp.shortURL) { + gcsr, err := NewGCSRegistry(cr.Name, trimmed, http.DefaultClient, nil) + if err != nil { + panic(fmt.Errorf("cannot create gcs registry: %s", err)) + } + + return &testGCSRegistry{ + GCSRegistry: *gcsr, + responses: tgrp.responses, + }, nil + } + + panic(fmt.Errorf("unknown registry: %v", cr)) +} diff --git a/util/httpclient.go b/util/httpclient.go index 699030a7e..e53cbb91f 100644 --- a/util/httpclient.go +++ b/util/httpclient.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -72,6 +72,10 @@ type httpClient struct { sleep Sleeper } +func DefaultHTTPClient() HTTPClient { + return NewHTTPClient(3, http.DefaultClient, NewSleeper()) +} + // NewHTTPClient returns a new HTTPClient. func NewHTTPClient(retries uint, c HTTPDoer, s Sleeper) HTTPClient { ret := httpClient{} diff --git a/util/kubernetes_kubectl.go b/util/kubernetes_kubectl.go index 08eab911f..8b74a91cd 100644 --- a/util/kubernetes_kubectl.go +++ b/util/kubernetes_kubectl.go @@ -126,15 +126,17 @@ func (k *KubernetesKubectl) execute(args []string, input string) (string, error) cmd.Stderr = combined if err := cmd.Start(); err != nil { - log.Printf("cannot start kubectl %s %#v", combined.String(), err) - return combined.String(), err + e := fmt.Errorf("cannot start kubectl %s %#v", combined.String(), err) + log.Printf("%s", e) + return combined.String(), e } if err := cmd.Wait(); err != nil { - log.Printf("kubectl failed: %s %#v", combined.String(), err) - return combined.String(), err + e := fmt.Errorf("kubectl failed %s", combined.String()) + log.Printf("%s", e) + return combined.String(), e } - log.Printf("kubectl succeeded: SysTime: %v UserTime: %v\n%v", - cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String()) + log.Printf("kubectl succeeded: SysTime: %v UserTime: %v", + cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime()) return combined.String(), nil }