From 9777e72692fa9fa3eb437a1a617ccf7b055c5ef8 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 8 Dec 2015 16:37:24 -0800 Subject: [PATCH] Add support for native kubernetes types, helm packages and helm/charts github repo --- common/types.go | 8 ++ dm/dm.go | 56 +++++---- examples/package/cassandra.yaml | 4 + expandybird/expansion/expansion.py | 2 +- manager/manager/expander.go | 2 +- manager/manager/typeresolver.go | 151 ++++++++++++++++-------- manager/manager/typeresolver_test.go | 49 +++++--- registry/github_package_registry.go | 120 ++++++++++++++++++++ registry/github_registry.go | 36 ++++-- registry/registry.go | 35 ++---- registry/registryprovider.go | 5 + util/kubernetesutil.go | 44 +++++++ util/kubernetesutil_test.go | 164 +++++++++++++++++++++++++++ util/templateutil.go | 17 ++- 14 files changed, 575 insertions(+), 118 deletions(-) create mode 100644 examples/package/cassandra.yaml create mode 100644 registry/github_package_registry.go create mode 100644 util/kubernetesutil.go create mode 100644 util/kubernetesutil_test.go diff --git a/common/types.go b/common/types.go index 63b1340f6..8cc238bf3 100644 --- a/common/types.go +++ b/common/types.go @@ -155,3 +155,11 @@ type TypeInstance struct { Manifest string `json:"manifest"` // manifest name Path string `json:"path"` // JSON path within manifest } + +// KubernetesObject represents a native 'bare' Kubernetes object. +type KubernetesObject struct { + Kind string `json:"kind"` + ApiVersion string `json:"apiVersion"` + Metadata map[string]interface{} `json:"metadata"` + Spec map[string]interface{} `json:"spec"` +} diff --git a/dm/dm.go b/dm/dm.go index 56beb3e90..5a988bbcf 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -78,7 +78,7 @@ var usage = func() { panic("\n") } -func getGitRegistry() *registry.GithubRegistry { +func getGitRegistry() registry.Registry { s := strings.Split(*template_registry, "/") if len(s) < 2 { panic(fmt.Errorf("invalid template registry: %s", *template_registry)) @@ -89,7 +89,11 @@ func getGitRegistry() *registry.GithubRegistry { path = strings.Join(s[2:], "/") } - return registry.NewGithubRegistry(s[0], s[1], path) + if s[0] == "helm" { + return registry.NewGithubPackageRegistry(s[0], s[1]) + } else { + return registry.NewGithubRegistry(s[0], s[1], path) + } } func main() { @@ -126,11 +130,17 @@ func execute() { if len(t.Collection) > 0 { typeSpec = t.Collection + "/" } - typeSpec = typeSpec + t.Name + ":" + t.Version + typeSpec = typeSpec + t.Name + if len(t.Version) > 0 { + typeSpec = typeSpec + ":" + t.Version + } + fmt.Printf("%s\n", typeSpec) - downloadURL := getDownloadUrl(t) fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec) - fmt.Printf("\tdownload URL: %s\n", downloadURL) + fmt.Printf("\tdownload URL(s):\n") + for _, downloadURL := range getDownloadURLs(t) { + fmt.Printf("\t%s\n", downloadURL) + } } case "describe": describeType(args) @@ -195,10 +205,14 @@ func execute() { usage() } - tUrl := getTypeUrl(args[1]) - if tUrl == "" { + tUrls := getTypeURLs(args[1]) + var tUrl = "" + if len(tUrls) == 0 { // Type is most likely a primitive. tUrl = args[1] + } else { + // TODO(vaikas): Support packages properly. + tUrl = tUrls[0] } path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl)) action := fmt.Sprintf("list deployed instances of type %s", tUrl) @@ -262,39 +276,39 @@ func describeType(args []string) { usage() } - tUrl := getTypeUrl(args[1]) - if tUrl == "" { + tUrls := getTypeURLs(args[1]) + if len(tUrls) == 0 { panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \":\": %s", args[1])) } - schemaUrl := tUrl + ".schema" - fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil)) + schemaUrl := tUrls[0] + ".schema" + fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil)) } -// getTypeUrl returns URL or empty if a primitive type. -func getTypeUrl(tName string) string { +// getTypeURLs returns URLs or empty list if a primitive type. +func getTypeURLs(tName string) []string { if util.IsHttpUrl(tName) { // User can pass raw URL to template. - return tName + return []string{tName} } // User can pass registry type. t := getRegistryType(tName) if t == nil { // Primitive types have no associated URL. - return "" + return []string{} } - return getDownloadUrl(*t) + return getDownloadURLs(*t) } -func getDownloadUrl(t registry.Type) string { +func getDownloadURLs(t registry.Type) []string { git := getGitRegistry() - url, err := git.GetURL(t) + urls, err := git.GetURLs(t) if err != nil { panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) } - return url + return urls } func isHttp(t string) bool { @@ -379,8 +393,6 @@ func getRegistryType(fullType string) *registry.Type { } func buildTemplateFromType(t registry.Type) *common.Template { - downloadURL := getDownloadUrl(t) - props := make(map[string]interface{}) if *properties != "" { plist := strings.Split(*properties, ",") @@ -406,7 +418,7 @@ func buildTemplateFromType(t registry.Type) *common.Template { config := common.Configuration{Resources: []*common.Resource{&common.Resource{ Name: name, - Type: downloadURL, + Type: getDownloadURLs(t)[0], Properties: props, }}} diff --git a/examples/package/cassandra.yaml b/examples/package/cassandra.yaml new file mode 100644 index 000000000..70c08b6f6 --- /dev/null +++ b/examples/package/cassandra.yaml @@ -0,0 +1,4 @@ +resources: +- name: cassandra + type: github.com/helm/charts/cassandra + properties: null diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index 96f992c68..917d616af 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -232,7 +232,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): except schema_validation.ValidationErrors as e: raise ExpansionError(resource['name'], e.message) - if path.endswith('jinja'): + if path.endswith('jinja') or path.endswith('yaml'): expanded_template = ExpandJinja( source_file, imports[source_file]['content'], resource, imports) elif path.endswith('py'): diff --git a/manager/manager/expander.go b/manager/manager/expander.go index 2d291a530..ed724f3e7 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -21,8 +21,8 @@ import ( "net/http" "github.com/ghodss/yaml" - "github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" ) const ( diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index 2667baa3d..85c0c2993 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -16,7 +16,6 @@ package manager import ( "fmt" "net/http" - "regexp" "time" "github.com/kubernetes/deployment-manager/common" @@ -31,8 +30,6 @@ const ( schemaSuffix = ".schema" ) -var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") - // TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file // or primitive, and attempts to replace them with a template from a URL. type TypeResolver interface { @@ -45,6 +42,10 @@ type typeResolver struct { rp registry.RegistryProvider } +type fetchUnit struct { + urls []string +} + // NewTypeResolver returns a new initialized TypeResolver. func NewTypeResolver() TypeResolver { ret := &typeResolver{} @@ -92,18 +93,27 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co } fetched := map[string][]*common.ImportFile{} - toFetch := make([]string, 0, tr.maxUrls) + // TODO(vaikas): Need to account for multiple URLs being fetched for a given type. + toFetch := make([]*fetchUnit, 0, tr.maxUrls) for _, r := range config.Resources { // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). - u, err := tr.MapFetchableURL(r.Type) + urls, err := tr.MapFetchableURLs(r.Type) if err != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err)) } - if len(u) > 0 && !existing[r.Type] { - toFetch = append(toFetch, u) - fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u}) - // Add to existing map so it is not fetched multiple times. - existing[r.Type] = true + if !existing[r.Type] { + f := &fetchUnit{} + for _, u := range urls { + if len(u) > 0 { + f.urls = append(f.urls, u) + // Add to existing map so it is not fetched multiple times. + existing[r.Type] = true + } + } + if len(f.urls) > 0 { + toFetch = append(toFetch, f) + fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]}) + } } } @@ -122,13 +132,21 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls)) } - url := toFetch[0] - template, err := performHTTPGet(tr.getter, url, false) - if err != nil { - return nil, resolverError(config, err) + templates := []string{} + url := toFetch[0].urls[0] + for _, u := range toFetch[0].urls { + template, err := performHTTPGet(tr.getter, u, false) + if err != nil { + return nil, resolverError(config, err) + } + templates = append(templates, template) } for _, i := range fetched[url] { + template, err := parseContent(templates) + if err != nil { + return nil, resolverError(config, err) + } i.Content = template } @@ -147,33 +165,35 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co for _, v := range s.Imports { i := &common.ImportFile{Name: v.Name} var existingSchema string - u, conversionErr := tr.MapFetchableURL(v.Path) + urls, conversionErr := tr.MapFetchableURLs(v.Path) if conversionErr != nil { return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr)) } - // If it's not a fetchable URL, we need to use the type name as is, since it is a short name - // for a schema. - if len(u) == 0 { - u = v.Path + if len(urls) == 0 { + // If it's not a fetchable URL, we need to use the type name as is, since it is a short name + // for a schema. + urls = []string{v.Path} } - if len(fetched[u]) == 0 { - // If this import URL is new to us, add it to the URLs to fetch. - toFetch = append(toFetch, u) - } else { - // If this is not a new import URL and we've already fetched its contents, - // reuse them. Also, check if we also found a schema for that import URL and - // record those contents for re-use as well. - if fetched[u][0].Content != "" { - i.Content = fetched[u][0].Content - if len(fetched[u+schemaSuffix]) > 0 { - existingSchema = fetched[u+schemaSuffix][0].Content + for _, u := range urls { + if len(fetched[u]) == 0 { + // If this import URL is new to us, add it to the URLs to fetch. + toFetch = append(toFetch, &fetchUnit{[]string{u}}) + } else { + // If this is not a new import URL and we've already fetched its contents, + // reuse them. Also, check if we also found a schema for that import URL and + // record those contents for re-use as well. + if fetched[u][0].Content != "" { + i.Content = fetched[u][0].Content + if len(fetched[u+schemaSuffix]) > 0 { + existingSchema = fetched[u+schemaSuffix][0].Content + } } } - } - fetched[u] = append(fetched[u], i) - if existingSchema != "" { - fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix], - &common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) + fetched[u] = append(fetched[u], i) + if existingSchema != "" { + fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix], + &common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) + } } } @@ -197,29 +217,70 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co return ret, nil } -// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL +// MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL // and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for // example) will return empty string. -func (tr *typeResolver) MapFetchableURL(t string) (string, error) { +func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) { if util.IsGithubShortType(t) { - return tr.ShortTypeToDownloadURL(t) + return tr.ShortTypeToDownloadURLs(t) + } else if util.IsGithubShortPackageType(t) { + return tr.ShortTypeToPackageDownloadURLs(t) } else if util.IsHttpUrl(t) { - return t, nil + return []string{t}, nil } - return "", nil + return []string{}, nil } -// ShortTypeToDownloadURL converts a github URL into downloadable URL from github. +// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github. // Input must be of the type and is assumed to have been validated before this call: // github.com/owner/repo/qualifier/type:version // for example: // github.com/kubernetes/application-dm-templates/storage/redis:v1 -func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) { - m := re.FindStringSubmatch(template) +func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) { + m := util.TemplateRegistryMatcher.FindStringSubmatch(template) if len(m) != 6 { - return "", fmt.Errorf("Failed to parse short github url: %s", template) + return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) } r := tr.rp.GetGithubRegistry(m[1], m[2]) t := registry.Type{m[3], m[4], m[5]} - return r.GetURL(t) + return r.GetURLs(t) +} + +// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github. +// Input must be of the type and is assumed to have been validated before this call: +// github.com/owner/repo/type +// for example: +// github.com/helm/charts/cassandra +func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) { + m := util.PackageRegistryMatcher.FindStringSubmatch(template) + if len(m) != 4 { + return []string{}, fmt.Errorf("Failed to parse short github url: %s", template) + } + r := tr.rp.GetGithubPackageRegistry(m[1], m[2]) + t := registry.Type{Name: m[3]} + return r.GetURLs(t) +} + +func parseContent(templates []string) (string, error) { + if len(templates) == 1 { + return templates[0], nil + } else { + // If there are multiple URLs that need to be fetched, that implies it's a package + // of raw Kubernetes objects. We need to fetch them all as a unit and create a + // template representing a package out of that below. + fakeConfig := &common.Configuration{} + for _, template := range templates { + o, err := util.ParseKubernetesObject([]byte(template)) + if err != nil { + return "", fmt.Errorf("not a kubernetes object: %+v", template) + } + // Looks like a native Kubernetes object, create a configuration out of it + fakeConfig.Resources = append(fakeConfig.Resources, o) + } + marshalled, err := yaml.Marshal(fakeConfig) + if err != nil { + return "", fmt.Errorf("Failed to marshal: %+v", fakeConfig) + } + return string(marshalled), nil + } } diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index 3e3091e19..d4f4f9a47 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -22,8 +22,8 @@ import ( "testing" "github.com/ghodss/yaml" - "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/registry" ) type responseAndError struct { @@ -76,33 +76,52 @@ func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository stri return trp.r[owner+repository] } +func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry { + return trp.r[owner+repository] +} + type testGithubRegistry struct { responses map[registry.Type]urlAndError count int } -func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) { +func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) { tgr.count = tgr.count + 1 ret := tgr.responses[t] - return ret.u, ret.e + return []string{ret.u}, ret.e } func (tgr *testGithubRegistry) List() ([]registry.Type, error) { return []registry.Type{}, fmt.Errorf("List should not be called in the test") } +type testGithubPackageRegistry struct { + responses map[registry.Type]urlAndError + count int +} + +func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) { + tgr.count = tgr.count + 1 + ret := tgr.responses[t] + return []string{ret.u}, ret.e +} + +func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) { + return []registry.Type{}, fmt.Errorf("List should not be called in the test") +} + func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) { r := &typeResolver{ rp: c.registryProvider, } for in, expected := range tests { - actual, err := r.ShortTypeToDownloadURL(in) - if actual != expected.u { - t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual) - } + actual, err := r.ShortTypeToDownloadURLs(in) if err != expected.e { t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) } + if actual[0] != expected.u { + t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual) + } } } @@ -325,12 +344,12 @@ func TestSharedImport(t *testing.T) { func TestShortGithubUrlMapping(t *testing.T) { githubUrlMaps := map[registry.Type]urlAndError{ registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, } tests := map[string]urlAndError{ "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, } test := resolverTestCase{ @@ -342,12 +361,12 @@ func TestShortGithubUrlMapping(t *testing.T) { func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { githubUrlMaps := map[registry.Type]urlAndError{ registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, } tests := map[string]urlAndError{ "github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, - "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, } test := resolverTestCase{ @@ -367,12 +386,12 @@ resources: func TestShortGithubUrl(t *testing.T) { finalImports := []*common.ImportFile{ &common.ImportFile{ - Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1", - Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", + Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1", + Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", Content: "my-content"}, &common.ImportFile{ - Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2", - Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", + Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2", + Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", Content: "my-content-2"}, } diff --git a/registry/github_package_registry.go b/registry/github_package_registry.go new file mode 100644 index 000000000..e74aed99e --- /dev/null +++ b/registry/github_package_registry.go @@ -0,0 +1,120 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "log" + "strings" + + "github.com/google/go-github/github" +) + +// GithubPackageRegistry implements the Registry interface that talks to github and +// expects packages in helm format without versioning and no qualifier in the path. +// Format of the directory for a type is like so: +// package/ +// Chart.yaml +// manifests/ +// foo.yaml +// bar.yaml +// ... +type GithubPackageRegistry struct { + owner string + repository string + client *github.Client +} + +// NewGithubRegistry creates a Registry that can be used to talk to github. +func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry { + return &GithubPackageRegistry{ + owner: owner, + repository: repository, + client: github.NewClient(nil), + } +} + +// List the types from the Registry. +// TODO(vaikas): Figure out how the versions work here. +func (g *GithubPackageRegistry) List() ([]Type, error) { + // Just list all the types at the top level. + types, err := g.getDirs("") + if err != nil { + log.Printf("Failed to list templates: %v", err) + return nil, err + } + + var retTypes []Type + for _, t := range types { + // Check to see if there's a Chart.yaml file in the directory + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil) + if err != nil { + log.Printf("Failed to list package files at path: %s: %v", t, err) + return nil, err + } + for _, f := range dc { + if *f.Type == "file" && *f.Name == "Chart.yaml" { + retTypes = append(retTypes, Type{Name: t}) + } + } + } + + return retTypes, nil +} + +// GetURLs fetches the download URLs for a given Type. +func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) { + path, err := g.MakeRepositoryPath(t) + if err != nil { + return []string{}, err + } + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) + if err != nil { + log.Printf("Failed to list package files at path: %s: %v", path, err) + return []string{}, err + } + downloadURLs := []string{} + for _, f := range dc { + if *f.Type == "file" { + if strings.HasSuffix(*f.Name, ".yaml") { + downloadURLs = append(downloadURLs, *f.DownloadURL) + } + } + } + return downloadURLs, nil +} + +func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) { + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil) + if err != nil { + log.Printf("Failed to get contents at path: %s: %v", dir, err) + return nil, err + } + + var dirs []string + for _, entry := range dc { + if *entry.Type == "dir" { + dirs = append(dirs, *entry.Name) + } + } + + return dirs, nil +} + +// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name. +// The returned repository path will be of the form: +// Type.Name/manifests +func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) { + // Construct the return path + return t.Name + "/manifests", nil +} diff --git a/registry/github_registry.go b/registry/github_registry.go index 67e98bb34..d5933fc00 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -21,7 +21,29 @@ import ( "strings" ) -// GithubRegistry implements the Registry interface that talks to github. +// GithubRegistry implements the Registry interface that talks to github and +// implements Deployment Manager templates registry. +// A registry root must be a directory that contains all the available templates, +// one directory per template. Each template directory then contains version +// directories, each of which in turn contains all the files necessary for that +// version of the template. +// +// For example, a template registry containing two versions of redis +// (implemented in jinja), and one version of replicatedservice (implemented +// in python) would have a directory structure that looks something like this: +// qualifier [optional] prefix to a virtual root within the repository. +// /redis +// /v1 +// redis.jinja +// redis.jinja.schema +// /v2 +// redis.jinja +// redis.jinja.schema +// /replicatedservice +// /v1 +// replicatedservice.python +// replicatedservice.python.schema + type GithubRegistry struct { owner string repository string @@ -74,15 +96,15 @@ func (g *GithubRegistry) List() ([]Type, error) { } // GetURL fetches the download URL for a given Type and checks for existence of a schema file. -func (g *GithubRegistry) GetURL(t Type) (string, error) { +func (g *GithubRegistry) GetURLs(t Type) ([]string, error) { path, err := g.MakeRepositoryPath(t) if err != nil { - return "", err + return []string{}, err } _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to list versions at path: %s: %v", path, err) - return "", err + return []string{}, err } var downloadURL, typeName, schemaName string for _, f := range dc { @@ -97,12 +119,12 @@ func (g *GithubRegistry) GetURL(t Type) (string, error) { } } if downloadURL == "" { - return "", fmt.Errorf("Can not find template %s:%s", t.Name, t.Version) + return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version) } if schemaName == typeName+".schema" { - return downloadURL, nil + return []string{downloadURL}, nil } - return "", fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema") + return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema") } func (g *GithubRegistry) getDirs(dir string) ([]string, error) { diff --git a/registry/registry.go b/registry/registry.go index bab1c18cf..884b84ba7 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -14,37 +14,22 @@ limitations under the License. package registry // Registry abstracts a registry that holds templates, which can be -// used in a Deployment Manager configurations. A registry root must be a -// directory that contains all the available templates, one directory per -// template. Each template directory then contains version directories, each -// of which in turn contains all the files necessary for that version of the -// template. -// For example, a template registry containing two versions of redis -// (implemented in jinja), and one version of replicatedservice (implemented -// in python) would have a directory structure that looks something like this: -// qualifier [optional] prefix to a virtual root within the repository. -// /redis -// /v1 -// redis.jinja -// redis.jinja.schema -// /v2 -// redis.jinja -// redis.jinja.schema -// /replicatedservice -// /v1 -// replicatedservice.python -// replicatedservice.python.schema - +// used in a Deployment Manager configurations. There can be multiple +// implementations of a registry. Currently we support Deployment Manager +// github.com/kubernetes/application-dm-templates +// and helm packages +// github.com/helm/charts +// type Type struct { Collection string - Name string - Version string + Name string + Version string } // Registry abstracts type interactions. type Registry interface { // List all the templates at the given path List() ([]Type, error) - // Get the download URL for a given template and version - GetURL(t Type) (string, error) + // Get the download URL(s) for a given type + GetURLs(t Type) ([]string, error) } diff --git a/registry/registryprovider.go b/registry/registryprovider.go index 2381e3995..e4a833fe0 100644 --- a/registry/registryprovider.go +++ b/registry/registryprovider.go @@ -16,6 +16,7 @@ package registry // RegistryProvider returns factories for creating registries for a given RegistryType. type RegistryProvider interface { GetGithubRegistry(owner string, repository string) Registry + GetGithubPackageRegistry(owner string, repository string) Registry } type DefaultRegistryProvider struct { @@ -24,3 +25,7 @@ type DefaultRegistryProvider struct { func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { return NewGithubRegistry(owner, repository, "") } + +func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry { + return NewGithubPackageRegistry(owner, repository) +} diff --git a/util/kubernetesutil.go b/util/kubernetesutil.go new file mode 100644 index 000000000..b27e6d7a6 --- /dev/null +++ b/util/kubernetesutil.go @@ -0,0 +1,44 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "time" + + "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/common" +) + +func ParseKubernetesObject(object []byte) (*common.Resource, error) { + o := &common.KubernetesObject{} + if err := yaml.Unmarshal(object, &o); err != nil { + return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err) + } + + // Ok, it appears to be a valid object, create a Resource out of it. + r := &common.Resource{} + r.Name = getRandomName(o.Metadata["name"].(string)) + r.Type = o.Kind + + r.Properties = make(map[string]interface{}) + if err := yaml.Unmarshal(object, &r.Properties); err != nil { + return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err) + } + return r, nil +} + +func getRandomName(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().UnixNano()) +} diff --git a/util/kubernetesutil_test.go b/util/kubernetesutil_test.go new file mode 100644 index 000000000..3864354b2 --- /dev/null +++ b/util/kubernetesutil_test.go @@ -0,0 +1,164 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "reflect" + "testing" + + "github.com/ghodss/yaml" + + "github.com/kubernetes/deployment-manager/common" +) + +var serviceInput = ` + kind: "Service" + apiVersion: "v1" + metadata: + name: "mock" + labels: + app: "mock" + spec: + ports: + - + protocol: "TCP" + port: 99 + targetPort: 9949 + selector: + app: "mock" +` + +var serviceExpected = ` +name: mock +type: Service +properties: + kind: "Service" + apiVersion: "v1" + metadata: + name: "mock" + labels: + app: "mock" + spec: + ports: + - + protocol: "TCP" + port: 99 + targetPort: 9949 + selector: + app: "mock" +` + +var rcInput = ` + kind: "ReplicationController" + apiVersion: "v1" + metadata: + name: "mockname" + labels: + app: "mockapp" + foo: "bar" + spec: + replicas: 1 + selector: + app: "mockapp" + template: + metadata: + labels: + app: "mocklabel" + spec: + containers: + - + name: "mock-container" + image: "kubernetes/pause" + ports: + - + containerPort: 9949 + protocol: "TCP" + - + containerPort: 9949 + protocol: "TCP" +` + +var rcExpected = ` +name: mockname +type: ReplicationController +properties: + kind: "ReplicationController" + apiVersion: "v1" + metadata: + name: "mockname" + labels: + app: "mockapp" + foo: "bar" + spec: + replicas: 1 + selector: + app: "mockapp" + template: + metadata: + labels: + app: "mocklabel" + spec: + containers: + - + name: "mock-container" + image: "kubernetes/pause" + ports: + - + containerPort: 9949 + protocol: "TCP" + - + containerPort: 9949 + protocol: "TCP" +` + +func unmarshalResource(t *testing.T, object []byte) (*common.Resource, error) { + r := &common.Resource{} + if err := yaml.Unmarshal([]byte(object), &r); err != nil { + t.Errorf("cannot unmarshal test object (%#v)", err) + return nil, err + } + return r, nil +} + +func testConversion(t *testing.T, object []byte, expected []byte) { + e, err := unmarshalResource(t, expected) + if err != nil { + t.Fatalf("Failed to unmarshal expected Resource: %v", err) + } + + result, err := ParseKubernetesObject(object) + if err != nil { + t.Fatalf("ParseKubernetesObject failed: %v") + } + // Since the object name gets created on the fly, we have to rejigger the returned object + // slightly to make sure the DeepEqual works as expected. + // First validate the name matches the expected format. + var i int + format := e.Name + "-%d" + count, err := fmt.Sscanf(result.Name, format, &i) + if err != nil || count != 1 { + t.Errorf("Name is not as expected, wanted of the form %s got %s", format, result.Name) + } + e.Name = result.Name + if !reflect.DeepEqual(result, e) { + t.Errorf("expected %+v but found %+v", e, result) + } + +} + +func TestSimple(t *testing.T) { + testConversion(t, []byte(rcInput), []byte(rcExpected)) + testConversion(t, []byte(serviceInput), []byte(serviceExpected)) +} diff --git a/util/templateutil.go b/util/templateutil.go index 6a30e81ad..e24da977b 100644 --- a/util/templateutil.go +++ b/util/templateutil.go @@ -19,7 +19,10 @@ import ( "github.com/kubernetes/deployment-manager/common" ) -var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") +var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") + +// RE for Registry that does not support versions and can have multiple files without imports. +var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)") // IsTemplate returns whether a given type is a template. func IsTemplate(t string, imports []*common.ImportFile) bool { @@ -37,5 +40,15 @@ func IsTemplate(t string, imports []*common.ImportFile) bool { // for example: // github.com/kubernetes/application-dm-templates/storage/redis:v1 func IsGithubShortType(t string) bool { - return re.MatchString(t) + return TemplateRegistryMatcher.MatchString(t) +} + +// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github +// package repository type. +// For now, this means using github types: +// github.com/owner/repo/type +// for example: +// github.com/helm/charts/cassandra +func IsGithubShortPackageType(t string) bool { + return PackageRegistryMatcher.MatchString(t) }