Add support for native kubernetes types, helm packages and helm/charts github repo

pull/166/head
Ville Aikas 9 years ago
parent 9654e63d19
commit 9777e72692

@ -155,3 +155,11 @@ type TypeInstance struct {
Manifest string `json:"manifest"` // manifest name Manifest string `json:"manifest"` // manifest name
Path string `json:"path"` // JSON path within manifest Path string `json:"path"` // JSON path within manifest
} }
// KubernetesObject represents a native 'bare' Kubernetes object.
type KubernetesObject struct {
Kind string `json:"kind"`
ApiVersion string `json:"apiVersion"`
Metadata map[string]interface{} `json:"metadata"`
Spec map[string]interface{} `json:"spec"`
}

@ -78,7 +78,7 @@ var usage = func() {
panic("\n") panic("\n")
} }
func getGitRegistry() *registry.GithubRegistry { func getGitRegistry() registry.Registry {
s := strings.Split(*template_registry, "/") s := strings.Split(*template_registry, "/")
if len(s) < 2 { if len(s) < 2 {
panic(fmt.Errorf("invalid template registry: %s", *template_registry)) panic(fmt.Errorf("invalid template registry: %s", *template_registry))
@ -89,7 +89,11 @@ func getGitRegistry() *registry.GithubRegistry {
path = strings.Join(s[2:], "/") path = strings.Join(s[2:], "/")
} }
if s[0] == "helm" {
return registry.NewGithubPackageRegistry(s[0], s[1])
} else {
return registry.NewGithubRegistry(s[0], s[1], path) return registry.NewGithubRegistry(s[0], s[1], path)
}
} }
func main() { func main() {
@ -126,11 +130,17 @@ func execute() {
if len(t.Collection) > 0 { if len(t.Collection) > 0 {
typeSpec = t.Collection + "/" typeSpec = t.Collection + "/"
} }
typeSpec = typeSpec + t.Name + ":" + t.Version typeSpec = typeSpec + t.Name
if len(t.Version) > 0 {
typeSpec = typeSpec + ":" + t.Version
}
fmt.Printf("%s\n", typeSpec) fmt.Printf("%s\n", typeSpec)
downloadURL := getDownloadUrl(t)
fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec) fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec)
fmt.Printf("\tdownload URL: %s\n", downloadURL) fmt.Printf("\tdownload URL(s):\n")
for _, downloadURL := range getDownloadURLs(t) {
fmt.Printf("\t%s\n", downloadURL)
}
} }
case "describe": case "describe":
describeType(args) describeType(args)
@ -195,10 +205,14 @@ func execute() {
usage() usage()
} }
tUrl := getTypeUrl(args[1]) tUrls := getTypeURLs(args[1])
if tUrl == "" { var tUrl = ""
if len(tUrls) == 0 {
// Type is most likely a primitive. // Type is most likely a primitive.
tUrl = args[1] tUrl = args[1]
} else {
// TODO(vaikas): Support packages properly.
tUrl = tUrls[0]
} }
path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl)) path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl))
action := fmt.Sprintf("list deployed instances of type %s", tUrl) action := fmt.Sprintf("list deployed instances of type %s", tUrl)
@ -262,39 +276,39 @@ func describeType(args []string) {
usage() usage()
} }
tUrl := getTypeUrl(args[1]) tUrls := getTypeURLs(args[1])
if tUrl == "" { if len(tUrls) == 0 {
panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1])) panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
} }
schemaUrl := tUrl + ".schema" schemaUrl := tUrls[0] + ".schema"
fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil)) fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
} }
// getTypeUrl returns URL or empty if a primitive type. // getTypeURLs returns URLs or empty list if a primitive type.
func getTypeUrl(tName string) string { func getTypeURLs(tName string) []string {
if util.IsHttpUrl(tName) { if util.IsHttpUrl(tName) {
// User can pass raw URL to template. // User can pass raw URL to template.
return tName return []string{tName}
} }
// User can pass registry type. // User can pass registry type.
t := getRegistryType(tName) t := getRegistryType(tName)
if t == nil { if t == nil {
// Primitive types have no associated URL. // Primitive types have no associated URL.
return "" return []string{}
} }
return getDownloadUrl(*t) return getDownloadURLs(*t)
} }
func getDownloadUrl(t registry.Type) string { func getDownloadURLs(t registry.Type) []string {
git := getGitRegistry() git := getGitRegistry()
url, err := git.GetURL(t) urls, err := git.GetURLs(t)
if err != nil { if err != nil {
panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))
} }
return url return urls
} }
func isHttp(t string) bool { func isHttp(t string) bool {
@ -379,8 +393,6 @@ func getRegistryType(fullType string) *registry.Type {
} }
func buildTemplateFromType(t registry.Type) *common.Template { func buildTemplateFromType(t registry.Type) *common.Template {
downloadURL := getDownloadUrl(t)
props := make(map[string]interface{}) props := make(map[string]interface{})
if *properties != "" { if *properties != "" {
plist := strings.Split(*properties, ",") plist := strings.Split(*properties, ",")
@ -406,7 +418,7 @@ func buildTemplateFromType(t registry.Type) *common.Template {
config := common.Configuration{Resources: []*common.Resource{&common.Resource{ config := common.Configuration{Resources: []*common.Resource{&common.Resource{
Name: name, Name: name,
Type: downloadURL, Type: getDownloadURLs(t)[0],
Properties: props, Properties: props,
}}} }}}

@ -0,0 +1,4 @@
resources:
- name: cassandra
type: github.com/helm/charts/cassandra
properties: null

@ -232,7 +232,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
except schema_validation.ValidationErrors as e: except schema_validation.ValidationErrors as e:
raise ExpansionError(resource['name'], e.message) raise ExpansionError(resource['name'], e.message)
if path.endswith('jinja'): if path.endswith('jinja') or path.endswith('yaml'):
expanded_template = ExpandJinja( expanded_template = ExpandJinja(
source_file, imports[source_file]['content'], resource, imports) source_file, imports[source_file]['content'], resource, imports)
elif path.endswith('py'): elif path.endswith('py'):

@ -21,8 +21,8 @@ import (
"net/http" "net/http"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
) )
const ( const (

@ -16,7 +16,6 @@ package manager
import ( import (
"fmt" "fmt"
"net/http" "net/http"
"regexp"
"time" "time"
"github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/common"
@ -31,8 +30,6 @@ const (
schemaSuffix = ".schema" schemaSuffix = ".schema"
) )
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file // TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL. // or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface { type TypeResolver interface {
@ -45,6 +42,10 @@ type typeResolver struct {
rp registry.RegistryProvider rp registry.RegistryProvider
} }
type fetchUnit struct {
urls []string
}
// NewTypeResolver returns a new initialized TypeResolver. // NewTypeResolver returns a new initialized TypeResolver.
func NewTypeResolver() TypeResolver { func NewTypeResolver() TypeResolver {
ret := &typeResolver{} ret := &typeResolver{}
@ -92,20 +93,29 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
} }
fetched := map[string][]*common.ImportFile{} fetched := map[string][]*common.ImportFile{}
toFetch := make([]string, 0, tr.maxUrls) // TODO(vaikas): Need to account for multiple URLs being fetched for a given type.
toFetch := make([]*fetchUnit, 0, tr.maxUrls)
for _, r := range config.Resources { for _, r := range config.Resources {
// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
u, err := tr.MapFetchableURL(r.Type) urls, err := tr.MapFetchableURLs(r.Type)
if err != nil { if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err)) return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
} }
if len(u) > 0 && !existing[r.Type] { if !existing[r.Type] {
toFetch = append(toFetch, u) f := &fetchUnit{}
fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u}) for _, u := range urls {
if len(u) > 0 {
f.urls = append(f.urls, u)
// Add to existing map so it is not fetched multiple times. // Add to existing map so it is not fetched multiple times.
existing[r.Type] = true existing[r.Type] = true
} }
} }
if len(f.urls) > 0 {
toFetch = append(toFetch, f)
fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]})
}
}
}
count := 0 count := 0
for len(toFetch) > 0 { for len(toFetch) > 0 {
@ -122,13 +132,21 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls)) fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
} }
url := toFetch[0] templates := []string{}
template, err := performHTTPGet(tr.getter, url, false) url := toFetch[0].urls[0]
for _, u := range toFetch[0].urls {
template, err := performHTTPGet(tr.getter, u, false)
if err != nil { if err != nil {
return nil, resolverError(config, err) return nil, resolverError(config, err)
} }
templates = append(templates, template)
}
for _, i := range fetched[url] { for _, i := range fetched[url] {
template, err := parseContent(templates)
if err != nil {
return nil, resolverError(config, err)
}
i.Content = template i.Content = template
} }
@ -147,18 +165,19 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
for _, v := range s.Imports { for _, v := range s.Imports {
i := &common.ImportFile{Name: v.Name} i := &common.ImportFile{Name: v.Name}
var existingSchema string var existingSchema string
u, conversionErr := tr.MapFetchableURL(v.Path) urls, conversionErr := tr.MapFetchableURLs(v.Path)
if conversionErr != nil { if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr)) return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
} }
if len(urls) == 0 {
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name // If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema. // for a schema.
if len(u) == 0 { urls = []string{v.Path}
u = v.Path
} }
for _, u := range urls {
if len(fetched[u]) == 0 { if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch. // If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, u) toFetch = append(toFetch, &fetchUnit{[]string{u}})
} else { } else {
// If this is not a new import URL and we've already fetched its contents, // If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and // reuse them. Also, check if we also found a schema for that import URL and
@ -176,6 +195,7 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) &common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
} }
} }
}
// Add the schema we've fetched as the schema for any templates which used this URL. // Add the schema we've fetched as the schema for any templates which used this URL.
for _, i := range fetched[url] { for _, i := range fetched[url] {
@ -197,29 +217,70 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
return ret, nil return ret, nil
} }
// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL // MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for // and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
// example) will return empty string. // example) will return empty string.
func (tr *typeResolver) MapFetchableURL(t string) (string, error) { func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) {
if util.IsGithubShortType(t) { if util.IsGithubShortType(t) {
return tr.ShortTypeToDownloadURL(t) return tr.ShortTypeToDownloadURLs(t)
} else if util.IsGithubShortPackageType(t) {
return tr.ShortTypeToPackageDownloadURLs(t)
} else if util.IsHttpUrl(t) { } else if util.IsHttpUrl(t) {
return t, nil return []string{t}, nil
} }
return "", nil return []string{}, nil
} }
// ShortTypeToDownloadURL converts a github URL into downloadable URL from github. // ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call: // Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version // github.com/owner/repo/qualifier/type:version
// for example: // for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1 // github.com/kubernetes/application-dm-templates/storage/redis:v1
func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) { func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) {
m := re.FindStringSubmatch(template) m := util.TemplateRegistryMatcher.FindStringSubmatch(template)
if len(m) != 6 { if len(m) != 6 {
return "", fmt.Errorf("Failed to parse short github url: %s", template) return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
} }
r := tr.rp.GetGithubRegistry(m[1], m[2]) r := tr.rp.GetGithubRegistry(m[1], m[2])
t := registry.Type{m[3], m[4], m[5]} t := registry.Type{m[3], m[4], m[5]}
return r.GetURL(t) return r.GetURLs(t)
}
// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) {
m := util.PackageRegistryMatcher.FindStringSubmatch(template)
if len(m) != 4 {
return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
}
r := tr.rp.GetGithubPackageRegistry(m[1], m[2])
t := registry.Type{Name: m[3]}
return r.GetURLs(t)
}
func parseContent(templates []string) (string, error) {
if len(templates) == 1 {
return templates[0], nil
} else {
// If there are multiple URLs that need to be fetched, that implies it's a package
// of raw Kubernetes objects. We need to fetch them all as a unit and create a
// template representing a package out of that below.
fakeConfig := &common.Configuration{}
for _, template := range templates {
o, err := util.ParseKubernetesObject([]byte(template))
if err != nil {
return "", fmt.Errorf("not a kubernetes object: %+v", template)
}
// Looks like a native Kubernetes object, create a configuration out of it
fakeConfig.Resources = append(fakeConfig.Resources, o)
}
marshalled, err := yaml.Marshal(fakeConfig)
if err != nil {
return "", fmt.Errorf("Failed to marshal: %+v", fakeConfig)
}
return string(marshalled), nil
}
} }

@ -22,8 +22,8 @@ import (
"testing" "testing"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/registry"
) )
type responseAndError struct { type responseAndError struct {
@ -76,33 +76,52 @@ func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository stri
return trp.r[owner+repository] return trp.r[owner+repository]
} }
func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry {
return trp.r[owner+repository]
}
type testGithubRegistry struct { type testGithubRegistry struct {
responses map[registry.Type]urlAndError responses map[registry.Type]urlAndError
count int count int
} }
func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) { func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1 tgr.count = tgr.count + 1
ret := tgr.responses[t] ret := tgr.responses[t]
return ret.u, ret.e return []string{ret.u}, ret.e
} }
func (tgr *testGithubRegistry) List() ([]registry.Type, error) { func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test") return []registry.Type{}, fmt.Errorf("List should not be called in the test")
} }
type testGithubPackageRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return []string{ret.u}, ret.e
}
func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) { func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
r := &typeResolver{ r := &typeResolver{
rp: c.registryProvider, rp: c.registryProvider,
} }
for in, expected := range tests { for in, expected := range tests {
actual, err := r.ShortTypeToDownloadURL(in) actual, err := r.ShortTypeToDownloadURLs(in)
if actual != expected.u {
t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual)
}
if err != expected.e { if err != expected.e {
t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
} }
if actual[0] != expected.u {
t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual)
}
} }
} }

@ -0,0 +1,120 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"log"
"strings"
"github.com/google/go-github/github"
)
// GithubPackageRegistry implements the Registry interface that talks to github and
// expects packages in helm format without versioning and no qualifier in the path.
// Format of the directory for a type is like so:
// package/
// Chart.yaml
// manifests/
// foo.yaml
// bar.yaml
// ...
type GithubPackageRegistry struct {
owner string
repository string
client *github.Client
}
// NewGithubRegistry creates a Registry that can be used to talk to github.
func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry {
return &GithubPackageRegistry{
owner: owner,
repository: repository,
client: github.NewClient(nil),
}
}
// List the types from the Registry.
// TODO(vaikas): Figure out how the versions work here.
func (g *GithubPackageRegistry) List() ([]Type, error) {
// Just list all the types at the top level.
types, err := g.getDirs("")
if err != nil {
log.Printf("Failed to list templates: %v", err)
return nil, err
}
var retTypes []Type
for _, t := range types {
// Check to see if there's a Chart.yaml file in the directory
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", t, err)
return nil, err
}
for _, f := range dc {
if *f.Type == "file" && *f.Name == "Chart.yaml" {
retTypes = append(retTypes, Type{Name: t})
}
}
}
return retTypes, nil
}
// GetURLs fetches the download URLs for a given Type.
func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return []string{}, err
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", path, err)
return []string{}, err
}
downloadURLs := []string{}
for _, f := range dc {
if *f.Type == "file" {
if strings.HasSuffix(*f.Name, ".yaml") {
downloadURLs = append(downloadURLs, *f.DownloadURL)
}
}
}
return downloadURLs, nil
}
func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil)
if err != nil {
log.Printf("Failed to get contents at path: %s: %v", dir, err)
return nil, err
}
var dirs []string
for _, entry := range dc {
if *entry.Type == "dir" {
dirs = append(dirs, *entry.Name)
}
}
return dirs, nil
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name.
// The returned repository path will be of the form:
// Type.Name/manifests
func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
// Construct the return path
return t.Name + "/manifests", nil
}

@ -21,7 +21,29 @@ import (
"strings" "strings"
) )
// GithubRegistry implements the Registry interface that talks to github. // GithubRegistry implements the Registry interface that talks to github and
// implements Deployment Manager templates registry.
// A registry root must be a directory that contains all the available templates,
// one directory per template. Each template directory then contains version
// directories, each of which in turn contains all the files necessary for that
// version of the template.
//
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
type GithubRegistry struct { type GithubRegistry struct {
owner string owner string
repository string repository string
@ -74,15 +96,15 @@ func (g *GithubRegistry) List() ([]Type, error) {
} }
// GetURL fetches the download URL for a given Type and checks for existence of a schema file. // GetURL fetches the download URL for a given Type and checks for existence of a schema file.
func (g *GithubRegistry) GetURL(t Type) (string, error) { func (g *GithubRegistry) GetURLs(t Type) ([]string, error) {
path, err := g.MakeRepositoryPath(t) path, err := g.MakeRepositoryPath(t)
if err != nil { if err != nil {
return "", err return []string{}, err
} }
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil { if err != nil {
log.Printf("Failed to list versions at path: %s: %v", path, err) log.Printf("Failed to list versions at path: %s: %v", path, err)
return "", err return []string{}, err
} }
var downloadURL, typeName, schemaName string var downloadURL, typeName, schemaName string
for _, f := range dc { for _, f := range dc {
@ -97,12 +119,12 @@ func (g *GithubRegistry) GetURL(t Type) (string, error) {
} }
} }
if downloadURL == "" { if downloadURL == "" {
return "", fmt.Errorf("Can not find template %s:%s", t.Name, t.Version) return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version)
} }
if schemaName == typeName+".schema" { if schemaName == typeName+".schema" {
return downloadURL, nil return []string{downloadURL}, nil
} }
return "", fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema") return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema")
} }
func (g *GithubRegistry) getDirs(dir string) ([]string, error) { func (g *GithubRegistry) getDirs(dir string) ([]string, error) {

@ -14,27 +14,12 @@ limitations under the License.
package registry package registry
// Registry abstracts a registry that holds templates, which can be // Registry abstracts a registry that holds templates, which can be
// used in a Deployment Manager configurations. A registry root must be a // used in a Deployment Manager configurations. There can be multiple
// directory that contains all the available templates, one directory per // implementations of a registry. Currently we support Deployment Manager
// template. Each template directory then contains version directories, each // github.com/kubernetes/application-dm-templates
// of which in turn contains all the files necessary for that version of the // and helm packages
// template. // github.com/helm/charts
// For example, a template registry containing two versions of redis //
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
type Type struct { type Type struct {
Collection string Collection string
Name string Name string
@ -45,6 +30,6 @@ type Type struct {
type Registry interface { type Registry interface {
// List all the templates at the given path // List all the templates at the given path
List() ([]Type, error) List() ([]Type, error)
// Get the download URL for a given template and version // Get the download URL(s) for a given type
GetURL(t Type) (string, error) GetURLs(t Type) ([]string, error)
} }

@ -16,6 +16,7 @@ package registry
// RegistryProvider returns factories for creating registries for a given RegistryType. // RegistryProvider returns factories for creating registries for a given RegistryType.
type RegistryProvider interface { type RegistryProvider interface {
GetGithubRegistry(owner string, repository string) Registry GetGithubRegistry(owner string, repository string) Registry
GetGithubPackageRegistry(owner string, repository string) Registry
} }
type DefaultRegistryProvider struct { type DefaultRegistryProvider struct {
@ -24,3 +25,7 @@ type DefaultRegistryProvider struct {
func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
return NewGithubRegistry(owner, repository, "") return NewGithubRegistry(owner, repository, "")
} }
func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry {
return NewGithubPackageRegistry(owner, repository)
}

@ -0,0 +1,44 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"time"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
func ParseKubernetesObject(object []byte) (*common.Resource, error) {
o := &common.KubernetesObject{}
if err := yaml.Unmarshal(object, &o); err != nil {
return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err)
}
// Ok, it appears to be a valid object, create a Resource out of it.
r := &common.Resource{}
r.Name = getRandomName(o.Metadata["name"].(string))
r.Type = o.Kind
r.Properties = make(map[string]interface{})
if err := yaml.Unmarshal(object, &r.Properties); err != nil {
return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err)
}
return r, nil
}
func getRandomName(prefix string) string {
return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().UnixNano())
}

@ -0,0 +1,164 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"reflect"
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
var serviceInput = `
kind: "Service"
apiVersion: "v1"
metadata:
name: "mock"
labels:
app: "mock"
spec:
ports:
-
protocol: "TCP"
port: 99
targetPort: 9949
selector:
app: "mock"
`
var serviceExpected = `
name: mock
type: Service
properties:
kind: "Service"
apiVersion: "v1"
metadata:
name: "mock"
labels:
app: "mock"
spec:
ports:
-
protocol: "TCP"
port: 99
targetPort: 9949
selector:
app: "mock"
`
var rcInput = `
kind: "ReplicationController"
apiVersion: "v1"
metadata:
name: "mockname"
labels:
app: "mockapp"
foo: "bar"
spec:
replicas: 1
selector:
app: "mockapp"
template:
metadata:
labels:
app: "mocklabel"
spec:
containers:
-
name: "mock-container"
image: "kubernetes/pause"
ports:
-
containerPort: 9949
protocol: "TCP"
-
containerPort: 9949
protocol: "TCP"
`
var rcExpected = `
name: mockname
type: ReplicationController
properties:
kind: "ReplicationController"
apiVersion: "v1"
metadata:
name: "mockname"
labels:
app: "mockapp"
foo: "bar"
spec:
replicas: 1
selector:
app: "mockapp"
template:
metadata:
labels:
app: "mocklabel"
spec:
containers:
-
name: "mock-container"
image: "kubernetes/pause"
ports:
-
containerPort: 9949
protocol: "TCP"
-
containerPort: 9949
protocol: "TCP"
`
func unmarshalResource(t *testing.T, object []byte) (*common.Resource, error) {
r := &common.Resource{}
if err := yaml.Unmarshal([]byte(object), &r); err != nil {
t.Errorf("cannot unmarshal test object (%#v)", err)
return nil, err
}
return r, nil
}
func testConversion(t *testing.T, object []byte, expected []byte) {
e, err := unmarshalResource(t, expected)
if err != nil {
t.Fatalf("Failed to unmarshal expected Resource: %v", err)
}
result, err := ParseKubernetesObject(object)
if err != nil {
t.Fatalf("ParseKubernetesObject failed: %v")
}
// Since the object name gets created on the fly, we have to rejigger the returned object
// slightly to make sure the DeepEqual works as expected.
// First validate the name matches the expected format.
var i int
format := e.Name + "-%d"
count, err := fmt.Sscanf(result.Name, format, &i)
if err != nil || count != 1 {
t.Errorf("Name is not as expected, wanted of the form %s got %s", format, result.Name)
}
e.Name = result.Name
if !reflect.DeepEqual(result, e) {
t.Errorf("expected %+v but found %+v", e, result)
}
}
func TestSimple(t *testing.T) {
testConversion(t, []byte(rcInput), []byte(rcExpected))
testConversion(t, []byte(serviceInput), []byte(serviceExpected))
}

@ -19,7 +19,10 @@ import (
"github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/common"
) )
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// RE for Registry that does not support versions and can have multiple files without imports.
var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
// IsTemplate returns whether a given type is a template. // IsTemplate returns whether a given type is a template.
func IsTemplate(t string, imports []*common.ImportFile) bool { func IsTemplate(t string, imports []*common.ImportFile) bool {
@ -37,5 +40,15 @@ func IsTemplate(t string, imports []*common.ImportFile) bool {
// for example: // for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1 // github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool { func IsGithubShortType(t string) bool {
return re.MatchString(t) return TemplateRegistryMatcher.MatchString(t)
}
// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
// package repository type.
// For now, this means using github types:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func IsGithubShortPackageType(t string) bool {
return PackageRegistryMatcher.MatchString(t)
} }

Loading…
Cancel
Save