Merge pull request #166 from vaikas-google/helm

Add support for helm packages, native kubernetes objects and helm repository format.
pull/168/head
Brendan Melville 9 years ago
commit 0793a2dfd7

@ -155,3 +155,11 @@ type TypeInstance struct {
Manifest string `json:"manifest"` // manifest name
Path string `json:"path"` // JSON path within manifest
}
// KubernetesObject represents a native 'bare' Kubernetes object.
type KubernetesObject struct {
Kind string `json:"kind"`
ApiVersion string `json:"apiVersion"`
Metadata map[string]interface{} `json:"metadata"`
Spec map[string]interface{} `json:"spec"`
}

@ -78,7 +78,7 @@ var usage = func() {
panic("\n")
}
func getGitRegistry() *registry.GithubRegistry {
func getGitRegistry() registry.Registry {
s := strings.Split(*template_registry, "/")
if len(s) < 2 {
panic(fmt.Errorf("invalid template registry: %s", *template_registry))
@ -89,7 +89,11 @@ func getGitRegistry() *registry.GithubRegistry {
path = strings.Join(s[2:], "/")
}
return registry.NewGithubRegistry(s[0], s[1], path)
if s[0] == "helm" {
return registry.NewGithubPackageRegistry(s[0], s[1])
} else {
return registry.NewGithubRegistry(s[0], s[1], path)
}
}
func main() {
@ -126,11 +130,17 @@ func execute() {
if len(t.Collection) > 0 {
typeSpec = t.Collection + "/"
}
typeSpec = typeSpec + t.Name + ":" + t.Version
typeSpec = typeSpec + t.Name
if len(t.Version) > 0 {
typeSpec = typeSpec + ":" + t.Version
}
fmt.Printf("%s\n", typeSpec)
downloadURL := getDownloadUrl(t)
fmt.Printf("\tshort URL: github.com/%s/%s\n", *template_registry, typeSpec)
fmt.Printf("\tdownload URL: %s\n", downloadURL)
fmt.Printf("\tdownload URL(s):\n")
for _, downloadURL := range getDownloadURLs(t) {
fmt.Printf("\t%s\n", downloadURL)
}
}
case "describe":
describeType(args)
@ -195,10 +205,14 @@ func execute() {
usage()
}
tUrl := getTypeUrl(args[1])
if tUrl == "" {
tUrls := getTypeURLs(args[1])
var tUrl = ""
if len(tUrls) == 0 {
// Type is most likely a primitive.
tUrl = args[1]
} else {
// TODO(vaikas): Support packages properly.
tUrl = tUrls[0]
}
path := fmt.Sprintf("types/%s/instances", url.QueryEscape(tUrl))
action := fmt.Sprintf("list deployed instances of type %s", tUrl)
@ -262,39 +276,39 @@ func describeType(args []string) {
usage()
}
tUrl := getTypeUrl(args[1])
if tUrl == "" {
tUrls := getTypeURLs(args[1])
if len(tUrls) == 0 {
panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
}
schemaUrl := tUrl + ".schema"
fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil))
schemaUrl := tUrls[0] + ".schema"
fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrls[0]+")", nil))
}
// getTypeUrl returns URL or empty if a primitive type.
func getTypeUrl(tName string) string {
// getTypeURLs returns URLs or empty list if a primitive type.
func getTypeURLs(tName string) []string {
if util.IsHttpUrl(tName) {
// User can pass raw URL to template.
return tName
return []string{tName}
}
// User can pass registry type.
t := getRegistryType(tName)
if t == nil {
// Primitive types have no associated URL.
return ""
return []string{}
}
return getDownloadUrl(*t)
return getDownloadURLs(*t)
}
func getDownloadUrl(t registry.Type) string {
func getDownloadURLs(t registry.Type) []string {
git := getGitRegistry()
url, err := git.GetURL(t)
urls, err := git.GetURLs(t)
if err != nil {
panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))
}
return url
return urls
}
func isHttp(t string) bool {
@ -379,8 +393,6 @@ func getRegistryType(fullType string) *registry.Type {
}
func buildTemplateFromType(t registry.Type) *common.Template {
downloadURL := getDownloadUrl(t)
props := make(map[string]interface{})
if *properties != "" {
plist := strings.Split(*properties, ",")
@ -406,7 +418,7 @@ func buildTemplateFromType(t registry.Type) *common.Template {
config := common.Configuration{Resources: []*common.Resource{&common.Resource{
Name: name,
Type: downloadURL,
Type: getDownloadURLs(t)[0],
Properties: props,
}}}

@ -0,0 +1,4 @@
resources:
- name: cassandra
type: github.com/helm/charts/cassandra
properties: null

@ -232,7 +232,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
except schema_validation.ValidationErrors as e:
raise ExpansionError(resource['name'], e.message)
if path.endswith('jinja'):
if path.endswith('jinja') or path.endswith('yaml'):
expanded_template = ExpandJinja(
source_file, imports[source_file]['content'], resource, imports)
elif path.endswith('py'):

@ -21,8 +21,8 @@ import (
"net/http"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
)
const (

@ -16,7 +16,6 @@ package manager
import (
"fmt"
"net/http"
"regexp"
"time"
"github.com/kubernetes/deployment-manager/common"
@ -31,8 +30,6 @@ const (
schemaSuffix = ".schema"
)
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface {
@ -45,6 +42,10 @@ type typeResolver struct {
rp registry.RegistryProvider
}
type fetchUnit struct {
urls []string
}
// NewTypeResolver returns a new initialized TypeResolver.
func NewTypeResolver() TypeResolver {
ret := &typeResolver{}
@ -92,18 +93,27 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
}
fetched := map[string][]*common.ImportFile{}
toFetch := make([]string, 0, tr.maxUrls)
// TODO(vaikas): Need to account for multiple URLs being fetched for a given type.
toFetch := make([]*fetchUnit, 0, tr.maxUrls)
for _, r := range config.Resources {
// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
u, err := tr.MapFetchableURL(r.Type)
urls, err := tr.MapFetchableURLs(r.Type)
if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
}
if len(u) > 0 && !existing[r.Type] {
toFetch = append(toFetch, u)
fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u})
// Add to existing map so it is not fetched multiple times.
existing[r.Type] = true
if !existing[r.Type] {
f := &fetchUnit{}
for _, u := range urls {
if len(u) > 0 {
f.urls = append(f.urls, u)
// Add to existing map so it is not fetched multiple times.
existing[r.Type] = true
}
}
if len(f.urls) > 0 {
toFetch = append(toFetch, f)
fetched[f.urls[0]] = append(fetched[f.urls[0]], &common.ImportFile{Name: r.Type, Path: f.urls[0]})
}
}
}
@ -122,13 +132,21 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
}
url := toFetch[0]
template, err := performHTTPGet(tr.getter, url, false)
if err != nil {
return nil, resolverError(config, err)
templates := []string{}
url := toFetch[0].urls[0]
for _, u := range toFetch[0].urls {
template, err := performHTTPGet(tr.getter, u, false)
if err != nil {
return nil, resolverError(config, err)
}
templates = append(templates, template)
}
for _, i := range fetched[url] {
template, err := parseContent(templates)
if err != nil {
return nil, resolverError(config, err)
}
i.Content = template
}
@ -147,33 +165,35 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
for _, v := range s.Imports {
i := &common.ImportFile{Name: v.Name}
var existingSchema string
u, conversionErr := tr.MapFetchableURL(v.Path)
urls, conversionErr := tr.MapFetchableURLs(v.Path)
if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
}
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema.
if len(u) == 0 {
u = v.Path
if len(urls) == 0 {
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema.
urls = []string{v.Path}
}
if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, u)
} else {
// If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and
// record those contents for re-use as well.
if fetched[u][0].Content != "" {
i.Content = fetched[u][0].Content
if len(fetched[u+schemaSuffix]) > 0 {
existingSchema = fetched[u+schemaSuffix][0].Content
for _, u := range urls {
if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, &fetchUnit{[]string{u}})
} else {
// If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and
// record those contents for re-use as well.
if fetched[u][0].Content != "" {
i.Content = fetched[u][0].Content
if len(fetched[u+schemaSuffix]) > 0 {
existingSchema = fetched[u+schemaSuffix][0].Content
}
}
}
}
fetched[u] = append(fetched[u], i)
if existingSchema != "" {
fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
fetched[u] = append(fetched[u], i)
if existingSchema != "" {
fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
}
}
}
@ -197,29 +217,70 @@ func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*co
return ret, nil
}
// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL
// MapFetchableUrls checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
// example) will return empty string.
func (tr *typeResolver) MapFetchableURL(t string) (string, error) {
func (tr *typeResolver) MapFetchableURLs(t string) ([]string, error) {
if util.IsGithubShortType(t) {
return tr.ShortTypeToDownloadURL(t)
return tr.ShortTypeToDownloadURLs(t)
} else if util.IsGithubShortPackageType(t) {
return tr.ShortTypeToPackageDownloadURLs(t)
} else if util.IsHttpUrl(t) {
return t, nil
return []string{t}, nil
}
return "", nil
return []string{}, nil
}
// ShortTypeToDownloadURL converts a github URL into downloadable URL from github.
// ShortTypeToDownloadURLs converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) {
m := re.FindStringSubmatch(template)
func (tr *typeResolver) ShortTypeToDownloadURLs(template string) ([]string, error) {
m := util.TemplateRegistryMatcher.FindStringSubmatch(template)
if len(m) != 6 {
return "", fmt.Errorf("Failed to parse short github url: %s", template)
return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
}
r := tr.rp.GetGithubRegistry(m[1], m[2])
t := registry.Type{m[3], m[4], m[5]}
return r.GetURL(t)
return r.GetURLs(t)
}
// ShortTypeToPackageDownloadURLs converts a github URL into downloadable URLs from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func (tr *typeResolver) ShortTypeToPackageDownloadURLs(template string) ([]string, error) {
m := util.PackageRegistryMatcher.FindStringSubmatch(template)
if len(m) != 4 {
return []string{}, fmt.Errorf("Failed to parse short github url: %s", template)
}
r := tr.rp.GetGithubPackageRegistry(m[1], m[2])
t := registry.Type{Name: m[3]}
return r.GetURLs(t)
}
func parseContent(templates []string) (string, error) {
if len(templates) == 1 {
return templates[0], nil
} else {
// If there are multiple URLs that need to be fetched, that implies it's a package
// of raw Kubernetes objects. We need to fetch them all as a unit and create a
// template representing a package out of that below.
fakeConfig := &common.Configuration{}
for _, template := range templates {
o, err := util.ParseKubernetesObject([]byte(template))
if err != nil {
return "", fmt.Errorf("not a kubernetes object: %+v", template)
}
// Looks like a native Kubernetes object, create a configuration out of it
fakeConfig.Resources = append(fakeConfig.Resources, o)
}
marshalled, err := yaml.Marshal(fakeConfig)
if err != nil {
return "", fmt.Errorf("Failed to marshal: %+v", fakeConfig)
}
return string(marshalled), nil
}
}

@ -22,8 +22,8 @@ import (
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/registry"
)
type responseAndError struct {
@ -76,33 +76,52 @@ func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository stri
return trp.r[owner+repository]
}
func (trp *testRegistryProvider) GetGithubPackageRegistry(owner string, repository string) registry.Registry {
return trp.r[owner+repository]
}
type testGithubRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) {
func (tgr *testGithubRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return ret.u, ret.e
return []string{ret.u}, ret.e
}
func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
type testGithubPackageRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubPackageRegistry) GetURLs(t registry.Type) ([]string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return []string{ret.u}, ret.e
}
func (tgr *testGithubPackageRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
r := &typeResolver{
rp: c.registryProvider,
}
for in, expected := range tests {
actual, err := r.ShortTypeToDownloadURL(in)
if actual != expected.u {
t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual)
}
actual, err := r.ShortTypeToDownloadURLs(in)
if err != expected.e {
t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
}
if actual[0] != expected.u {
t.Errorf("failed on: %s : expected %s but got %v", in, expected.u, actual)
}
}
}
@ -325,12 +344,12 @@ func TestSharedImport(t *testing.T) {
func TestShortGithubUrlMapping(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
@ -342,12 +361,12 @@ func TestShortGithubUrlMapping(t *testing.T) {
func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
"github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
@ -367,12 +386,12 @@ resources:
func TestShortGithubUrl(t *testing.T) {
finalImports := []*common.ImportFile{
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py",
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py",
Content: "my-content"},
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py",
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py",
Content: "my-content-2"},
}

@ -0,0 +1,120 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"log"
"strings"
"github.com/google/go-github/github"
)
// GithubPackageRegistry implements the Registry interface that talks to github and
// expects packages in helm format without versioning and no qualifier in the path.
// Format of the directory for a type is like so:
// package/
// Chart.yaml
// manifests/
// foo.yaml
// bar.yaml
// ...
type GithubPackageRegistry struct {
owner string
repository string
client *github.Client
}
// NewGithubRegistry creates a Registry that can be used to talk to github.
func NewGithubPackageRegistry(owner, repository string) *GithubPackageRegistry {
return &GithubPackageRegistry{
owner: owner,
repository: repository,
client: github.NewClient(nil),
}
}
// List the types from the Registry.
// TODO(vaikas): Figure out how the versions work here.
func (g *GithubPackageRegistry) List() ([]Type, error) {
// Just list all the types at the top level.
types, err := g.getDirs("")
if err != nil {
log.Printf("Failed to list templates: %v", err)
return nil, err
}
var retTypes []Type
for _, t := range types {
// Check to see if there's a Chart.yaml file in the directory
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, t, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", t, err)
return nil, err
}
for _, f := range dc {
if *f.Type == "file" && *f.Name == "Chart.yaml" {
retTypes = append(retTypes, Type{Name: t})
}
}
}
return retTypes, nil
}
// GetURLs fetches the download URLs for a given Type.
func (g *GithubPackageRegistry) GetURLs(t Type) ([]string, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return []string{}, err
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list package files at path: %s: %v", path, err)
return []string{}, err
}
downloadURLs := []string{}
for _, f := range dc {
if *f.Type == "file" {
if strings.HasSuffix(*f.Name, ".yaml") {
downloadURLs = append(downloadURLs, *f.DownloadURL)
}
}
}
return downloadURLs, nil
}
func (g *GithubPackageRegistry) getDirs(dir string) ([]string, error) {
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil)
if err != nil {
log.Printf("Failed to get contents at path: %s: %v", dir, err)
return nil, err
}
var dirs []string
for _, entry := range dc {
if *entry.Type == "dir" {
dirs = append(dirs, *entry.Name)
}
}
return dirs, nil
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name.
// The returned repository path will be of the form:
// Type.Name/manifests
func (g *GithubPackageRegistry) MakeRepositoryPath(t Type) (string, error) {
// Construct the return path
return t.Name + "/manifests", nil
}

@ -21,7 +21,29 @@ import (
"strings"
)
// GithubRegistry implements the Registry interface that talks to github.
// GithubRegistry implements the Registry interface that talks to github and
// implements Deployment Manager templates registry.
// A registry root must be a directory that contains all the available templates,
// one directory per template. Each template directory then contains version
// directories, each of which in turn contains all the files necessary for that
// version of the template.
//
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
type GithubRegistry struct {
owner string
repository string
@ -74,15 +96,15 @@ func (g *GithubRegistry) List() ([]Type, error) {
}
// GetURL fetches the download URL for a given Type and checks for existence of a schema file.
func (g *GithubRegistry) GetURL(t Type) (string, error) {
func (g *GithubRegistry) GetURLs(t Type) ([]string, error) {
path, err := g.MakeRepositoryPath(t)
if err != nil {
return "", err
return []string{}, err
}
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list versions at path: %s: %v", path, err)
return "", err
return []string{}, err
}
var downloadURL, typeName, schemaName string
for _, f := range dc {
@ -97,12 +119,12 @@ func (g *GithubRegistry) GetURL(t Type) (string, error) {
}
}
if downloadURL == "" {
return "", fmt.Errorf("Can not find template %s:%s", t.Name, t.Version)
return []string{}, fmt.Errorf("Can not find template %s:%s", t.Name, t.Version)
}
if schemaName == typeName+".schema" {
return downloadURL, nil
return []string{downloadURL}, nil
}
return "", fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema")
return []string{}, fmt.Errorf("Can not find schema for %s:%s, expected to find %s", t.Name, t.Version, typeName+".schema")
}
func (g *GithubRegistry) getDirs(dir string) ([]string, error) {

@ -14,37 +14,22 @@ limitations under the License.
package registry
// Registry abstracts a registry that holds templates, which can be
// used in a Deployment Manager configurations. A registry root must be a
// directory that contains all the available templates, one directory per
// template. Each template directory then contains version directories, each
// of which in turn contains all the files necessary for that version of the
// template.
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
// redis.jinja.schema
// /v2
// redis.jinja
// redis.jinja.schema
// /replicatedservice
// /v1
// replicatedservice.python
// replicatedservice.python.schema
// used in a Deployment Manager configurations. There can be multiple
// implementations of a registry. Currently we support Deployment Manager
// github.com/kubernetes/application-dm-templates
// and helm packages
// github.com/helm/charts
//
type Type struct {
Collection string
Name string
Version string
Name string
Version string
}
// Registry abstracts type interactions.
type Registry interface {
// List all the templates at the given path
List() ([]Type, error)
// Get the download URL for a given template and version
GetURL(t Type) (string, error)
// Get the download URL(s) for a given type
GetURLs(t Type) ([]string, error)
}

@ -16,6 +16,7 @@ package registry
// RegistryProvider returns factories for creating registries for a given RegistryType.
type RegistryProvider interface {
GetGithubRegistry(owner string, repository string) Registry
GetGithubPackageRegistry(owner string, repository string) Registry
}
type DefaultRegistryProvider struct {
@ -24,3 +25,7 @@ type DefaultRegistryProvider struct {
func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
return NewGithubRegistry(owner, repository, "")
}
func (drp *DefaultRegistryProvider) GetGithubPackageRegistry(owner string, repository string) Registry {
return NewGithubPackageRegistry(owner, repository)
}

@ -0,0 +1,44 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"time"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
func ParseKubernetesObject(object []byte) (*common.Resource, error) {
o := &common.KubernetesObject{}
if err := yaml.Unmarshal(object, &o); err != nil {
return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err)
}
// Ok, it appears to be a valid object, create a Resource out of it.
r := &common.Resource{}
r.Name = getRandomName(o.Metadata["name"].(string))
r.Type = o.Kind
r.Properties = make(map[string]interface{})
if err := yaml.Unmarshal(object, &r.Properties); err != nil {
return nil, fmt.Errorf("cannot unmarshal native kubernetes object (%#v)", err)
}
return r, nil
}
func getRandomName(prefix string) string {
return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().UnixNano())
}

@ -0,0 +1,164 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"reflect"
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
var serviceInput = `
kind: "Service"
apiVersion: "v1"
metadata:
name: "mock"
labels:
app: "mock"
spec:
ports:
-
protocol: "TCP"
port: 99
targetPort: 9949
selector:
app: "mock"
`
var serviceExpected = `
name: mock
type: Service
properties:
kind: "Service"
apiVersion: "v1"
metadata:
name: "mock"
labels:
app: "mock"
spec:
ports:
-
protocol: "TCP"
port: 99
targetPort: 9949
selector:
app: "mock"
`
var rcInput = `
kind: "ReplicationController"
apiVersion: "v1"
metadata:
name: "mockname"
labels:
app: "mockapp"
foo: "bar"
spec:
replicas: 1
selector:
app: "mockapp"
template:
metadata:
labels:
app: "mocklabel"
spec:
containers:
-
name: "mock-container"
image: "kubernetes/pause"
ports:
-
containerPort: 9949
protocol: "TCP"
-
containerPort: 9949
protocol: "TCP"
`
var rcExpected = `
name: mockname
type: ReplicationController
properties:
kind: "ReplicationController"
apiVersion: "v1"
metadata:
name: "mockname"
labels:
app: "mockapp"
foo: "bar"
spec:
replicas: 1
selector:
app: "mockapp"
template:
metadata:
labels:
app: "mocklabel"
spec:
containers:
-
name: "mock-container"
image: "kubernetes/pause"
ports:
-
containerPort: 9949
protocol: "TCP"
-
containerPort: 9949
protocol: "TCP"
`
func unmarshalResource(t *testing.T, object []byte) (*common.Resource, error) {
r := &common.Resource{}
if err := yaml.Unmarshal([]byte(object), &r); err != nil {
t.Errorf("cannot unmarshal test object (%#v)", err)
return nil, err
}
return r, nil
}
func testConversion(t *testing.T, object []byte, expected []byte) {
e, err := unmarshalResource(t, expected)
if err != nil {
t.Fatalf("Failed to unmarshal expected Resource: %v", err)
}
result, err := ParseKubernetesObject(object)
if err != nil {
t.Fatalf("ParseKubernetesObject failed: %v")
}
// Since the object name gets created on the fly, we have to rejigger the returned object
// slightly to make sure the DeepEqual works as expected.
// First validate the name matches the expected format.
var i int
format := e.Name + "-%d"
count, err := fmt.Sscanf(result.Name, format, &i)
if err != nil || count != 1 {
t.Errorf("Name is not as expected, wanted of the form %s got %s", format, result.Name)
}
e.Name = result.Name
if !reflect.DeepEqual(result, e) {
t.Errorf("expected %+v but found %+v", e, result)
}
}
func TestSimple(t *testing.T) {
testConversion(t, []byte(rcInput), []byte(rcExpected))
testConversion(t, []byte(serviceInput), []byte(serviceExpected))
}

@ -19,7 +19,10 @@ import (
"github.com/kubernetes/deployment-manager/common"
)
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
var TemplateRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
// RE for Registry that does not support versions and can have multiple files without imports.
var PackageRegistryMatcher = regexp.MustCompile("github.com/(.*)/(.*)/(.*)")
// IsTemplate returns whether a given type is a template.
func IsTemplate(t string, imports []*common.ImportFile) bool {
@ -37,5 +40,15 @@ func IsTemplate(t string, imports []*common.ImportFile) bool {
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool {
return re.MatchString(t)
return TemplateRegistryMatcher.MatchString(t)
}
// IsGithubShortPackageType returns whether a given type is a type description in a short format to a github
// package repository type.
// For now, this means using github types:
// github.com/owner/repo/type
// for example:
// github.com/helm/charts/cassandra
func IsGithubShortPackageType(t string) bool {
return PackageRegistryMatcher.MatchString(t)
}

Loading…
Cancel
Save