msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

Add support for short github.com types. Refactor manager/manager/types to common/types and reuse them rather than redefine them.

Browse Source
pull/141/head
vaikas-google 9 years ago
parent 92371f1f4c
commit 92ec9a0fe1
25 changed files with 585 additions and 290 deletions
Show Stats Download Patch File Download Diff File

3
manager/manager/types.go → common/types.go
Unescape View File

@ -11,7 +11,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
package common
import (
"time"
@ -120,6 +120,7 @@ type Template struct {
// ImportFile describes a base64 encoded file imported by a Template.
type ImportFile struct {
Name string `json:"name,omitempty"`
Path string `json:"path",omitempty` // Actual URL for the file
Content string `json:"content"`
}

14
dm/dm.go
Unescape View File

@ -16,8 +16,8 @@ package main
import (
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/util"
@ -286,8 +286,8 @@ func isHttp(t string) bool {
return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://")
}
func loadTemplate(args []string) *expander.Template {
var template *expander.Template
func loadTemplate(args []string) *common.Template {
var template *common.Template
var err error
if len(args) < 2 {
fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied")
@ -329,7 +329,7 @@ func getRegistryType(fullType string) *registry.Type {
}
}
func buildTemplateFromType(t registry.Type) *expander.Template {
func buildTemplateFromType(t registry.Type) *common.Template {
downloadURL := getDownloadUrl(t)
props := make(map[string]interface{})
@ -355,7 +355,7 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
// Name the deployment after the type name.
name := fmt.Sprintf("%s:%s", t.Name, t.Version)
config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{
config := common.Configuration{Resources: []*common.Resource{&common.Resource{
Name: name,
Type: downloadURL,
Properties: props,
@ -366,14 +366,14 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
}
return &expander.Template{
return &common.Template{
Name: name,
Content: string(y),
// No imports, as this is a single type from repository.
}
}
func marshalTemplate(template *expander.Template) io.ReadCloser {
func marshalTemplate(template *common.Template) io.ReadCloser {
j, err := json.Marshal(template)
if err != nil {
log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)

31
expandybird/expander/expander.go
Unescape View File

@ -23,11 +23,12 @@ import (
"path/filepath"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
// Expander abstracts interactions with the expander and deployer services.
type Expander interface {
ExpandTemplate(template *Template) (string, error)
ExpandTemplate(template *common.Template) (string, error)
}
type expander struct {
@ -39,24 +40,10 @@ func NewExpander(binary string) Expander {
return &expander{binary}
}
// ImportFile describes a file that we import into our templates
// TODO: Encode the Content so that it doesn't get mangled.
type ImportFile struct {
Name string `json:"name,omitempty"`
Content string `json:"content"`
}
// A Template defines a single deployment.
type Template struct {
Name string `json:"name"`
Content string `json:"content"`
Imports []*ImportFile `json:"imports"`
}
// NewTemplateFromRootTemplate creates and returns a new template whose content
// and imported files are constructed from reading the root template, parsing out
// the imports section and reading the imports from there
func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) {
func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) {
templateDir := filepath.Dir(templateFileName)
content, err := ioutil.ReadFile(templateFileName)
if err != nil {
@ -85,14 +72,14 @@ func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) {
func NewTemplateFromFileNames(
templateFileName string,
importFileNames []string,
) (*Template, error) {
) (*common.Template, error) {
name := path.Base(templateFileName)
content, err := ioutil.ReadFile(templateFileName)
if err != nil {
return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName)
}
imports := []*ImportFile{}
imports := []*common.ImportFile{}
for _, importFileName := range importFileNames {
importFileData, err := ioutil.ReadFile(importFileName)
if err != nil {
@ -100,13 +87,13 @@ func NewTemplateFromFileNames(
}
imports = append(imports,
&ImportFile{
&common.ImportFile{
Name: path.Base(importFileName),
Content: string(importFileData),
})
}
return &Template{
return &common.Template{
Name: name,
Content: string(content),
Imports: imports,
@ -190,7 +177,7 @@ func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) {
// ExpandTemplate passes the given configuration to the expander and returns the
// expanded configuration as a string on success.
func (e *expander) ExpandTemplate(template *Template) (string, error) {
func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
if e.ExpansionBinary == "" {
message := fmt.Sprintf("expansion binary cannot be empty")
return "", fmt.Errorf("error expanding template %s: %s", template.Name, message)
@ -216,7 +203,7 @@ func (e *expander) ExpandTemplate(template *Template) (string, error) {
}
for _, imp := range template.Imports {
cmd.Args = append(cmd.Args, imp.Name, imp.Content)
cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content)
}
if err := cmd.Start(); err != nil {

4
expandybird/expander/expander_test.go
Unescape View File

@ -19,6 +19,8 @@ import (
"reflect"
"strings"
"testing"
"github.com/kubernetes/deployment-manager/common"
)
const invalidFileName = "afilethatdoesnotexist"
@ -36,7 +38,7 @@ type ExpanderTestCase struct {
ExpectedError string
}
func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *Template {
func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template {
template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames)
if err != nil {
t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err)

33
expandybird/expansion/expansion.py
Unescape View File

@ -136,7 +136,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
layout = {'name': resource['name'],
'type': resource['type']}
if IsTemplate(resource['type']) and resource['type'] in imports:
if resource['type'] in imports:
# A template resource, which contains sub-resources.
expanded_template = ExpandTemplate(resource, imports, env, validate_schema)
@ -183,11 +183,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'):
# If this resource doesn't have a name, we will report that error later
def IsTemplate(resource_type):
"""Returns whether a given resource type is a Template."""
return resource_type.endswith('.py') or resource_type.endswith('.jinja')
def ExpandTemplate(resource, imports, env, validate_schema=False):
"""Expands a template, calling expansion mechanism based on type.
@ -206,6 +201,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
ExpansionError: if there is any error occurred during expansion
"""
source_file = resource['type']
path = resource['type']
# Look for Template in imports.
if source_file not in imports:
@ -213,6 +209,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
source_file,
'Unable to find source file %s in imports.' % (source_file))
# source_file could be a short version of the template (say github short name)
# so we need to potentially map this into the fully resolvable name.
if 'path' in imports[source_file]:
if imports[source_file]['path']:
path = imports[source_file]['path']
resource['imports'] = imports
# Populate the additional environment variables.
@ -231,13 +233,13 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
except schema_validation.ValidationErrors as e:
raise ExpansionError(resource['name'], e.message)
if source_file.endswith('jinja'):
if path.endswith('jinja'):
expanded_template = ExpandJinja(
source_file, imports[source_file], resource, imports)
elif source_file.endswith('py'):
source_file, imports[source_file]['content'], resource, imports)
elif path.endswith('py'):
# This is a Python template.
expanded_template = ExpandPython(
imports[source_file], source_file, resource)
imports[source_file]['content'], source_file, resource)
else:
# The source file is not a jinja file or a python file.
# This in fact should never happen due to the IsTemplate check above.
@ -262,8 +264,8 @@ def ExpandJinja(file_name, source_template, resource, imports):
source_template: string, the content of jinja file to be render
resource: resource object, the resource that contains parameters to the
jinja file
imports: map from string to string, the map of imported files names
and contents
imports: map from string to map {name, path}, the map of imported files names
fully resolved path and contents
Returns:
The final expanded template
Raises:
@ -362,9 +364,10 @@ def main():
print >>sys.stderr, 'Invalid import definition at argv pos %d' % idx
sys.exit(1)
name = sys.argv[idx]
value = sys.argv[idx + 1]
imports[name] = value
idx += 2
path = sys.argv[idx + 1]
value = sys.argv[idx + 2]
imports[name] = {'content': value, 'path': path}
idx += 3
env = {}
env['deployment'] = os.environ['DEPLOYMENT_NAME']

2
expandybird/expansion/sandbox_loader.py
Unescape View File

@ -95,7 +95,7 @@ def process_imports(imports):
# Now build the hierarchical modules.
for k in imports.keys():
if imports[k].endswith('.jinja'):
if imports[k]['path'].endswith('.jinja'):
continue
# Normalize paths and trim .py extension, if any.
normalized = os.path.splitext(os.path.normpath(k))[0]

5
expandybird/service/service.go
Unescape View File

@ -15,6 +15,7 @@ package service
import (
"github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
"errors"
@ -40,7 +41,7 @@ func NewService(handler restful.RouteFunction) *Service {
webService.Produces(restful.MIME_JSON, restful.MIME_XML)
webService.Route(webService.POST("/expand").To(handler).
Doc("Expand a template.").
Reads(&expander.Template{}))
Reads(&common.Template{}))
return &Service{webService}
}
@ -60,7 +61,7 @@ func (s *Service) Register(container *restful.Container) {
func NewExpansionHandler(backend expander.Expander) restful.RouteFunction {
return func(req *restful.Request, resp *restful.Response) {
util.LogHandlerEntry("expandybird: expand", req.Request)
template := &expander.Template{}
template := &common.Template{}
if err := req.ReadEntity(&template); err != nil {
logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp)
return

3
expandybird/service/service_test.go
Unescape View File

@ -24,6 +24,7 @@ import (
"testing"
"github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util"
restful "github.com/emicklei/go-restful"
@ -180,7 +181,7 @@ type mockExpander struct {
// ExpandTemplate passes the given configuration to the expander and returns the
// expanded configuration as a string on success.
func (e *mockExpander) ExpandTemplate(template *expander.Template) (string, error) {
func (e *mockExpander) ExpandTemplate(template *common.Template) (string, error) {
switch template.Name {
case "InvalidFileName.yaml":
return "", fmt.Errorf("expansion error")

11
manager/deployments.go
Unescape View File

@ -22,6 +22,7 @@ import (
"log"
"net"
"net/http"
"net/url"
"os"
"strings"
@ -29,6 +30,7 @@ import (
"github.com/gorilla/mux"
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/manager/repository"
"github.com/kubernetes/deployment-manager/util"
)
@ -198,17 +200,16 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) {
func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
vars := mux.Vars(r)
variable, ok := vars[variable]
retVariable, ok := vars[variable]
if !ok {
e := fmt.Errorf("%s parameter not found in URL", variable)
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
return "", e
}
return variable, nil
return retVariable, nil
}
func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manager.Template {
func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
util.LogHandlerEntry(handler, r)
b := io.LimitReader(r.Body, *maxLength*1024)
y, err := ioutil.ReadAll(b)
@ -237,7 +238,7 @@ func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manage
return nil
}
t := &manager.Template{}
t := &common.Template{}
if err := json.Unmarshal(j, t); err != nil {
e := fmt.Errorf("%v\n%v", err, string(j))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

25
manager/manager/deployer.go
Unescape View File

@ -25,14 +25,15 @@ import (
"strings"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
)
// Deployer abstracts interactions with the expander and deployer services.
type Deployer interface {
GetConfiguration(cached *Configuration) (*Configuration, error)
CreateConfiguration(configuration *Configuration) (*Configuration, error)
DeleteConfiguration(configuration *Configuration) (*Configuration, error)
PutConfiguration(configuration *Configuration) (*Configuration, error)
GetConfiguration(cached *common.Configuration) (*common.Configuration, error)
CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error)
DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error)
PutConfiguration(configuration *common.Configuration) (*common.Configuration, error)
}
// NewDeployer returns a new initialized Deployer.
@ -54,9 +55,9 @@ type formatter func(err error) error
// GetConfiguration reads and returns the actual configuration
// of the resources described by a cached configuration.
func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, error) {
func (d *deployer) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) {
errors := &Error{}
actual := &Configuration{}
actual := &common.Configuration{}
for _, resource := range cached.Resources {
rtype := url.QueryEscape(resource.Type)
rname := url.QueryEscape(resource.Name)
@ -70,7 +71,7 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
}
if len(body) != 0 {
result := &Resource{Name: resource.Name, Type: resource.Type}
result := &common.Resource{Name: resource.Name, Type: resource.Type}
if err := yaml.Unmarshal(body, &result.Properties); err != nil {
return nil, fmt.Errorf("cannot get configuration for resource (%v)", err)
}
@ -88,22 +89,22 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
// CreateConfiguration deploys the set of resources described by a configuration and returns
// the Configuration with status for each resource filled in.
func (d *deployer) CreateConfiguration(configuration *Configuration) (*Configuration, error) {
func (d *deployer) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("POST", "create", configuration)
}
// DeleteConfiguration deletes the set of resources described by a configuration.
func (d *deployer) DeleteConfiguration(configuration *Configuration) (*Configuration, error) {
func (d *deployer) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("DELETE", "delete", configuration)
}
// PutConfiguration replaces the set of resources described by a configuration and returns
// the Configuration with status for each resource filled in.
func (d *deployer) PutConfiguration(configuration *Configuration) (*Configuration, error) {
func (d *deployer) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("PUT", "replace", configuration)
}
func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *Configuration) (*Configuration, error) {
func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *common.Configuration) (*common.Configuration, error) {
callback := func(e error) error {
return fmt.Errorf("cannot %s configuration: %s", operation, e)
}
@ -120,7 +121,7 @@ func (d *deployer) callServiceWithConfiguration(method, operation string, config
return nil, err
}
result := &Configuration{}
result := &common.Configuration{}
if len(resp) != 0 {
if err := yaml.Unmarshal(resp, &result); err != nil {
return nil, fmt.Errorf("cannot unmarshal response: (%v)", err)

9
manager/manager/deployer_test.go
Unescape View File

@ -24,6 +24,7 @@ import (
"testing"
"github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml"
)
@ -250,8 +251,8 @@ func TestPutConfiguration(t *testing.T) {
}
}
func getValidConfiguration(t *testing.T) *Configuration {
valid := &Configuration{}
func getValidConfiguration(t *testing.T) *common.Configuration {
valid := &common.Configuration{}
err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
if err != nil {
t.Errorf("cannot unmarshal test case data:%s\n", err)
@ -266,7 +267,7 @@ func deployerErrorHandler(w http.ResponseWriter, r *http.Request) {
}
func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
valid := &Configuration{}
valid := &common.Configuration{}
err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
if err != nil {
status := fmt.Sprintf("cannot unmarshal test case data:%s", err)
@ -282,7 +283,7 @@ func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
return
}
result := &Configuration{}
result := &common.Configuration{}
if err := yaml.Unmarshal(body, result); err != nil {
status := fmt.Sprintf("cannot unmarshal request body:%s", err)
http.Error(w, status, http.StatusInternalServerError)

27
manager/manager/expander.go
Unescape View File

@ -22,6 +22,7 @@ import (
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
)
const (
@ -31,13 +32,13 @@ const (
// ExpandedTemplate is the structure returned by the expansion service.
type ExpandedTemplate struct {
Config *Configuration `json:"config"`
Layout *Layout `json:"layout"`
Config *common.Configuration `json:"config"`
Layout *common.Layout `json:"layout"`
}
// Expander abstracts interactions with the expander and deployer services.
type Expander interface {
ExpandTemplate(t *Template) (*ExpandedTemplate, error)
ExpandTemplate(t *common.Template) (*ExpandedTemplate, error)
}
// NewExpander returns a new initialized Expander.
@ -54,7 +55,7 @@ func (e *expander) getBaseURL() string {
return fmt.Sprintf("%s/expand", e.expanderURL)
}
func expanderError(t *Template, err error) error {
func expanderError(t *common.Template, err error) error {
return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content)
}
@ -90,14 +91,14 @@ func expanderError(t *Template, err error) error {
// between the name#template key to exist in the layout given a particular choice of naming.
// In practice, it would be nearly impossible to hit, but consider including properties/name/type
// into a hash of sorts to make this robust...
func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*LayoutResource {
ret := map[string]*LayoutResource{}
func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource {
ret := map[string]*common.LayoutResource{}
toVisit := l.Resources
for len(toVisit) > 0 {
lr := toVisit[0]
nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type
if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type) {
if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) {
ret[nodeKey] = lr
} else if toReplace[nodeKey] != nil {
toReplace[nodeKey].Resources = lr.Resources
@ -112,20 +113,20 @@ func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*Lay
// ExpandTemplate expands the supplied template, and returns a configuration.
// It will also update the imports in the provided template if any were added
// during type resolution.
func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) {
// We have a fencepost problem here.
// 1. Start by trying to resolve any missing templates
// 2. Expand the configuration using all the of the imports available to us at this point
// 3. Expansion may yield additional templates, so we run the type resolution again
// 4. If type resolution resulted in new imports being available, return to 2.
config := &Configuration{}
config := &common.Configuration{}
if err := yaml.Unmarshal([]byte(t.Content), config); err != nil {
e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content)
return nil, e
}
var finalLayout *Layout
needResolve := map[string]*LayoutResource{}
var finalLayout *common.Layout
needResolve := map[string]*common.LayoutResource{}
// Start things off by attempting to resolve the templates in a first pass.
newImp, err := e.typeResolver.ResolveTypes(config, t.Imports)
@ -151,7 +152,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
if finalLayout == nil {
finalLayout = result.Layout
}
needResolve = walkLayout(result.Layout, needResolve)
needResolve = walkLayout(result.Layout, t.Imports, needResolve)
newImp, err = e.typeResolver.ResolveTypes(result.Config, t.Imports)
if err != nil {
@ -170,7 +171,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
}
}
func (e *expander) expandTemplate(t *Template) (*ExpandedTemplate, error) {
func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) {
j, err := json.Marshal(t)
if err != nil {
return nil, err

27
manager/manager/expander_test.go
Unescape View File

@ -24,16 +24,17 @@ import (
"testing"
"github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml"
)
type mockResolver struct {
responses [][]*ImportFile
responses [][]*common.ImportFile
t *testing.T
}
func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*ImportFile, error) {
func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
if len(r.responses) < 1 {
return nil, nil
}
@ -43,7 +44,7 @@ func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*Impor
return ret, nil
}
var validTemplateTestCaseData = Template{
var validTemplateTestCaseData = common.Template{
Name: "TestTemplate",
Content: string(validContentTestCaseData),
Imports: validImportFilesTestCaseData,
@ -59,11 +60,19 @@ resources:
test-property: test-value
`)
var validImportFilesTestCaseData = []*ImportFile{
&ImportFile{
var validImportFilesTestCaseData = []*common.ImportFile{
&common.ImportFile{
Name: "test-type.py",
Content: "test-type.py validTemplateTestCaseData content",
},
&common.ImportFile{
Name: "test.py",
Content: "test.py validTemplateTestCaseData content",
},
&common.ImportFile{
Name: "test2.py",
Content: "test2.py validTemplateTestCaseData content",
},
}
var validConfigTestCaseData = []byte(`
@ -210,7 +219,7 @@ layout:
test: test
`
var roundTripTemplate = Template{
var roundTripTemplate = common.Template{
Name: "TestTemplate",
Content: roundTripContent,
Imports: nil,
@ -249,9 +258,9 @@ func TestExpandTemplate(t *testing.T) {
"expect success for ExpandTemplate with two expansions",
"",
roundTripHandler,
&mockResolver{[][]*ImportFile{
&mockResolver{[][]*common.ImportFile{
{},
{&ImportFile{Name: "test.py"}},
{&common.ImportFile{Name: "test.py"}},
}, t},
roundTripResponse,
},
@ -334,7 +343,7 @@ func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) {
return
}
template := &Template{}
template := &common.Template{}
if err := json.Unmarshal(body, template); err != nil {
status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body)
http.Error(w, status, http.StatusInternalServerError)

72
manager/manager/manager.go
Unescape View File

@ -17,35 +17,37 @@ import (
"fmt"
"log"
"time"
"github.com/kubernetes/deployment-manager/common"
)
// Manager manages a persistent set of Deployments.
type Manager interface {
ListDeployments() ([]Deployment, error)
GetDeployment(name string) (*Deployment, error)
CreateDeployment(t *Template) (*Deployment, error)
DeleteDeployment(name string, forget bool) (*Deployment, error)
PutDeployment(name string, t *Template) (*Deployment, error)
ListManifests(deploymentName string) (map[string]*Manifest, error)
GetManifest(deploymentName string, manifest string) (*Manifest, error)
Expand(t *Template) (*Manifest, error)
ListDeployments() ([]common.Deployment, error)
GetDeployment(name string) (*common.Deployment, error)
CreateDeployment(t *common.Template) (*common.Deployment, error)
DeleteDeployment(name string, forget bool) (*common.Deployment, error)
PutDeployment(name string, t *common.Template) (*common.Deployment, error)
ListManifests(deploymentName string) (map[string]*common.Manifest, error)
GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
Expand(t *common.Template) (*common.Manifest, error)
ListTypes() []string
ListInstances(typeName string) []*TypeInstance
ListInstances(typeName string) []*common.TypeInstance
}
type manager struct {
expander Expander
deployer Deployer
repository Repository
repository common.Repository
}
// NewManager returns a new initialized Manager.
func NewManager(expander Expander, deployer Deployer, repository Repository) Manager {
func NewManager(expander Expander, deployer Deployer, repository common.Repository) Manager {
return &manager{expander, deployer, repository}
}
// ListDeployments returns the list of deployments
func (m *manager) ListDeployments() ([]Deployment, error) {
func (m *manager) ListDeployments() ([]common.Deployment, error) {
l, err := m.repository.ListDeployments()
if err != nil {
return nil, err
@ -55,7 +57,7 @@ func (m *manager) ListDeployments() ([]Deployment, error) {
// GetDeployment retrieves the configuration stored for a given deployment
// as well as the current configuration from the cluster.
func (m *manager) GetDeployment(name string) (*Deployment, error) {
func (m *manager) GetDeployment(name string) (*common.Deployment, error) {
d, err := m.repository.GetDeployment(name)
if err != nil {
return nil, err
@ -66,7 +68,7 @@ func (m *manager) GetDeployment(name string) (*Deployment, error) {
// ListManifests retrieves the manifests for a given deployment
// of each of the deployments in the repository and returns the deployments.
func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, error) {
func (m *manager) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
l, err := m.repository.ListManifests(deploymentName)
if err != nil {
return nil, err
@ -76,7 +78,7 @@ func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, er
}
// GetManifest retrieves the specified manifest for a given deployment
func (m *manager) GetManifest(deploymentName string, manifestName string) (*Manifest, error) {
func (m *manager) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
d, err := m.repository.GetManifest(deploymentName, manifestName)
if err != nil {
return nil, err
@ -88,7 +90,7 @@ func (m *manager) GetManifest(deploymentName string, manifestName string) (*Mani
// CreateDeployment expands the supplied template, creates the resulting
// configuration in the cluster, creates a new deployment that tracks it,
// and stores the deployment in the repository. Returns the deployment.
func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
func (m *manager) CreateDeployment(t *common.Template) (*common.Deployment, error) {
log.Printf("Creating deployment: %s", t.Name)
_, err := m.repository.CreateDeployment(t.Name)
if err != nil {
@ -99,7 +101,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
manifest, err := m.createManifest(t)
if err != nil {
log.Printf("Manifest creation failed: %v", err)
m.repository.SetDeploymentStatus(t.Name, FailedStatus)
m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
return nil, err
}
@ -107,7 +109,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
if createErr != nil {
// Deployment failed, mark as failed
log.Printf("CreateConfiguration failed: %v", err)
m.repository.SetDeploymentStatus(t.Name, FailedStatus)
m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
// If we failed before being able to create some of the resources, then
// return the failure as such. Otherwise, we're going to add the manifest
// and hence resource specific errors down below.
@ -115,7 +117,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
return nil, createErr
}
} else {
m.repository.SetDeploymentStatus(t.Name, DeployedStatus)
m.repository.SetDeploymentStatus(t.Name, common.DeployedStatus)
}
// Update the manifest with the actual state of the reified resources
@ -123,7 +125,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
aErr := m.repository.AddManifest(t.Name, manifest)
if aErr != nil {
log.Printf("AddManifest failed %v", aErr)
m.repository.SetDeploymentStatus(t.Name, FailedStatus)
m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
// If there's an earlier error, return that instead since it contains
// more applicable error message. Adding manifest failure is more akin
// to a check fail (either deployment doesn't exist, or a manifest with the same
@ -141,14 +143,14 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
return m.repository.GetValidDeployment(t.Name)
}
func (m *manager) createManifest(t *Template) (*Manifest, error) {
func (m *manager) createManifest(t *common.Template) (*common.Manifest, error) {
et, err := m.expander.ExpandTemplate(t)
if err != nil {
log.Printf("Expansion failed %v", err)
return nil, err
}
return &Manifest{
return &common.Manifest{
Name: generateManifestName(),
Deployment: t.Name,
InputConfig: t,
@ -157,10 +159,10 @@ func (m *manager) createManifest(t *Template) (*Manifest, error) {
}, nil
}
func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *Layout) {
func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *common.Layout) {
m.repository.ClearTypeInstances(deploymentName)
instances := make(map[string][]*TypeInstance)
instances := make(map[string][]*common.TypeInstance)
for i, r := range layout.Resources {
addTypeInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i))
}
@ -168,9 +170,9 @@ func (m *manager) addTypeInstances(deploymentName string, manifestName string, l
m.repository.SetTypeInstances(deploymentName, instances)
}
func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource, deploymentName string, manifestName string, jsonPath string) {
func addTypeInstances(instances *map[string][]*common.TypeInstance, r *common.LayoutResource, deploymentName string, manifestName string, jsonPath string) {
// Add this resource.
inst := &TypeInstance{
inst := &common.TypeInstance{
Name: r.Name,
Type: r.Type,
Deployment: deploymentName,
@ -189,7 +191,7 @@ func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource,
// the supplied identifier from the cluster.repository. If forget is true, then
// the deployment is removed from the repository. Otherwise, it is marked
// as deleted and retained.
func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error) {
func (m *manager) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
log.Printf("Deleting deployment: %s", name)
d, err := m.repository.GetValidDeployment(name)
if err != nil {
@ -210,7 +212,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error
}
// Create an empty manifest since resources have been deleted.
err = m.repository.AddManifest(name, &Manifest{Deployment: name, Name: generateManifestName()})
err = m.repository.AddManifest(name, &common.Manifest{Deployment: name, Name: generateManifestName()})
if err != nil {
log.Printf("Failed to add empty manifest")
return nil, err
@ -230,7 +232,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error
// PutDeployment replaces the configuration of the deployment with
// the supplied identifier in the cluster, and returns the deployment.
func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) {
func (m *manager) PutDeployment(name string, t *common.Template) (*common.Deployment, error) {
_, err := m.repository.GetValidDeployment(name)
if err != nil {
return nil, err
@ -239,20 +241,20 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) {
manifest, err := m.createManifest(t)
if err != nil {
log.Printf("Manifest creation failed: %v", err)
m.repository.SetDeploymentStatus(name, FailedStatus)
m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err
}
actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig)
if err != nil {
m.repository.SetDeploymentStatus(name, FailedStatus)
m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err
}
manifest.ExpandedConfig = actualConfig
err = m.repository.AddManifest(t.Name, manifest)
if err != nil {
m.repository.SetDeploymentStatus(name, FailedStatus)
m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err
}
@ -262,14 +264,14 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) {
return m.repository.GetValidDeployment(t.Name)
}
func (m *manager) Expand(t *Template) (*Manifest, error) {
func (m *manager) Expand(t *common.Template) (*common.Manifest, error) {
et, err := m.expander.ExpandTemplate(t)
if err != nil {
log.Printf("Expansion failed %v", err)
return nil, err
}
return &Manifest{
return &common.Manifest{
ExpandedConfig: et.Config,
Layout: et.Layout,
}, nil
@ -279,7 +281,7 @@ func (m *manager) ListTypes() []string {
return m.repository.ListTypes()
}
func (m *manager) ListInstances(typeName string) []*TypeInstance {
func (m *manager) ListInstances(typeName string) []*common.TypeInstance {
return m.repository.GetTypeInstances(typeName)
}

90
manager/manager/manager_test.go
Unescape View File

@ -18,25 +18,27 @@ import (
"reflect"
"strings"
"testing"
"github.com/kubernetes/deployment-manager/common"
)
var template = Template{Name: "test", Content: "test"}
var template = common.Template{Name: "test", Content: "test"}
var layout = Layout{
Resources: []*LayoutResource{&LayoutResource{Resource: Resource{Name: "test", Type: "test"}}},
var layout = common.Layout{
Resources: []*common.LayoutResource{&common.LayoutResource{Resource: common.Resource{Name: "test", Type: "test"}}},
}
var configuration = Configuration{
Resources: []*Resource{&Resource{Name: "test", Type: "test"}},
var configuration = common.Configuration{
Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test"}},
}
var resourcesWithSuccessState = Configuration{
Resources: []*Resource{&Resource{Name: "test", Type: "test", State: &ResourceState{Status: Created}}},
var resourcesWithSuccessState = common.Configuration{
Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test", State: &common.ResourceState{Status: common.Created}}},
}
var resourcesWithFailureState = Configuration{
Resources: []*Resource{&Resource{
var resourcesWithFailureState = common.Configuration{
Resources: []*common.Resource{&common.Resource{
Name: "test",
Type: "test",
State: &ResourceState{
Status: Failed,
State: &common.ResourceState{
Status: common.Failed,
Errors: []string{"test induced error"},
},
}},
@ -49,14 +51,14 @@ var expandedConfig = ExpandedTemplate{
var deploymentName = "deployment"
var manifestName = "manifest-2"
var manifest = Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout}
var manifestMap = map[string]*Manifest{manifest.Name: &manifest}
var manifest = common.Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout}
var manifestMap = map[string]*common.Manifest{manifest.Name: &manifest}
var deployment = Deployment{
var deployment = common.Deployment{
Name: "test",
}
var deploymentList = []Deployment{deployment, {Name: "test2"}}
var deploymentList = []common.Deployment{deployment, {Name: "test2"}}
var typeInstMap = map[string][]string{"test": []string{"test"}}
@ -64,7 +66,7 @@ var errTest = errors.New("test")
type expanderStub struct{}
func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
func (expander *expanderStub) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) {
if reflect.DeepEqual(*t, template) {
return &expandedConfig, nil
}
@ -74,17 +76,17 @@ func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, er
type deployerStub struct {
FailCreate bool
Created []*Configuration
Created []*common.Configuration
FailDelete bool
Deleted []*Configuration
Deleted []*common.Configuration
FailCreateResource bool
}
func (deployer *deployerStub) reset() {
deployer.FailCreate = false
deployer.Created = make([]*Configuration, 0)
deployer.Created = make([]*common.Configuration, 0)
deployer.FailDelete = false
deployer.Deleted = make([]*Configuration, 0)
deployer.Deleted = make([]*common.Configuration, 0)
deployer.FailCreateResource = false
}
@ -93,11 +95,11 @@ func newDeployerStub() *deployerStub {
return ret
}
func (deployer *deployerStub) GetConfiguration(cached *Configuration) (*Configuration, error) {
func (deployer *deployerStub) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) {
return nil, nil
}
func (deployer *deployerStub) CreateConfiguration(configuration *Configuration) (*Configuration, error) {
func (deployer *deployerStub) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
if deployer.FailCreate {
return nil, errTest
}
@ -109,7 +111,7 @@ func (deployer *deployerStub) CreateConfiguration(configuration *Configuration)
return &resourcesWithSuccessState, nil
}
func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration) (*Configuration, error) {
func (deployer *deployerStub) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
if deployer.FailDelete {
return nil, errTest
}
@ -117,34 +119,34 @@ func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration)
return nil, nil
}
func (deployer *deployerStub) PutConfiguration(configuration *Configuration) (*Configuration, error) {
func (deployer *deployerStub) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return nil, nil
}
type repositoryStub struct {
FailListDeployments bool
Created []string
ManifestAdd map[string]*Manifest
ManifestAdd map[string]*common.Manifest
Deleted []string
GetValid []string
TypeInstances map[string][]string
TypeInstancesCleared bool
GetTypeInstancesCalled bool
ListTypesCalled bool
DeploymentStatuses []DeploymentStatus
DeploymentStatuses []common.DeploymentStatus
}
func (repository *repositoryStub) reset() {
repository.FailListDeployments = false
repository.Created = make([]string, 0)
repository.ManifestAdd = make(map[string]*Manifest)
repository.ManifestAdd = make(map[string]*common.Manifest)
repository.Deleted = make([]string, 0)
repository.GetValid = make([]string, 0)
repository.TypeInstances = make(map[string][]string)
repository.TypeInstancesCleared = false
repository.GetTypeInstancesCalled = false
repository.ListTypesCalled = false
repository.DeploymentStatuses = make([]DeploymentStatus, 0)
repository.DeploymentStatuses = make([]common.DeploymentStatus, 0)
}
func newRepositoryStub() *repositoryStub {
@ -152,14 +154,14 @@ func newRepositoryStub() *repositoryStub {
return ret
}
func (repository *repositoryStub) ListDeployments() ([]Deployment, error) {
func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error) {
if repository.FailListDeployments {
return deploymentList, errTest
}
return deploymentList, nil
}
func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) {
func (repository *repositoryStub) GetDeployment(d string) (*common.Deployment, error) {
if d == deploymentName {
return &deployment, nil
}
@ -167,32 +169,32 @@ func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) {
return nil, errTest
}
func (repository *repositoryStub) GetValidDeployment(d string) (*Deployment, error) {
func (repository *repositoryStub) GetValidDeployment(d string) (*common.Deployment, error) {
repository.GetValid = append(repository.GetValid, d)
return &deployment, nil
}
func (repository *repositoryStub) SetDeploymentStatus(name string, status DeploymentStatus) error {
func (repository *repositoryStub) SetDeploymentStatus(name string, status common.DeploymentStatus) error {
repository.DeploymentStatuses = append(repository.DeploymentStatuses, status)
return nil
}
func (repository *repositoryStub) CreateDeployment(d string) (*Deployment, error) {
func (repository *repositoryStub) CreateDeployment(d string) (*common.Deployment, error) {
repository.Created = append(repository.Created, d)
return &deployment, nil
}
func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*Deployment, error) {
func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*common.Deployment, error) {
repository.Deleted = append(repository.Deleted, d)
return &deployment, nil
}
func (repository *repositoryStub) AddManifest(d string, manifest *Manifest) error {
func (repository *repositoryStub) AddManifest(d string, manifest *common.Manifest) error {
repository.ManifestAdd[d] = manifest
return nil
}
func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error) {
func (repository *repositoryStub) GetLatestManifest(d string) (*common.Manifest, error) {
if d == deploymentName {
return repository.ManifestAdd[d], nil
}
@ -200,7 +202,7 @@ func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error)
return nil, errTest
}
func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest, error) {
func (repository *repositoryStub) ListManifests(d string) (map[string]*common.Manifest, error) {
if d == deploymentName {
return manifestMap, nil
}
@ -208,7 +210,7 @@ func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest,
return nil, errTest
}
func (repository *repositoryStub) GetManifest(d string, m string) (*Manifest, error) {
func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manifest, error) {
if d == deploymentName && m == manifestName {
return &manifest, nil
}
@ -221,16 +223,16 @@ func (r *repositoryStub) ListTypes() []string {
return []string{}
}
func (r *repositoryStub) GetTypeInstances(t string) []*TypeInstance {
func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
r.GetTypeInstancesCalled = true
return []*TypeInstance{}
return []*common.TypeInstance{}
}
func (r *repositoryStub) ClearTypeInstances(d string) {
r.TypeInstancesCleared = true
}
func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*TypeInstance) {
func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
for k, _ := range is {
r.TypeInstances[d] = append(r.TypeInstances[d], k)
}
@ -326,7 +328,7 @@ func TestCreateDeployment(t *testing.T) {
testDeployer.Created[0], configuration)
}
if testRepository.DeploymentStatuses[0] != DeployedStatus {
if testRepository.DeploymentStatuses[0] != common.DeployedStatus {
t.Fatal("CreateDeployment success did not mark deployment as deployed")
}
@ -355,7 +357,7 @@ func TestCreateDeploymentCreationFailure(t *testing.T) {
testRepository.Created[0])
}
if testRepository.DeploymentStatuses[0] != FailedStatus {
if testRepository.DeploymentStatuses[0] != common.FailedStatus {
t.Fatal("CreateDeployment failure did not mark deployment as failed")
}
@ -385,7 +387,7 @@ func TestCreateDeploymentCreationResourceFailure(t *testing.T) {
testRepository.Created[0])
}
if testRepository.DeploymentStatuses[0] != FailedStatus {
if testRepository.DeploymentStatuses[0] != common.FailedStatus {
t.Fatal("CreateDeployment failure did not mark deployment as failed")
}

106
manager/manager/typeresolver.go
Unescape View File

@ -16,8 +16,11 @@ package manager
import (
"fmt"
"net/http"
"regexp"
"time"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/util"
"github.com/ghodss/yaml"
@ -31,12 +34,14 @@ const (
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface {
ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error)
ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
}
type typeResolver struct {
getter util.HTTPClient
maxUrls int
re *regexp.Regexp
rp registry.RegistryProvider
}
// NewTypeResolver returns a new initialized TypeResolver.
@ -48,10 +53,12 @@ func NewTypeResolver() TypeResolver {
client.Timeout = timeout
ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
ret.maxUrls = maxURLImports
ret.re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
ret.rp = &registry.DefaultRegistryProvider{}
return ret
}
func resolverError(c *Configuration, err error) error {
func resolverError(c *common.Configuration, err error) error {
return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n",
c, err)
}
@ -78,21 +85,23 @@ func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content str
// resolved type definitions in t.ImportFiles. Types can be either
// primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote
// (i.e., described by a URL that must be fetched to resolve the type).
func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) {
func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) {
existing := map[string]bool{}
for _, v := range imports {
existing[v.Name] = true
}
fetched := map[string][]*ImportFile{}
fetched := map[string][]*common.ImportFile{}
toFetch := make([]string, 0, tr.maxUrls)
for _, r := range config.Resources {
// Only fetch HTTP URLs that we haven't already imported.
if util.IsHttpUrl(r.Type) && !existing[r.Type] {
toFetch = append(toFetch, r.Type)
fetched[r.Type] = append(fetched[r.Type], &ImportFile{Name: r.Type})
// Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
u, err := tr.MapFetchableURL(r.Type)
if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
}
if len(u) > 0 && !existing[r.Type] {
toFetch = append(toFetch, u)
fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u})
// Add to existing map so it is not fetched multiple times.
existing[r.Type] = true
}
@ -100,13 +109,14 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
count := 0
for len(toFetch) > 0 {
//1. Fetch import URL. Exit if no URLs left
//2. Check/handle HTTP status
//3. Store results in all ImportFiles from that URL
//4. Check for the optional schema file at import URL + .schema
//5. Repeat 2,3 for schema file
//6. Add each schema import to fetch if not already done
//7. Mark URL done. Return to 1.
//1. If short github URL, resolve to a download URL
//2. Fetch import URL. Exit if no URLs left
//3. Check/handle HTTP status
//4. Store results in all ImportFiles from that URL
//5. Check for the optional schema file at import URL + .schema
//6. Repeat 2,3 for schema file
//7. Add each schema import to fetch if not already done
//8. Mark URL done. Return to 1.
if count >= tr.maxUrls {
return nil, resolverError(config,
fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
@ -129,32 +139,41 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
}
if sch != "" {
var s Schema
var s common.Schema
if err := yaml.Unmarshal([]byte(sch), &s); err != nil {
return nil, resolverError(config, err)
}
// Here we handle any nested imports in the schema we've just fetched.
for _, v := range s.Imports {
i := &ImportFile{Name: v.Name}
i := &common.ImportFile{Name: v.Name}
var existingSchema string
if len(fetched[v.Path]) == 0 {
u, conversionErr := tr.MapFetchableURL(v.Path)
if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
}
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema.
if len(u) == 0 {
u = v.Path
}
if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, v.Path)
toFetch = append(toFetch, u)
} else {
// If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and
// record those contents for re-use as well.
if fetched[v.Path][0].Content != "" {
i.Content = fetched[v.Path][0].Content
if len(fetched[v.Path+schemaSuffix]) > 0 {
existingSchema = fetched[v.Path+schemaSuffix][0].Content
if fetched[u][0].Content != "" {
i.Content = fetched[u][0].Content
if len(fetched[u+schemaSuffix]) > 0 {
existingSchema = fetched[u+schemaSuffix][0].Content
}
}
}
fetched[v.Path] = append(fetched[v.Path], i)
fetched[u] = append(fetched[u], i)
if existingSchema != "" {
fetched[v.Path+schemaSuffix] = append(fetched[v.Path+schemaSuffix],
&ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
&common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
}
}
@ -162,7 +181,7 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
for _, i := range fetched[url] {
schemaImportName := i.Name + schemaSuffix
fetched[schemaURL] = append(fetched[schemaURL],
&ImportFile{Name: schemaImportName, Content: sch})
&common.ImportFile{Name: schemaImportName, Content: sch})
}
}
@ -170,10 +189,37 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
toFetch = toFetch[1:]
}
ret := []*ImportFile{}
ret := []*common.ImportFile{}
for _, v := range fetched {
ret = append(ret, v...)
}
return ret, nil
}
// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
// example) will return empty string.
func (tr *typeResolver) MapFetchableURL(t string) (string, error) {
if util.IsGithubShortType(t) {
return tr.ShortTypeToDownloadURL(t)
} else if util.IsHttpUrl(t) {
return t, nil
}
return "", nil
}
// ShortTypeToDownloadURL converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) {
m := tr.re.FindStringSubmatch(template)
if len(m) != 6 {
return "", fmt.Errorf("Failed to parse short github url: %s", template)
}
r := tr.rp.GetGithubRegistry(m[1], m[2])
t := registry.Type{m[3], m[4], m[5]}
return r.GetURL(t)
}

186
manager/manager/typeresolver_test.go
Unescape View File

@ -15,14 +15,20 @@ package manager
import (
"errors"
"fmt"
"net/http"
"reflect"
"regexp"
"strings"
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/common"
)
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
type responseAndError struct {
err error
code int
@ -30,12 +36,13 @@ type responseAndError struct {
}
type resolverTestCase struct {
config string
imports []*ImportFile
responses map[string]responseAndError
urlcount int
expectedErr error
importOut []*ImportFile
config string
imports []*common.ImportFile
responses map[string]responseAndError
urlcount int
expectedErr error
importOut []*common.ImportFile
registryProvider registry.RegistryProvider
}
type testGetter struct {
@ -51,11 +58,68 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) {
return ret.resp, ret.code, ret.err
}
type urlAndError struct {
u string
e error
}
type testRegistryProvider struct {
owner string
repo string
r map[string]registry.Registry
}
func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
r := make(map[string]registry.Registry)
r[owner+repository] = &testGithubRegistry{tests, count}
return &testRegistryProvider{owner, repository, r}
}
func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry {
return trp.r[owner+repository]
}
type testGithubRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return ret.u, ret.e
}
func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
r := &typeResolver{
re: re,
rp: c.registryProvider,
}
for in, expected := range tests {
actual, err := r.ShortTypeToDownloadURL(in)
if actual != expected.u {
t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual)
}
if err != expected.e {
t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
}
}
}
func testDriver(c resolverTestCase, t *testing.T) {
g := &testGetter{test: t, responses: c.responses}
r := &typeResolver{getter: g, maxUrls: 5}
r := &typeResolver{
getter: g,
maxUrls: 5,
re: re,
rp: c.registryProvider,
}
conf := &Configuration{}
conf := &common.Configuration{}
dataErr := yaml.Unmarshal([]byte(c.config), conf)
if dataErr != nil {
panic("bad test data")
@ -73,8 +137,8 @@ func testDriver(c resolverTestCase, t *testing.T) {
t.Errorf("Expected error %s but found %s", c.expectedErr, err)
}
resultImport := map[ImportFile]bool{}
expectedImport := map[ImportFile]bool{}
resultImport := map[common.ImportFile]bool{}
expectedImport := map[common.ImportFile]bool{}
for _, i := range result {
resultImport[*i] = true
}
@ -106,7 +170,7 @@ resources:
`
func TestIncludedImport(t *testing.T) {
imports := []*ImportFile{&ImportFile{Name: "foo.py"}}
imports := []*common.ImportFile{&common.ImportFile{Name: "foo.py"}}
test := resolverTestCase{
config: includeImport,
imports: imports,
@ -121,7 +185,7 @@ resources:
`
func TestSingleUrl(t *testing.T) {
finalImports := []*ImportFile{&ImportFile{Name: "http://my-fake-url", Content: "my-content"}}
finalImports := []*common.ImportFile{&common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}}
responses := map[string]responseAndError{
"http://my-fake-url": responseAndError{nil, http.StatusOK, "my-content"},
@ -158,10 +222,10 @@ imports:
`
func TestSingleUrlWithSchema(t *testing.T) {
finalImports := []*ImportFile{
&ImportFile{Name: "http://my-fake-url", Content: "my-content"},
&ImportFile{Name: "schema-import", Content: "schema-import"},
&ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
finalImports := []*common.ImportFile{
&common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
&common.ImportFile{Name: "schema-import", Content: "schema-import"},
&common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
}
responses := map[string]responseAndError{
@ -236,13 +300,13 @@ imports:
`
func TestSharedImport(t *testing.T) {
finalImports := []*ImportFile{
&ImportFile{Name: "http://my-fake-url", Content: "my-content"},
&ImportFile{Name: "http://my-fake-url1", Content: "my-content-1"},
&ImportFile{Name: "schema-import", Content: "schema-import"},
&ImportFile{Name: "schema-import-1", Content: "schema-import"},
&ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
&ImportFile{Name: "http://my-fake-url1.schema", Content: schema2},
finalImports := []*common.ImportFile{
&common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
&common.ImportFile{Name: "http://my-fake-url1", Path: "http://my-fake-url1", Content: "my-content-1"},
&common.ImportFile{Name: "schema-import", Content: "schema-import"},
&common.ImportFile{Name: "schema-import-1", Content: "schema-import"},
&common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
&common.ImportFile{Name: "http://my-fake-url1.schema", Content: schema2},
}
responses := map[string]responseAndError{
@ -262,3 +326,79 @@ func TestSharedImport(t *testing.T) {
}
testDriver(test, t)
}
func TestShortGithubUrlMapping(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
}
func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
}
var templateShortGithubTemplate = `
resources:
- name: foo
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v1
- name: foo1
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
`
func TestShortGithubUrl(t *testing.T) {
finalImports := []*common.ImportFile{
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py",
Content: "my-content"},
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py",
Content: "my-content-2"},
}
responses := map[string]responseAndError{
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content-2"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
}
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
}
test := resolverTestCase{
config: templateShortGithubTemplate,
importOut: finalImports,
urlcount: 4,
responses: responses,
registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
}
testDriver(test, t)
}

60
manager/repository/repository.go
Unescape View File

@ -22,37 +22,37 @@ import (
"sync"
"time"
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
)
// deploymentTypeInstanceMap stores type instances mapped by deployment name.
// This allows for simple updating and deleting of per-deployment instances
// when deployments are created/updated/deleted.
type deploymentTypeInstanceMap map[string][]*manager.TypeInstance
type deploymentTypeInstanceMap map[string][]*common.TypeInstance
type typeInstanceMap map[string]deploymentTypeInstanceMap
type mapBasedRepository struct {
sync.RWMutex
deployments map[string]manager.Deployment
manifests map[string]map[string]*manager.Manifest
deployments map[string]common.Deployment
manifests map[string]map[string]*common.Manifest
instances typeInstanceMap
}
// NewMapBasedRepository returns a new map based repository.
func NewMapBasedRepository() manager.Repository {
func NewMapBasedRepository() common.Repository {
return &mapBasedRepository{
deployments: make(map[string]manager.Deployment, 0),
manifests: make(map[string]map[string]*manager.Manifest, 0),
deployments: make(map[string]common.Deployment, 0),
manifests: make(map[string]map[string]*common.Manifest, 0),
instances: typeInstanceMap{},
}
}
// ListDeployments returns of all of the deployments in the repository.
func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) {
func (r *mapBasedRepository) ListDeployments() ([]common.Deployment, error) {
r.RLock()
defer r.RUnlock()
l := []manager.Deployment{}
l := []common.Deployment{}
for _, deployment := range r.deployments {
l = append(l, deployment)
}
@ -62,7 +62,7 @@ func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) {
// GetDeployment returns the deployment with the supplied name.
// If the deployment is not found, it returns an error.
func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, error) {
func (r *mapBasedRepository) GetDeployment(name string) (*common.Deployment, error) {
d, ok := r.deployments[name]
if !ok {
return nil, fmt.Errorf("deployment %s not found", name)
@ -72,13 +72,13 @@ func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, er
// GetValidDeployment returns the deployment with the supplied name.
// If the deployment is not found or marked as deleted, it returns an error.
func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deployment, error) {
func (r *mapBasedRepository) GetValidDeployment(name string) (*common.Deployment, error) {
d, err := r.GetDeployment(name)
if err != nil {
return nil, err
}
if d.Status == manager.DeletedStatus {
if d.Status == common.DeletedStatus {
return nil, fmt.Errorf("deployment %s is deleted", name)
}
@ -86,7 +86,7 @@ func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deploymen
}
// SetDeploymentStatus sets the DeploymentStatus of the deployment and updates ModifiedAt
func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.DeploymentStatus) error {
func (r *mapBasedRepository) SetDeploymentStatus(name string, status common.DeploymentStatus) error {
return func() error {
r.Lock()
defer r.Unlock()
@ -104,8 +104,8 @@ func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.Dep
}
// CreateDeployment creates a new deployment and stores it in the repository.
func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, error) {
d, err := func() (*manager.Deployment, error) {
func (r *mapBasedRepository) CreateDeployment(name string) (*common.Deployment, error) {
d, err := func() (*common.Deployment, error) {
r.Lock()
defer r.Unlock()
@ -114,8 +114,8 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
return nil, fmt.Errorf("Deployment %s already exists", name)
}
d := manager.NewDeployment(name)
d.Status = manager.CreatedStatus
d := common.NewDeployment(name)
d.Status = common.CreatedStatus
d.DeployedAt = time.Now()
r.deployments[name] = *d
return d, nil
@ -129,7 +129,7 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
return d, nil
}
func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manager.Manifest) error {
func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *common.Manifest) error {
err := func() error {
r.Lock()
defer r.Unlock()
@ -168,8 +168,8 @@ func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manage
// DeleteDeployment deletes the deployment with the supplied name.
// If forget is true, then the deployment is removed from the repository.
// Otherwise, it is marked as deleted and retained.
func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manager.Deployment, error) {
d, err := func() (*manager.Deployment, error) {
func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
d, err := func() (*common.Deployment, error) {
r.Lock()
defer r.Unlock()
@ -180,7 +180,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
if !forget {
d.DeletedAt = time.Now()
d.Status = manager.DeletedStatus
d.Status = common.DeletedStatus
r.deployments[name] = *d
} else {
delete(r.deployments, name)
@ -199,7 +199,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
return d, nil
}
func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*manager.Manifest, error) {
func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
r.Lock()
defer r.Unlock()
@ -211,17 +211,17 @@ func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*m
return r.listManifestsForDeployment(deploymentName)
}
func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*manager.Manifest, error) {
func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*common.Manifest, error) {
l, ok := r.manifests[deploymentName]
if !ok {
l = make(map[string]*manager.Manifest, 0)
l = make(map[string]*common.Manifest, 0)
r.manifests[deploymentName] = l
}
return l, nil
}
func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*manager.Manifest, error) {
func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
r.Lock()
defer r.Unlock()
@ -233,7 +233,7 @@ func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName str
return r.getManifestForDeployment(deploymentName, manifestName)
}
func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*manager.Manifest, error) {
func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*common.Manifest, error) {
l, err := r.listManifestsForDeployment(deploymentName)
if err != nil {
return nil, err
@ -249,7 +249,7 @@ func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, man
// GetLatestManifest returns the latest manifest for a given deployment,
// which by definition is the manifest with the largest time stamp.
func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*manager.Manifest, error) {
func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*common.Manifest, error) {
r.Lock()
defer r.Unlock()
@ -273,11 +273,11 @@ func (r *mapBasedRepository) ListTypes() []string {
// GetTypeInstances returns all instances of a given type. If type is empty,
// returns all instances for all types.
func (r *mapBasedRepository) GetTypeInstances(typeName string) []*manager.TypeInstance {
func (r *mapBasedRepository) GetTypeInstances(typeName string) []*common.TypeInstance {
r.Lock()
defer r.Unlock()
var instances []*manager.TypeInstance
var instances []*common.TypeInstance
for t, dInstMap := range r.instances {
if t == typeName || typeName == "all" {
for _, i := range dInstMap {
@ -307,7 +307,7 @@ func (r *mapBasedRepository) ClearTypeInstances(deploymentName string) {
//
// To clear the current set of instances first, caller should first use
// ClearTypeInstances().
func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*manager.TypeInstance) {
func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*common.TypeInstance) {
r.Lock()
defer r.Unlock()

32
manager/repository/repository_test.go
Unescape View File

@ -14,7 +14,7 @@ limitations under the License.
package repository
import (
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
"fmt"
"testing"
@ -77,7 +77,7 @@ func testCreateDeploymentWithManifests(t *testing.T, count int) {
for i := 0; i < count; i++ {
var manifestName = fmt.Sprintf("manifest-%d", i)
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
err := r.AddManifest(deploymentName, &manifest)
if err != nil {
t.Fatalf("AddManifest failed: %v", err)
@ -136,7 +136,7 @@ func TestRepositoryDeleteWorksWithNoLatestManifest(t *testing.T) {
if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err)
}
if dDeleted.Status != manager.DeletedStatus {
if dDeleted.Status != common.DeletedStatus {
t.Fatalf("Deployment Status is not deleted")
}
if _, err := r.ListManifests(deploymentName); err == nil {
@ -148,7 +148,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
var deploymentName = "mydeployment"
var manifestName = "manifest-0"
r := NewMapBasedRepository()
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
_, err := r.CreateDeployment(deploymentName)
if err != nil {
t.Fatalf("CreateDeployment failed: %v", err)
@ -161,7 +161,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err)
}
if dDeleted.Status != manager.DeletedStatus {
if dDeleted.Status != common.DeletedStatus {
t.Fatalf("Deployment Status is not deleted")
}
}
@ -170,7 +170,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
var deploymentName = "mydeployment"
var manifestName = "manifest-0"
r := NewMapBasedRepository()
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName}
manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
_, err := r.CreateDeployment(deploymentName)
if err != nil {
t.Fatalf("CreateDeployment failed: %v", err)
@ -183,7 +183,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err)
}
if dDeleted.Status != manager.CreatedStatus {
if dDeleted.Status != common.CreatedStatus {
t.Fatalf("Deployment Status is not created")
}
}
@ -191,9 +191,9 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
func TestRepositoryTypeInstances(t *testing.T) {
r := NewMapBasedRepository()
d1Map := map[string][]*manager.TypeInstance{
"t1": []*manager.TypeInstance{
&manager.TypeInstance{
d1Map := map[string][]*common.TypeInstance{
"t1": []*common.TypeInstance{
&common.TypeInstance{
Name: "i1",
Type: "t1",
Deployment: "d1",
@ -203,9 +203,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
},
}
d2Map := map[string][]*manager.TypeInstance{
"t2": []*manager.TypeInstance{
&manager.TypeInstance{
d2Map := map[string][]*common.TypeInstance{
"t2": []*common.TypeInstance{
&common.TypeInstance{
Name: "i2",
Type: "t2",
Deployment: "d2",
@ -215,9 +215,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
},
}
d3Map := map[string][]*manager.TypeInstance{
"t2": []*manager.TypeInstance{
&manager.TypeInstance{
d3Map := map[string][]*common.TypeInstance{
"t2": []*common.TypeInstance{
&common.TypeInstance{
Name: "i3",
Type: "t2",
Deployment: "d3",

38
registry/github_registry.go
Unescape View File

@ -18,6 +18,7 @@ import (
"fmt"
"log"
"strings"
)
// GithubRegistry implements the Registry interface that talks to github.
@ -66,7 +67,11 @@ func (g *GithubRegistry) List() ([]Type, error) {
// GetURL fetches the download URL for a given Type and checks for existence of a schema file.
func (g *GithubRegistry) GetURL(t Type) (string, error) {
path := g.path + "/" + t.Name + "/" + t.Version
path,err := g.MakeRepositoryPath(t)
if err != nil {
return "", err
}
log.Printf("Got repository path: %s", path)
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil {
log.Printf("Failed to list versions at path: %s: %v", path, err)
@ -114,3 +119,34 @@ func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
return dirs, nil
}
func (g *GithubRegistry) mapCollection(collection string) (string, error) {
if strings.ContainsAny(collection, "/") {
return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
}
// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
return collection, nil
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
// The returned repository path will be of the form:
// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version
// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
// 1:1 mapping (if given)
func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) {
log.Printf("Making repository path: %v", t)
// First map the collection
collection, err := g.mapCollection(t.Collection)
if err != nil {
return "", err
}
// Construct the return path
p := ""
if len(g.path) > 0 {
p += g.path + "/"
}
if len(collection) > 0 {
p += collection + "/"
}
return p + t.Name + "/" + t.Version, nil
}

5
registry/registry.go
Unescape View File

@ -22,6 +22,7 @@ package registry
// For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis
// /v1
// redis.jinja
@ -35,6 +36,7 @@ package registry
// replicatedservice.python.schema
type Type struct {
Collection string
Name string
Version string
}
@ -46,3 +48,6 @@ type Registry interface {
// Get the download URL for a given template and version
GetURL(t Type) (string, error)
}

26
registry/registryprovider.go
Unescape View File

@ -0,0 +1,26 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
// RegistryProvider returns factories for creating registries for a given RegistryType.
type RegistryProvider interface {
GetGithubRegistry(owner string, repository string) Registry
}
type DefaultRegistryProvider struct {
}
func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
return NewGithubRegistry(owner, repository, "")
}

14
resourcifier/configurations.go
Unescape View File

@ -14,7 +14,7 @@ limitations under the License.
package main
import (
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/resourcifier/configurator"
"github.com/kubernetes/deployment-manager/util"
@ -76,8 +76,8 @@ func listConfigurationsHandlerFunc(w http.ResponseWriter, r *http.Request) {
return
}
c := &manager.Configuration{
[]*manager.Resource{
c := &common.Configuration{
[]*common.Resource{
{Type: rtype},
},
}
@ -105,8 +105,8 @@ func getConfigurationHandlerFunc(w http.ResponseWriter, r *http.Request) {
return
}
c := &manager.Configuration{
[]*manager.Resource{
c := &common.Configuration{
[]*common.Resource{
{Name: rname, Type: rtype},
},
}
@ -253,7 +253,7 @@ func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler s
return unescaped, nil
}
func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *manager.Configuration {
func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *common.Configuration {
b := io.LimitReader(r.Body, *maxLength*1024)
y, err := ioutil.ReadAll(b)
if err != nil {
@ -276,7 +276,7 @@ func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *m
return nil
}
c := &manager.Configuration{}
c := &common.Configuration{}
if err := json.Unmarshal(j, c); err != nil {
e := errors.New(err.Error() + "\n" + string(j))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

18
resourcifier/configurator/configurator.go
Unescape View File

@ -20,7 +20,7 @@ import (
"os/exec"
"strings"
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml"
)
@ -77,7 +77,7 @@ func (e *Error) appendError(err error) error {
// action on it (create/delete/replace) and updates the State of the resource with the resulting
// status. In case of errors with a resource, Resource.State.Errors is set.
// and then updates the deployment with the completion status and completion time.
func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, error) {
func (a *Configurator) Configure(c *common.Configuration, o operation) (string, error) {
errors := &Error{}
var output []string
for i, resource := range c.Resources {
@ -99,8 +99,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
if err != nil {
e := fmt.Errorf("yaml marshal failed for resource: %v: %v", resource.Name, err)
log.Println(errors.appendError(e))
c.Resources[i].State = &manager.ResourceState{
Status: manager.Aborted,
c.Resources[i].State = &common.ResourceState{
Status: common.Aborted,
Errors: []string{e.Error()},
}
continue
@ -122,8 +122,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
if err := cmd.Start(); err != nil {
e := fmt.Errorf("cannot start kubetcl for resource: %v: %v", resource.Name, err)
c.Resources[i].State = &manager.ResourceState{
Status: manager.Failed,
c.Resources[i].State = &common.ResourceState{
Status: common.Failed,
Errors: []string{e.Error()},
}
log.Println(errors.appendError(e))
@ -137,8 +137,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
log.Println(resource.Name + " not found, treating as success for delete")
} else {
e := fmt.Errorf("kubetcl failed for resource: %v: %v: %v", resource.Name, err, combined.String())
c.Resources[i].State = &manager.ResourceState{
Status: manager.Failed,
c.Resources[i].State = &common.ResourceState{
Status: common.Failed,
Errors: []string{e.Error()},
}
log.Println(errors.appendError(e))
@ -147,7 +147,7 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
}
output = append(output, combined.String())
c.Resources[i].State = &manager.ResourceState{Status: manager.Created}
c.Resources[i].State = &common.ResourceState{Status: common.Created}
log.Printf("kubectl succeeded for resource: %v: SysTime: %v UserTime: %v\n%v",
resource.Name, cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
}

34
util/templateutil.go
Unescape View File

@ -15,9 +15,39 @@ package util
import (
"strings"
"log"
"github.com/kubernetes/deployment-manager/common"
)
// IsTemplate returns whether a given type is a template.
func IsTemplate(t string) bool {
return strings.HasSuffix(t, ".py") || strings.HasSuffix(t, ".jinja")
func IsTemplate(t string, imports []*common.ImportFile) bool {
log.Printf("IsTemplate: %s : %+v", t, imports)
for _, imp := range imports {
log.Printf("Checking: %s", imp.Name)
if imp.Name == t {
return true
}
}
return false
}
// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
// For now, this means using github types:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool {
if !strings.HasPrefix(t, "github.com/") {
return false
}
s := strings.Split(t, "/")
if len(s) != 5 {
return false
}
v := strings.Split(s[4], ":")
if len(v) != 2 {
return false
}
return true
}

Write Preview
Loading…
Cancel
Save