msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

Add support for short github.com types. Refactor manager/manager/types to common/types and reuse them rather than redefine them.

Browse Source
pull/141/head
vaikas-google 9 years ago
parent 92371f1f4c
commit 92ec9a0fe1
25 changed files with 585 additions and 290 deletions
Show Stats Download Patch File Download Diff File

3
manager/manager/types.go → common/types.go
Unescape View File

@ -11,7 +11,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package manager package common
import ( import (
"time" "time"
@ -120,6 +120,7 @@ type Template struct {
// ImportFile describes a base64 encoded file imported by a Template. // ImportFile describes a base64 encoded file imported by a Template.
type ImportFile struct { type ImportFile struct {
Name string `json:"name,omitempty"` Name string `json:"name,omitempty"`
Path string `json:"path",omitempty` // Actual URL for the file
Content string `json:"content"` Content string `json:"content"`
} }

14
dm/dm.go
Unescape View File

@ -16,8 +16,8 @@ package main
import ( import (
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
@ -286,8 +286,8 @@ func isHttp(t string) bool {
return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://") return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://")
} }
func loadTemplate(args []string) *expander.Template { func loadTemplate(args []string) *common.Template {
var template *expander.Template var template *common.Template
var err error var err error
if len(args) < 2 { if len(args) < 2 {
fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied") fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied")
@ -329,7 +329,7 @@ func getRegistryType(fullType string) *registry.Type {
} }
} }
func buildTemplateFromType(t registry.Type) *expander.Template { func buildTemplateFromType(t registry.Type) *common.Template {
downloadURL := getDownloadUrl(t) downloadURL := getDownloadUrl(t)
props := make(map[string]interface{}) props := make(map[string]interface{})
@ -355,7 +355,7 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
// Name the deployment after the type name. // Name the deployment after the type name.
name := fmt.Sprintf("%s:%s", t.Name, t.Version) name := fmt.Sprintf("%s:%s", t.Name, t.Version)
config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{ config := common.Configuration{Resources: []*common.Resource{&common.Resource{
Name: name, Name: name,
Type: downloadURL, Type: downloadURL,
Properties: props, Properties: props,
@ -366,14 +366,14 @@ func buildTemplateFromType(t registry.Type) *expander.Template {
log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config) log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
} }
return &expander.Template{ return &common.Template{
Name: name, Name: name,
Content: string(y), Content: string(y),
// No imports, as this is a single type from repository. // No imports, as this is a single type from repository.
} }
} }
func marshalTemplate(template *expander.Template) io.ReadCloser { func marshalTemplate(template *common.Template) io.ReadCloser {
j, err := json.Marshal(template) j, err := json.Marshal(template)
if err != nil { if err != nil {
log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err) log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)

31
expandybird/expander/expander.go
Unescape View File

@ -23,11 +23,12 @@ import (
"path/filepath" "path/filepath"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
) )
// Expander abstracts interactions with the expander and deployer services. // Expander abstracts interactions with the expander and deployer services.
type Expander interface { type Expander interface {
ExpandTemplate(template *Template) (string, error) ExpandTemplate(template *common.Template) (string, error)
} }
type expander struct { type expander struct {
@ -39,24 +40,10 @@ func NewExpander(binary string) Expander {
return &expander{binary} return &expander{binary}
} }
// ImportFile describes a file that we import into our templates
// TODO: Encode the Content so that it doesn't get mangled.
type ImportFile struct {
Name string `json:"name,omitempty"`
Content string `json:"content"`
}
// A Template defines a single deployment.
type Template struct {
Name string `json:"name"`
Content string `json:"content"`
Imports []*ImportFile `json:"imports"`
}
// NewTemplateFromRootTemplate creates and returns a new template whose content // NewTemplateFromRootTemplate creates and returns a new template whose content
// and imported files are constructed from reading the root template, parsing out // and imported files are constructed from reading the root template, parsing out
// the imports section and reading the imports from there // the imports section and reading the imports from there
func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) { func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) {
templateDir := filepath.Dir(templateFileName) templateDir := filepath.Dir(templateFileName)
content, err := ioutil.ReadFile(templateFileName) content, err := ioutil.ReadFile(templateFileName)
if err != nil { if err != nil {
@ -85,14 +72,14 @@ func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) {
func NewTemplateFromFileNames( func NewTemplateFromFileNames(
templateFileName string, templateFileName string,
importFileNames []string, importFileNames []string,
) (*Template, error) { ) (*common.Template, error) {
name := path.Base(templateFileName) name := path.Base(templateFileName)
content, err := ioutil.ReadFile(templateFileName) content, err := ioutil.ReadFile(templateFileName)
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName) return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName)
} }
imports := []*ImportFile{} imports := []*common.ImportFile{}
for _, importFileName := range importFileNames { for _, importFileName := range importFileNames {
importFileData, err := ioutil.ReadFile(importFileName) importFileData, err := ioutil.ReadFile(importFileName)
if err != nil { if err != nil {
@ -100,13 +87,13 @@ func NewTemplateFromFileNames(
} }
imports = append(imports, imports = append(imports,
&ImportFile{ &common.ImportFile{
Name: path.Base(importFileName), Name: path.Base(importFileName),
Content: string(importFileData), Content: string(importFileData),
}) })
} }
return &Template{ return &common.Template{
Name: name, Name: name,
Content: string(content), Content: string(content),
Imports: imports, Imports: imports,
@ -190,7 +177,7 @@ func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) {
// ExpandTemplate passes the given configuration to the expander and returns the // ExpandTemplate passes the given configuration to the expander and returns the
// expanded configuration as a string on success. // expanded configuration as a string on success.
func (e *expander) ExpandTemplate(template *Template) (string, error) { func (e *expander) ExpandTemplate(template *common.Template) (string, error) {
if e.ExpansionBinary == "" { if e.ExpansionBinary == "" {
message := fmt.Sprintf("expansion binary cannot be empty") message := fmt.Sprintf("expansion binary cannot be empty")
return "", fmt.Errorf("error expanding template %s: %s", template.Name, message) return "", fmt.Errorf("error expanding template %s: %s", template.Name, message)
@ -216,7 +203,7 @@ func (e *expander) ExpandTemplate(template *Template) (string, error) {
} }
for _, imp := range template.Imports { for _, imp := range template.Imports {
cmd.Args = append(cmd.Args, imp.Name, imp.Content) cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content)
} }
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {

4
expandybird/expander/expander_test.go
Unescape View File

@ -19,6 +19,8 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"github.com/kubernetes/deployment-manager/common"
) )
const invalidFileName = "afilethatdoesnotexist" const invalidFileName = "afilethatdoesnotexist"
@ -36,7 +38,7 @@ type ExpanderTestCase struct {
ExpectedError string ExpectedError string
} }
func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *Template { func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template {
template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames) template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames)
if err != nil { if err != nil {
t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err) t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err)

33
expandybird/expansion/expansion.py
Unescape View File

@ -136,7 +136,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False):
layout = {'name': resource['name'], layout = {'name': resource['name'],
'type': resource['type']} 'type': resource['type']}
if IsTemplate(resource['type']) and resource['type'] in imports: if resource['type'] in imports:
# A template resource, which contains sub-resources. # A template resource, which contains sub-resources.
expanded_template = ExpandTemplate(resource, imports, env, validate_schema) expanded_template = ExpandTemplate(resource, imports, env, validate_schema)
@ -183,11 +183,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'):
# If this resource doesn't have a name, we will report that error later # If this resource doesn't have a name, we will report that error later
def IsTemplate(resource_type):
"""Returns whether a given resource type is a Template."""
return resource_type.endswith('.py') or resource_type.endswith('.jinja')
def ExpandTemplate(resource, imports, env, validate_schema=False): def ExpandTemplate(resource, imports, env, validate_schema=False):
"""Expands a template, calling expansion mechanism based on type. """Expands a template, calling expansion mechanism based on type.
@ -206,6 +201,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
ExpansionError: if there is any error occurred during expansion ExpansionError: if there is any error occurred during expansion
""" """
source_file = resource['type'] source_file = resource['type']
path = resource['type']
# Look for Template in imports. # Look for Template in imports.
if source_file not in imports: if source_file not in imports:
@ -213,6 +209,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
source_file, source_file,
'Unable to find source file %s in imports.' % (source_file)) 'Unable to find source file %s in imports.' % (source_file))
# source_file could be a short version of the template (say github short name)
# so we need to potentially map this into the fully resolvable name.
if 'path' in imports[source_file]:
if imports[source_file]['path']:
path = imports[source_file]['path']
resource['imports'] = imports resource['imports'] = imports
# Populate the additional environment variables. # Populate the additional environment variables.
@ -231,13 +233,13 @@ def ExpandTemplate(resource, imports, env, validate_schema=False):
except schema_validation.ValidationErrors as e: except schema_validation.ValidationErrors as e:
raise ExpansionError(resource['name'], e.message) raise ExpansionError(resource['name'], e.message)
if source_file.endswith('jinja'): if path.endswith('jinja'):
expanded_template = ExpandJinja( expanded_template = ExpandJinja(
source_file, imports[source_file], resource, imports) source_file, imports[source_file]['content'], resource, imports)
elif source_file.endswith('py'): elif path.endswith('py'):
# This is a Python template. # This is a Python template.
expanded_template = ExpandPython( expanded_template = ExpandPython(
imports[source_file], source_file, resource) imports[source_file]['content'], source_file, resource)
else: else:
# The source file is not a jinja file or a python file. # The source file is not a jinja file or a python file.
# This in fact should never happen due to the IsTemplate check above. # This in fact should never happen due to the IsTemplate check above.
@ -262,8 +264,8 @@ def ExpandJinja(file_name, source_template, resource, imports):
source_template: string, the content of jinja file to be render source_template: string, the content of jinja file to be render
resource: resource object, the resource that contains parameters to the resource: resource object, the resource that contains parameters to the
jinja file jinja file
imports: map from string to string, the map of imported files names imports: map from string to map {name, path}, the map of imported files names
and contents fully resolved path and contents
Returns: Returns:
The final expanded template The final expanded template
Raises: Raises:
@ -362,9 +364,10 @@ def main():
print >>sys.stderr, 'Invalid import definition at argv pos %d' % idx print >>sys.stderr, 'Invalid import definition at argv pos %d' % idx
sys.exit(1) sys.exit(1)
name = sys.argv[idx] name = sys.argv[idx]
value = sys.argv[idx + 1] path = sys.argv[idx + 1]
imports[name] = value value = sys.argv[idx + 2]
idx += 2 imports[name] = {'content': value, 'path': path}
idx += 3
env = {} env = {}
env['deployment'] = os.environ['DEPLOYMENT_NAME'] env['deployment'] = os.environ['DEPLOYMENT_NAME']

2
expandybird/expansion/sandbox_loader.py
Unescape View File

@ -95,7 +95,7 @@ def process_imports(imports):
# Now build the hierarchical modules. # Now build the hierarchical modules.
for k in imports.keys(): for k in imports.keys():
if imports[k].endswith('.jinja'): if imports[k]['path'].endswith('.jinja'):
continue continue
# Normalize paths and trim .py extension, if any. # Normalize paths and trim .py extension, if any.
normalized = os.path.splitext(os.path.normpath(k))[0] normalized = os.path.splitext(os.path.normpath(k))[0]

5
expandybird/service/service.go
Unescape View File

@ -15,6 +15,7 @@ package service
import ( import (
"github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
"errors" "errors"
@ -40,7 +41,7 @@ func NewService(handler restful.RouteFunction) *Service {
webService.Produces(restful.MIME_JSON, restful.MIME_XML) webService.Produces(restful.MIME_JSON, restful.MIME_XML)
webService.Route(webService.POST("/expand").To(handler). webService.Route(webService.POST("/expand").To(handler).
Doc("Expand a template."). Doc("Expand a template.").
Reads(&expander.Template{})) Reads(&common.Template{}))
return &Service{webService} return &Service{webService}
} }
@ -60,7 +61,7 @@ func (s *Service) Register(container *restful.Container) {
func NewExpansionHandler(backend expander.Expander) restful.RouteFunction { func NewExpansionHandler(backend expander.Expander) restful.RouteFunction {
return func(req *restful.Request, resp *restful.Response) { return func(req *restful.Request, resp *restful.Response) {
util.LogHandlerEntry("expandybird: expand", req.Request) util.LogHandlerEntry("expandybird: expand", req.Request)
template := &expander.Template{} template := &common.Template{}
if err := req.ReadEntity(&template); err != nil { if err := req.ReadEntity(&template); err != nil {
logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp) logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp)
return return

3
expandybird/service/service_test.go
Unescape View File

@ -24,6 +24,7 @@ import (
"testing" "testing"
"github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/expandybird/expander"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
restful "github.com/emicklei/go-restful" restful "github.com/emicklei/go-restful"
@ -180,7 +181,7 @@ type mockExpander struct {
// ExpandTemplate passes the given configuration to the expander and returns the // ExpandTemplate passes the given configuration to the expander and returns the
// expanded configuration as a string on success. // expanded configuration as a string on success.
func (e *mockExpander) ExpandTemplate(template *expander.Template) (string, error) { func (e *mockExpander) ExpandTemplate(template *common.Template) (string, error) {
switch template.Name { switch template.Name {
case "InvalidFileName.yaml": case "InvalidFileName.yaml":
return "", fmt.Errorf("expansion error") return "", fmt.Errorf("expansion error")

11
manager/deployments.go
Unescape View File

@ -22,6 +22,7 @@ import (
"log" "log"
"net" "net"
"net/http" "net/http"
"net/url"
"os" "os"
"strings" "strings"
@ -29,6 +30,7 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/manager/manager"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/manager/repository" "github.com/kubernetes/deployment-manager/manager/repository"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
) )
@ -198,17 +200,16 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) {
func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) { func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) {
vars := mux.Vars(r) vars := mux.Vars(r)
variable, ok := vars[variable] retVariable, ok := vars[variable]
if !ok { if !ok {
e := fmt.Errorf("%s parameter not found in URL", variable) e := fmt.Errorf("%s parameter not found in URL", variable)
util.LogAndReturnError(handler, http.StatusBadRequest, e, w) util.LogAndReturnError(handler, http.StatusBadRequest, e, w)
return "", e return "", e
} }
return retVariable, nil
return variable, nil
} }
func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manager.Template { func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template {
util.LogHandlerEntry(handler, r) util.LogHandlerEntry(handler, r)
b := io.LimitReader(r.Body, *maxLength*1024) b := io.LimitReader(r.Body, *maxLength*1024)
y, err := ioutil.ReadAll(b) y, err := ioutil.ReadAll(b)
@ -237,7 +238,7 @@ func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manage
return nil return nil
} }
t := &manager.Template{} t := &common.Template{}
if err := json.Unmarshal(j, t); err != nil { if err := json.Unmarshal(j, t); err != nil {
e := fmt.Errorf("%v\n%v", err, string(j)) e := fmt.Errorf("%v\n%v", err, string(j))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w) util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

25
manager/manager/deployer.go
Unescape View File

@ -25,14 +25,15 @@ import (
"strings" "strings"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/common"
) )
// Deployer abstracts interactions with the expander and deployer services. // Deployer abstracts interactions with the expander and deployer services.
type Deployer interface { type Deployer interface {
GetConfiguration(cached *Configuration) (*Configuration, error) GetConfiguration(cached *common.Configuration) (*common.Configuration, error)
CreateConfiguration(configuration *Configuration) (*Configuration, error) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error)
DeleteConfiguration(configuration *Configuration) (*Configuration, error) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error)
PutConfiguration(configuration *Configuration) (*Configuration, error) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error)
} }
// NewDeployer returns a new initialized Deployer. // NewDeployer returns a new initialized Deployer.
@ -54,9 +55,9 @@ type formatter func(err error) error
// GetConfiguration reads and returns the actual configuration // GetConfiguration reads and returns the actual configuration
// of the resources described by a cached configuration. // of the resources described by a cached configuration.
func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, error) { func (d *deployer) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) {
errors := &Error{} errors := &Error{}
actual := &Configuration{} actual := &common.Configuration{}
for _, resource := range cached.Resources { for _, resource := range cached.Resources {
rtype := url.QueryEscape(resource.Type) rtype := url.QueryEscape(resource.Type)
rname := url.QueryEscape(resource.Name) rname := url.QueryEscape(resource.Name)
@ -70,7 +71,7 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
} }
if len(body) != 0 { if len(body) != 0 {
result := &Resource{Name: resource.Name, Type: resource.Type} result := &common.Resource{Name: resource.Name, Type: resource.Type}
if err := yaml.Unmarshal(body, &result.Properties); err != nil { if err := yaml.Unmarshal(body, &result.Properties); err != nil {
return nil, fmt.Errorf("cannot get configuration for resource (%v)", err) return nil, fmt.Errorf("cannot get configuration for resource (%v)", err)
} }
@ -88,22 +89,22 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro
// CreateConfiguration deploys the set of resources described by a configuration and returns // CreateConfiguration deploys the set of resources described by a configuration and returns
// the Configuration with status for each resource filled in. // the Configuration with status for each resource filled in.
func (d *deployer) CreateConfiguration(configuration *Configuration) (*Configuration, error) { func (d *deployer) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("POST", "create", configuration) return d.callServiceWithConfiguration("POST", "create", configuration)
} }
// DeleteConfiguration deletes the set of resources described by a configuration. // DeleteConfiguration deletes the set of resources described by a configuration.
func (d *deployer) DeleteConfiguration(configuration *Configuration) (*Configuration, error) { func (d *deployer) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("DELETE", "delete", configuration) return d.callServiceWithConfiguration("DELETE", "delete", configuration)
} }
// PutConfiguration replaces the set of resources described by a configuration and returns // PutConfiguration replaces the set of resources described by a configuration and returns
// the Configuration with status for each resource filled in. // the Configuration with status for each resource filled in.
func (d *deployer) PutConfiguration(configuration *Configuration) (*Configuration, error) { func (d *deployer) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return d.callServiceWithConfiguration("PUT", "replace", configuration) return d.callServiceWithConfiguration("PUT", "replace", configuration)
} }
func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *Configuration) (*Configuration, error) { func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *common.Configuration) (*common.Configuration, error) {
callback := func(e error) error { callback := func(e error) error {
return fmt.Errorf("cannot %s configuration: %s", operation, e) return fmt.Errorf("cannot %s configuration: %s", operation, e)
} }
@ -120,7 +121,7 @@ func (d *deployer) callServiceWithConfiguration(method, operation string, config
return nil, err return nil, err
} }
result := &Configuration{} result := &common.Configuration{}
if len(resp) != 0 { if len(resp) != 0 {
if err := yaml.Unmarshal(resp, &result); err != nil { if err := yaml.Unmarshal(resp, &result); err != nil {
return nil, fmt.Errorf("cannot unmarshal response: (%v)", err) return nil, fmt.Errorf("cannot unmarshal response: (%v)", err)

9
manager/manager/deployer_test.go
Unescape View File

@ -24,6 +24,7 @@ import (
"testing" "testing"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
) )
@ -250,8 +251,8 @@ func TestPutConfiguration(t *testing.T) {
} }
} }
func getValidConfiguration(t *testing.T) *Configuration { func getValidConfiguration(t *testing.T) *common.Configuration {
valid := &Configuration{} valid := &common.Configuration{}
err := yaml.Unmarshal(validConfigurationTestCaseData, valid) err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
if err != nil { if err != nil {
t.Errorf("cannot unmarshal test case data:%s\n", err) t.Errorf("cannot unmarshal test case data:%s\n", err)
@ -266,7 +267,7 @@ func deployerErrorHandler(w http.ResponseWriter, r *http.Request) {
} }
func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) { func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
valid := &Configuration{} valid := &common.Configuration{}
err := yaml.Unmarshal(validConfigurationTestCaseData, valid) err := yaml.Unmarshal(validConfigurationTestCaseData, valid)
if err != nil { if err != nil {
status := fmt.Sprintf("cannot unmarshal test case data:%s", err) status := fmt.Sprintf("cannot unmarshal test case data:%s", err)
@ -282,7 +283,7 @@ func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
result := &Configuration{} result := &common.Configuration{}
if err := yaml.Unmarshal(body, result); err != nil { if err := yaml.Unmarshal(body, result); err != nil {
status := fmt.Sprintf("cannot unmarshal request body:%s", err) status := fmt.Sprintf("cannot unmarshal request body:%s", err)
http.Error(w, status, http.StatusInternalServerError) http.Error(w, status, http.StatusInternalServerError)

27
manager/manager/expander.go
Unescape View File

@ -22,6 +22,7 @@ import (
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
) )
const ( const (
@ -31,13 +32,13 @@ const (
// ExpandedTemplate is the structure returned by the expansion service. // ExpandedTemplate is the structure returned by the expansion service.
type ExpandedTemplate struct { type ExpandedTemplate struct {
Config *Configuration `json:"config"` Config *common.Configuration `json:"config"`
Layout *Layout `json:"layout"` Layout *common.Layout `json:"layout"`
} }
// Expander abstracts interactions with the expander and deployer services. // Expander abstracts interactions with the expander and deployer services.
type Expander interface { type Expander interface {
ExpandTemplate(t *Template) (*ExpandedTemplate, error) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error)
} }
// NewExpander returns a new initialized Expander. // NewExpander returns a new initialized Expander.
@ -54,7 +55,7 @@ func (e *expander) getBaseURL() string {
return fmt.Sprintf("%s/expand", e.expanderURL) return fmt.Sprintf("%s/expand", e.expanderURL)
} }
func expanderError(t *Template, err error) error { func expanderError(t *common.Template, err error) error {
return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content) return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content)
} }
@ -90,14 +91,14 @@ func expanderError(t *Template, err error) error {
// between the name#template key to exist in the layout given a particular choice of naming. // between the name#template key to exist in the layout given a particular choice of naming.
// In practice, it would be nearly impossible to hit, but consider including properties/name/type // In practice, it would be nearly impossible to hit, but consider including properties/name/type
// into a hash of sorts to make this robust... // into a hash of sorts to make this robust...
func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*LayoutResource { func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource {
ret := map[string]*LayoutResource{} ret := map[string]*common.LayoutResource{}
toVisit := l.Resources toVisit := l.Resources
for len(toVisit) > 0 { for len(toVisit) > 0 {
lr := toVisit[0] lr := toVisit[0]
nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type
if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type) { if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) {
ret[nodeKey] = lr ret[nodeKey] = lr
} else if toReplace[nodeKey] != nil { } else if toReplace[nodeKey] != nil {
toReplace[nodeKey].Resources = lr.Resources toReplace[nodeKey].Resources = lr.Resources
@ -112,20 +113,20 @@ func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*Lay
// ExpandTemplate expands the supplied template, and returns a configuration. // ExpandTemplate expands the supplied template, and returns a configuration.
// It will also update the imports in the provided template if any were added // It will also update the imports in the provided template if any were added
// during type resolution. // during type resolution.
func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) {
// We have a fencepost problem here. // We have a fencepost problem here.
// 1. Start by trying to resolve any missing templates // 1. Start by trying to resolve any missing templates
// 2. Expand the configuration using all the of the imports available to us at this point // 2. Expand the configuration using all the of the imports available to us at this point
// 3. Expansion may yield additional templates, so we run the type resolution again // 3. Expansion may yield additional templates, so we run the type resolution again
// 4. If type resolution resulted in new imports being available, return to 2. // 4. If type resolution resulted in new imports being available, return to 2.
config := &Configuration{} config := &common.Configuration{}
if err := yaml.Unmarshal([]byte(t.Content), config); err != nil { if err := yaml.Unmarshal([]byte(t.Content), config); err != nil {
e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content) e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content)
return nil, e return nil, e
} }
var finalLayout *Layout var finalLayout *common.Layout
needResolve := map[string]*LayoutResource{} needResolve := map[string]*common.LayoutResource{}
// Start things off by attempting to resolve the templates in a first pass. // Start things off by attempting to resolve the templates in a first pass.
newImp, err := e.typeResolver.ResolveTypes(config, t.Imports) newImp, err := e.typeResolver.ResolveTypes(config, t.Imports)
@ -151,7 +152,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
if finalLayout == nil { if finalLayout == nil {
finalLayout = result.Layout finalLayout = result.Layout
} }
needResolve = walkLayout(result.Layout, needResolve) needResolve = walkLayout(result.Layout, t.Imports, needResolve)
newImp, err = e.typeResolver.ResolveTypes(result.Config, t.Imports) newImp, err = e.typeResolver.ResolveTypes(result.Config, t.Imports)
if err != nil { if err != nil {
@ -170,7 +171,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) {
} }
} }
func (e *expander) expandTemplate(t *Template) (*ExpandedTemplate, error) { func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) {
j, err := json.Marshal(t) j, err := json.Marshal(t)
if err != nil { if err != nil {
return nil, err return nil, err

27
manager/manager/expander_test.go
Unescape View File

@ -24,16 +24,17 @@ import (
"testing" "testing"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
"github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
) )
type mockResolver struct { type mockResolver struct {
responses [][]*ImportFile responses [][]*common.ImportFile
t *testing.T t *testing.T
} }
func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*ImportFile, error) { func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
if len(r.responses) < 1 { if len(r.responses) < 1 {
return nil, nil return nil, nil
} }
@ -43,7 +44,7 @@ func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*Impor
return ret, nil return ret, nil
} }
var validTemplateTestCaseData = Template{ var validTemplateTestCaseData = common.Template{
Name: "TestTemplate", Name: "TestTemplate",
Content: string(validContentTestCaseData), Content: string(validContentTestCaseData),
Imports: validImportFilesTestCaseData, Imports: validImportFilesTestCaseData,
@ -59,11 +60,19 @@ resources:
test-property: test-value test-property: test-value
`) `)
var validImportFilesTestCaseData = []*ImportFile{ var validImportFilesTestCaseData = []*common.ImportFile{
&ImportFile{ &common.ImportFile{
Name: "test-type.py", Name: "test-type.py",
Content: "test-type.py validTemplateTestCaseData content", Content: "test-type.py validTemplateTestCaseData content",
}, },
&common.ImportFile{
Name: "test.py",
Content: "test.py validTemplateTestCaseData content",
},
&common.ImportFile{
Name: "test2.py",
Content: "test2.py validTemplateTestCaseData content",
},
} }
var validConfigTestCaseData = []byte(` var validConfigTestCaseData = []byte(`
@ -210,7 +219,7 @@ layout:
test: test test: test
` `
var roundTripTemplate = Template{ var roundTripTemplate = common.Template{
Name: "TestTemplate", Name: "TestTemplate",
Content: roundTripContent, Content: roundTripContent,
Imports: nil, Imports: nil,
@ -249,9 +258,9 @@ func TestExpandTemplate(t *testing.T) {
"expect success for ExpandTemplate with two expansions", "expect success for ExpandTemplate with two expansions",
"", "",
roundTripHandler, roundTripHandler,
&mockResolver{[][]*ImportFile{ &mockResolver{[][]*common.ImportFile{
{}, {},
{&ImportFile{Name: "test.py"}}, {&common.ImportFile{Name: "test.py"}},
}, t}, }, t},
roundTripResponse, roundTripResponse,
}, },
@ -334,7 +343,7 @@ func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
template := &Template{} template := &common.Template{}
if err := json.Unmarshal(body, template); err != nil { if err := json.Unmarshal(body, template); err != nil {
status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body) status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body)
http.Error(w, status, http.StatusInternalServerError) http.Error(w, status, http.StatusInternalServerError)

72
manager/manager/manager.go
Unescape View File

@ -17,35 +17,37 @@ import (
"fmt" "fmt"
"log" "log"
"time" "time"
"github.com/kubernetes/deployment-manager/common"
) )
// Manager manages a persistent set of Deployments. // Manager manages a persistent set of Deployments.
type Manager interface { type Manager interface {
ListDeployments() ([]Deployment, error) ListDeployments() ([]common.Deployment, error)
GetDeployment(name string) (*Deployment, error) GetDeployment(name string) (*common.Deployment, error)
CreateDeployment(t *Template) (*Deployment, error) CreateDeployment(t *common.Template) (*common.Deployment, error)
DeleteDeployment(name string, forget bool) (*Deployment, error) DeleteDeployment(name string, forget bool) (*common.Deployment, error)
PutDeployment(name string, t *Template) (*Deployment, error) PutDeployment(name string, t *common.Template) (*common.Deployment, error)
ListManifests(deploymentName string) (map[string]*Manifest, error) ListManifests(deploymentName string) (map[string]*common.Manifest, error)
GetManifest(deploymentName string, manifest string) (*Manifest, error) GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
Expand(t *Template) (*Manifest, error) Expand(t *common.Template) (*common.Manifest, error)
ListTypes() []string ListTypes() []string
ListInstances(typeName string) []*TypeInstance ListInstances(typeName string) []*common.TypeInstance
} }
type manager struct { type manager struct {
expander Expander expander Expander
deployer Deployer deployer Deployer
repository Repository repository common.Repository
} }
// NewManager returns a new initialized Manager. // NewManager returns a new initialized Manager.
func NewManager(expander Expander, deployer Deployer, repository Repository) Manager { func NewManager(expander Expander, deployer Deployer, repository common.Repository) Manager {
return &manager{expander, deployer, repository} return &manager{expander, deployer, repository}
} }
// ListDeployments returns the list of deployments // ListDeployments returns the list of deployments
func (m *manager) ListDeployments() ([]Deployment, error) { func (m *manager) ListDeployments() ([]common.Deployment, error) {
l, err := m.repository.ListDeployments() l, err := m.repository.ListDeployments()
if err != nil { if err != nil {
return nil, err return nil, err
@ -55,7 +57,7 @@ func (m *manager) ListDeployments() ([]Deployment, error) {
// GetDeployment retrieves the configuration stored for a given deployment // GetDeployment retrieves the configuration stored for a given deployment
// as well as the current configuration from the cluster. // as well as the current configuration from the cluster.
func (m *manager) GetDeployment(name string) (*Deployment, error) { func (m *manager) GetDeployment(name string) (*common.Deployment, error) {
d, err := m.repository.GetDeployment(name) d, err := m.repository.GetDeployment(name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -66,7 +68,7 @@ func (m *manager) GetDeployment(name string) (*Deployment, error) {
// ListManifests retrieves the manifests for a given deployment // ListManifests retrieves the manifests for a given deployment
// of each of the deployments in the repository and returns the deployments. // of each of the deployments in the repository and returns the deployments.
func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, error) { func (m *manager) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
l, err := m.repository.ListManifests(deploymentName) l, err := m.repository.ListManifests(deploymentName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -76,7 +78,7 @@ func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, er
} }
// GetManifest retrieves the specified manifest for a given deployment // GetManifest retrieves the specified manifest for a given deployment
func (m *manager) GetManifest(deploymentName string, manifestName string) (*Manifest, error) { func (m *manager) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
d, err := m.repository.GetManifest(deploymentName, manifestName) d, err := m.repository.GetManifest(deploymentName, manifestName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -88,7 +90,7 @@ func (m *manager) GetManifest(deploymentName string, manifestName string) (*Mani
// CreateDeployment expands the supplied template, creates the resulting // CreateDeployment expands the supplied template, creates the resulting
// configuration in the cluster, creates a new deployment that tracks it, // configuration in the cluster, creates a new deployment that tracks it,
// and stores the deployment in the repository. Returns the deployment. // and stores the deployment in the repository. Returns the deployment.
func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { func (m *manager) CreateDeployment(t *common.Template) (*common.Deployment, error) {
log.Printf("Creating deployment: %s", t.Name) log.Printf("Creating deployment: %s", t.Name)
_, err := m.repository.CreateDeployment(t.Name) _, err := m.repository.CreateDeployment(t.Name)
if err != nil { if err != nil {
@ -99,7 +101,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
manifest, err := m.createManifest(t) manifest, err := m.createManifest(t)
if err != nil { if err != nil {
log.Printf("Manifest creation failed: %v", err) log.Printf("Manifest creation failed: %v", err)
m.repository.SetDeploymentStatus(t.Name, FailedStatus) m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
return nil, err return nil, err
} }
@ -107,7 +109,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
if createErr != nil { if createErr != nil {
// Deployment failed, mark as failed // Deployment failed, mark as failed
log.Printf("CreateConfiguration failed: %v", err) log.Printf("CreateConfiguration failed: %v", err)
m.repository.SetDeploymentStatus(t.Name, FailedStatus) m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
// If we failed before being able to create some of the resources, then // If we failed before being able to create some of the resources, then
// return the failure as such. Otherwise, we're going to add the manifest // return the failure as such. Otherwise, we're going to add the manifest
// and hence resource specific errors down below. // and hence resource specific errors down below.
@ -115,7 +117,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
return nil, createErr return nil, createErr
} }
} else { } else {
m.repository.SetDeploymentStatus(t.Name, DeployedStatus) m.repository.SetDeploymentStatus(t.Name, common.DeployedStatus)
} }
// Update the manifest with the actual state of the reified resources // Update the manifest with the actual state of the reified resources
@ -123,7 +125,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
aErr := m.repository.AddManifest(t.Name, manifest) aErr := m.repository.AddManifest(t.Name, manifest)
if aErr != nil { if aErr != nil {
log.Printf("AddManifest failed %v", aErr) log.Printf("AddManifest failed %v", aErr)
m.repository.SetDeploymentStatus(t.Name, FailedStatus) m.repository.SetDeploymentStatus(t.Name, common.FailedStatus)
// If there's an earlier error, return that instead since it contains // If there's an earlier error, return that instead since it contains
// more applicable error message. Adding manifest failure is more akin // more applicable error message. Adding manifest failure is more akin
// to a check fail (either deployment doesn't exist, or a manifest with the same // to a check fail (either deployment doesn't exist, or a manifest with the same
@ -141,14 +143,14 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) {
return m.repository.GetValidDeployment(t.Name) return m.repository.GetValidDeployment(t.Name)
} }
func (m *manager) createManifest(t *Template) (*Manifest, error) { func (m *manager) createManifest(t *common.Template) (*common.Manifest, error) {
et, err := m.expander.ExpandTemplate(t) et, err := m.expander.ExpandTemplate(t)
if err != nil { if err != nil {
log.Printf("Expansion failed %v", err) log.Printf("Expansion failed %v", err)
return nil, err return nil, err
} }
return &Manifest{ return &common.Manifest{
Name: generateManifestName(), Name: generateManifestName(),
Deployment: t.Name, Deployment: t.Name,
InputConfig: t, InputConfig: t,
@ -157,10 +159,10 @@ func (m *manager) createManifest(t *Template) (*Manifest, error) {
}, nil }, nil
} }
func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *Layout) { func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *common.Layout) {
m.repository.ClearTypeInstances(deploymentName) m.repository.ClearTypeInstances(deploymentName)
instances := make(map[string][]*TypeInstance) instances := make(map[string][]*common.TypeInstance)
for i, r := range layout.Resources { for i, r := range layout.Resources {
addTypeInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i)) addTypeInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i))
} }
@ -168,9 +170,9 @@ func (m *manager) addTypeInstances(deploymentName string, manifestName string, l
m.repository.SetTypeInstances(deploymentName, instances) m.repository.SetTypeInstances(deploymentName, instances)
} }
func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource, deploymentName string, manifestName string, jsonPath string) { func addTypeInstances(instances *map[string][]*common.TypeInstance, r *common.LayoutResource, deploymentName string, manifestName string, jsonPath string) {
// Add this resource. // Add this resource.
inst := &TypeInstance{ inst := &common.TypeInstance{
Name: r.Name, Name: r.Name,
Type: r.Type, Type: r.Type,
Deployment: deploymentName, Deployment: deploymentName,
@ -189,7 +191,7 @@ func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource,
// the supplied identifier from the cluster.repository. If forget is true, then // the supplied identifier from the cluster.repository. If forget is true, then
// the deployment is removed from the repository. Otherwise, it is marked // the deployment is removed from the repository. Otherwise, it is marked
// as deleted and retained. // as deleted and retained.
func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error) { func (m *manager) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
log.Printf("Deleting deployment: %s", name) log.Printf("Deleting deployment: %s", name)
d, err := m.repository.GetValidDeployment(name) d, err := m.repository.GetValidDeployment(name)
if err != nil { if err != nil {
@ -210,7 +212,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error
} }
// Create an empty manifest since resources have been deleted. // Create an empty manifest since resources have been deleted.
err = m.repository.AddManifest(name, &Manifest{Deployment: name, Name: generateManifestName()}) err = m.repository.AddManifest(name, &common.Manifest{Deployment: name, Name: generateManifestName()})
if err != nil { if err != nil {
log.Printf("Failed to add empty manifest") log.Printf("Failed to add empty manifest")
return nil, err return nil, err
@ -230,7 +232,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error
// PutDeployment replaces the configuration of the deployment with // PutDeployment replaces the configuration of the deployment with
// the supplied identifier in the cluster, and returns the deployment. // the supplied identifier in the cluster, and returns the deployment.
func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) { func (m *manager) PutDeployment(name string, t *common.Template) (*common.Deployment, error) {
_, err := m.repository.GetValidDeployment(name) _, err := m.repository.GetValidDeployment(name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -239,20 +241,20 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) {
manifest, err := m.createManifest(t) manifest, err := m.createManifest(t)
if err != nil { if err != nil {
log.Printf("Manifest creation failed: %v", err) log.Printf("Manifest creation failed: %v", err)
m.repository.SetDeploymentStatus(name, FailedStatus) m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err return nil, err
} }
actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig) actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig)
if err != nil { if err != nil {
m.repository.SetDeploymentStatus(name, FailedStatus) m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err return nil, err
} }
manifest.ExpandedConfig = actualConfig manifest.ExpandedConfig = actualConfig
err = m.repository.AddManifest(t.Name, manifest) err = m.repository.AddManifest(t.Name, manifest)
if err != nil { if err != nil {
m.repository.SetDeploymentStatus(name, FailedStatus) m.repository.SetDeploymentStatus(name, common.FailedStatus)
return nil, err return nil, err
} }
@ -262,14 +264,14 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) {
return m.repository.GetValidDeployment(t.Name) return m.repository.GetValidDeployment(t.Name)
} }
func (m *manager) Expand(t *Template) (*Manifest, error) { func (m *manager) Expand(t *common.Template) (*common.Manifest, error) {
et, err := m.expander.ExpandTemplate(t) et, err := m.expander.ExpandTemplate(t)
if err != nil { if err != nil {
log.Printf("Expansion failed %v", err) log.Printf("Expansion failed %v", err)
return nil, err return nil, err
} }
return &Manifest{ return &common.Manifest{
ExpandedConfig: et.Config, ExpandedConfig: et.Config,
Layout: et.Layout, Layout: et.Layout,
}, nil }, nil
@ -279,7 +281,7 @@ func (m *manager) ListTypes() []string {
return m.repository.ListTypes() return m.repository.ListTypes()
} }
func (m *manager) ListInstances(typeName string) []*TypeInstance { func (m *manager) ListInstances(typeName string) []*common.TypeInstance {
return m.repository.GetTypeInstances(typeName) return m.repository.GetTypeInstances(typeName)
} }

90
manager/manager/manager_test.go
Unescape View File

@ -18,25 +18,27 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"github.com/kubernetes/deployment-manager/common"
) )
var template = Template{Name: "test", Content: "test"} var template = common.Template{Name: "test", Content: "test"}
var layout = Layout{ var layout = common.Layout{
Resources: []*LayoutResource{&LayoutResource{Resource: Resource{Name: "test", Type: "test"}}}, Resources: []*common.LayoutResource{&common.LayoutResource{Resource: common.Resource{Name: "test", Type: "test"}}},
} }
var configuration = Configuration{ var configuration = common.Configuration{
Resources: []*Resource{&Resource{Name: "test", Type: "test"}}, Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test"}},
} }
var resourcesWithSuccessState = Configuration{ var resourcesWithSuccessState = common.Configuration{
Resources: []*Resource{&Resource{Name: "test", Type: "test", State: &ResourceState{Status: Created}}}, Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test", State: &common.ResourceState{Status: common.Created}}},
} }
var resourcesWithFailureState = Configuration{ var resourcesWithFailureState = common.Configuration{
Resources: []*Resource{&Resource{ Resources: []*common.Resource{&common.Resource{
Name: "test", Name: "test",
Type: "test", Type: "test",
State: &ResourceState{ State: &common.ResourceState{
Status: Failed, Status: common.Failed,
Errors: []string{"test induced error"}, Errors: []string{"test induced error"},
}, },
}}, }},
@ -49,14 +51,14 @@ var expandedConfig = ExpandedTemplate{
var deploymentName = "deployment" var deploymentName = "deployment"
var manifestName = "manifest-2" var manifestName = "manifest-2"
var manifest = Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout} var manifest = common.Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout}
var manifestMap = map[string]*Manifest{manifest.Name: &manifest} var manifestMap = map[string]*common.Manifest{manifest.Name: &manifest}
var deployment = Deployment{ var deployment = common.Deployment{
Name: "test", Name: "test",
} }
var deploymentList = []Deployment{deployment, {Name: "test2"}} var deploymentList = []common.Deployment{deployment, {Name: "test2"}}
var typeInstMap = map[string][]string{"test": []string{"test"}} var typeInstMap = map[string][]string{"test": []string{"test"}}
@ -64,7 +66,7 @@ var errTest = errors.New("test")
type expanderStub struct{} type expanderStub struct{}
func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { func (expander *expanderStub) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) {
if reflect.DeepEqual(*t, template) { if reflect.DeepEqual(*t, template) {
return &expandedConfig, nil return &expandedConfig, nil
} }
@ -74,17 +76,17 @@ func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, er
type deployerStub struct { type deployerStub struct {
FailCreate bool FailCreate bool
Created []*Configuration Created []*common.Configuration
FailDelete bool FailDelete bool
Deleted []*Configuration Deleted []*common.Configuration
FailCreateResource bool FailCreateResource bool
} }
func (deployer *deployerStub) reset() { func (deployer *deployerStub) reset() {
deployer.FailCreate = false deployer.FailCreate = false
deployer.Created = make([]*Configuration, 0) deployer.Created = make([]*common.Configuration, 0)
deployer.FailDelete = false deployer.FailDelete = false
deployer.Deleted = make([]*Configuration, 0) deployer.Deleted = make([]*common.Configuration, 0)
deployer.FailCreateResource = false deployer.FailCreateResource = false
} }
@ -93,11 +95,11 @@ func newDeployerStub() *deployerStub {
return ret return ret
} }
func (deployer *deployerStub) GetConfiguration(cached *Configuration) (*Configuration, error) { func (deployer *deployerStub) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) {
return nil, nil return nil, nil
} }
func (deployer *deployerStub) CreateConfiguration(configuration *Configuration) (*Configuration, error) { func (deployer *deployerStub) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
if deployer.FailCreate { if deployer.FailCreate {
return nil, errTest return nil, errTest
} }
@ -109,7 +111,7 @@ func (deployer *deployerStub) CreateConfiguration(configuration *Configuration)
return &resourcesWithSuccessState, nil return &resourcesWithSuccessState, nil
} }
func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration) (*Configuration, error) { func (deployer *deployerStub) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
if deployer.FailDelete { if deployer.FailDelete {
return nil, errTest return nil, errTest
} }
@ -117,34 +119,34 @@ func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration)
return nil, nil return nil, nil
} }
func (deployer *deployerStub) PutConfiguration(configuration *Configuration) (*Configuration, error) { func (deployer *deployerStub) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) {
return nil, nil return nil, nil
} }
type repositoryStub struct { type repositoryStub struct {
FailListDeployments bool FailListDeployments bool
Created []string Created []string
ManifestAdd map[string]*Manifest ManifestAdd map[string]*common.Manifest
Deleted []string Deleted []string
GetValid []string GetValid []string
TypeInstances map[string][]string TypeInstances map[string][]string
TypeInstancesCleared bool TypeInstancesCleared bool
GetTypeInstancesCalled bool GetTypeInstancesCalled bool
ListTypesCalled bool ListTypesCalled bool
DeploymentStatuses []DeploymentStatus DeploymentStatuses []common.DeploymentStatus
} }
func (repository *repositoryStub) reset() { func (repository *repositoryStub) reset() {
repository.FailListDeployments = false repository.FailListDeployments = false
repository.Created = make([]string, 0) repository.Created = make([]string, 0)
repository.ManifestAdd = make(map[string]*Manifest) repository.ManifestAdd = make(map[string]*common.Manifest)
repository.Deleted = make([]string, 0) repository.Deleted = make([]string, 0)
repository.GetValid = make([]string, 0) repository.GetValid = make([]string, 0)
repository.TypeInstances = make(map[string][]string) repository.TypeInstances = make(map[string][]string)
repository.TypeInstancesCleared = false repository.TypeInstancesCleared = false
repository.GetTypeInstancesCalled = false repository.GetTypeInstancesCalled = false
repository.ListTypesCalled = false repository.ListTypesCalled = false
repository.DeploymentStatuses = make([]DeploymentStatus, 0) repository.DeploymentStatuses = make([]common.DeploymentStatus, 0)
} }
func newRepositoryStub() *repositoryStub { func newRepositoryStub() *repositoryStub {
@ -152,14 +154,14 @@ func newRepositoryStub() *repositoryStub {
return ret return ret
} }
func (repository *repositoryStub) ListDeployments() ([]Deployment, error) { func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error) {
if repository.FailListDeployments { if repository.FailListDeployments {
return deploymentList, errTest return deploymentList, errTest
} }
return deploymentList, nil return deploymentList, nil
} }
func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) { func (repository *repositoryStub) GetDeployment(d string) (*common.Deployment, error) {
if d == deploymentName { if d == deploymentName {
return &deployment, nil return &deployment, nil
} }
@ -167,32 +169,32 @@ func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) {
return nil, errTest return nil, errTest
} }
func (repository *repositoryStub) GetValidDeployment(d string) (*Deployment, error) { func (repository *repositoryStub) GetValidDeployment(d string) (*common.Deployment, error) {
repository.GetValid = append(repository.GetValid, d) repository.GetValid = append(repository.GetValid, d)
return &deployment, nil return &deployment, nil
} }
func (repository *repositoryStub) SetDeploymentStatus(name string, status DeploymentStatus) error { func (repository *repositoryStub) SetDeploymentStatus(name string, status common.DeploymentStatus) error {
repository.DeploymentStatuses = append(repository.DeploymentStatuses, status) repository.DeploymentStatuses = append(repository.DeploymentStatuses, status)
return nil return nil
} }
func (repository *repositoryStub) CreateDeployment(d string) (*Deployment, error) { func (repository *repositoryStub) CreateDeployment(d string) (*common.Deployment, error) {
repository.Created = append(repository.Created, d) repository.Created = append(repository.Created, d)
return &deployment, nil return &deployment, nil
} }
func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*Deployment, error) { func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*common.Deployment, error) {
repository.Deleted = append(repository.Deleted, d) repository.Deleted = append(repository.Deleted, d)
return &deployment, nil return &deployment, nil
} }
func (repository *repositoryStub) AddManifest(d string, manifest *Manifest) error { func (repository *repositoryStub) AddManifest(d string, manifest *common.Manifest) error {
repository.ManifestAdd[d] = manifest repository.ManifestAdd[d] = manifest
return nil return nil
} }
func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error) { func (repository *repositoryStub) GetLatestManifest(d string) (*common.Manifest, error) {
if d == deploymentName { if d == deploymentName {
return repository.ManifestAdd[d], nil return repository.ManifestAdd[d], nil
} }
@ -200,7 +202,7 @@ func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error)
return nil, errTest return nil, errTest
} }
func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest, error) { func (repository *repositoryStub) ListManifests(d string) (map[string]*common.Manifest, error) {
if d == deploymentName { if d == deploymentName {
return manifestMap, nil return manifestMap, nil
} }
@ -208,7 +210,7 @@ func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest,
return nil, errTest return nil, errTest
} }
func (repository *repositoryStub) GetManifest(d string, m string) (*Manifest, error) { func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manifest, error) {
if d == deploymentName && m == manifestName { if d == deploymentName && m == manifestName {
return &manifest, nil return &manifest, nil
} }
@ -221,16 +223,16 @@ func (r *repositoryStub) ListTypes() []string {
return []string{} return []string{}
} }
func (r *repositoryStub) GetTypeInstances(t string) []*TypeInstance { func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance {
r.GetTypeInstancesCalled = true r.GetTypeInstancesCalled = true
return []*TypeInstance{} return []*common.TypeInstance{}
} }
func (r *repositoryStub) ClearTypeInstances(d string) { func (r *repositoryStub) ClearTypeInstances(d string) {
r.TypeInstancesCleared = true r.TypeInstancesCleared = true
} }
func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*TypeInstance) { func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) {
for k, _ := range is { for k, _ := range is {
r.TypeInstances[d] = append(r.TypeInstances[d], k) r.TypeInstances[d] = append(r.TypeInstances[d], k)
} }
@ -326,7 +328,7 @@ func TestCreateDeployment(t *testing.T) {
testDeployer.Created[0], configuration) testDeployer.Created[0], configuration)
} }
if testRepository.DeploymentStatuses[0] != DeployedStatus { if testRepository.DeploymentStatuses[0] != common.DeployedStatus {
t.Fatal("CreateDeployment success did not mark deployment as deployed") t.Fatal("CreateDeployment success did not mark deployment as deployed")
} }
@ -355,7 +357,7 @@ func TestCreateDeploymentCreationFailure(t *testing.T) {
testRepository.Created[0]) testRepository.Created[0])
} }
if testRepository.DeploymentStatuses[0] != FailedStatus { if testRepository.DeploymentStatuses[0] != common.FailedStatus {
t.Fatal("CreateDeployment failure did not mark deployment as failed") t.Fatal("CreateDeployment failure did not mark deployment as failed")
} }
@ -385,7 +387,7 @@ func TestCreateDeploymentCreationResourceFailure(t *testing.T) {
testRepository.Created[0]) testRepository.Created[0])
} }
if testRepository.DeploymentStatuses[0] != FailedStatus { if testRepository.DeploymentStatuses[0] != common.FailedStatus {
t.Fatal("CreateDeployment failure did not mark deployment as failed") t.Fatal("CreateDeployment failure did not mark deployment as failed")
} }

106
manager/manager/typeresolver.go
Unescape View File

@ -16,8 +16,11 @@ package manager
import ( import (
"fmt" "fmt"
"net/http" "net/http"
"regexp"
"time" "time"
"github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
@ -31,12 +34,14 @@ const (
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file // TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL. // or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface { type TypeResolver interface {
ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
} }
type typeResolver struct { type typeResolver struct {
getter util.HTTPClient getter util.HTTPClient
maxUrls int maxUrls int
re *regexp.Regexp
rp registry.RegistryProvider
} }
// NewTypeResolver returns a new initialized TypeResolver. // NewTypeResolver returns a new initialized TypeResolver.
@ -48,10 +53,12 @@ func NewTypeResolver() TypeResolver {
client.Timeout = timeout client.Timeout = timeout
ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.getter = util.NewHTTPClient(3, client, util.NewSleeper())
ret.maxUrls = maxURLImports ret.maxUrls = maxURLImports
ret.re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
ret.rp = &registry.DefaultRegistryProvider{}
return ret return ret
} }
func resolverError(c *Configuration, err error) error { func resolverError(c *common.Configuration, err error) error {
return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n", return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n",
c, err) c, err)
} }
@ -78,21 +85,23 @@ func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content str
// resolved type definitions in t.ImportFiles. Types can be either // resolved type definitions in t.ImportFiles. Types can be either
// primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote // primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote
// (i.e., described by a URL that must be fetched to resolve the type). // (i.e., described by a URL that must be fetched to resolve the type).
func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) { func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) {
existing := map[string]bool{} existing := map[string]bool{}
for _, v := range imports { for _, v := range imports {
existing[v.Name] = true existing[v.Name] = true
} }
fetched := map[string][]*ImportFile{} fetched := map[string][]*common.ImportFile{}
toFetch := make([]string, 0, tr.maxUrls) toFetch := make([]string, 0, tr.maxUrls)
for _, r := range config.Resources { for _, r := range config.Resources {
// Only fetch HTTP URLs that we haven't already imported. // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example).
if util.IsHttpUrl(r.Type) && !existing[r.Type] { u, err := tr.MapFetchableURL(r.Type)
toFetch = append(toFetch, r.Type) if err != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err))
fetched[r.Type] = append(fetched[r.Type], &ImportFile{Name: r.Type}) }
if len(u) > 0 && !existing[r.Type] {
toFetch = append(toFetch, u)
fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u})
// Add to existing map so it is not fetched multiple times. // Add to existing map so it is not fetched multiple times.
existing[r.Type] = true existing[r.Type] = true
} }
@ -100,13 +109,14 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
count := 0 count := 0
for len(toFetch) > 0 { for len(toFetch) > 0 {
//1. Fetch import URL. Exit if no URLs left //1. If short github URL, resolve to a download URL
//2. Check/handle HTTP status //2. Fetch import URL. Exit if no URLs left
//3. Store results in all ImportFiles from that URL //3. Check/handle HTTP status
//4. Check for the optional schema file at import URL + .schema //4. Store results in all ImportFiles from that URL
//5. Repeat 2,3 for schema file //5. Check for the optional schema file at import URL + .schema
//6. Add each schema import to fetch if not already done //6. Repeat 2,3 for schema file
//7. Mark URL done. Return to 1. //7. Add each schema import to fetch if not already done
//8. Mark URL done. Return to 1.
if count >= tr.maxUrls { if count >= tr.maxUrls {
return nil, resolverError(config, return nil, resolverError(config,
fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls)) fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls))
@ -129,32 +139,41 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
} }
if sch != "" { if sch != "" {
var s Schema var s common.Schema
if err := yaml.Unmarshal([]byte(sch), &s); err != nil { if err := yaml.Unmarshal([]byte(sch), &s); err != nil {
return nil, resolverError(config, err) return nil, resolverError(config, err)
} }
// Here we handle any nested imports in the schema we've just fetched. // Here we handle any nested imports in the schema we've just fetched.
for _, v := range s.Imports { for _, v := range s.Imports {
i := &ImportFile{Name: v.Name} i := &common.ImportFile{Name: v.Name}
var existingSchema string var existingSchema string
if len(fetched[v.Path]) == 0 { u, conversionErr := tr.MapFetchableURL(v.Path)
if conversionErr != nil {
return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr))
}
// If it's not a fetchable URL, we need to use the type name as is, since it is a short name
// for a schema.
if len(u) == 0 {
u = v.Path
}
if len(fetched[u]) == 0 {
// If this import URL is new to us, add it to the URLs to fetch. // If this import URL is new to us, add it to the URLs to fetch.
toFetch = append(toFetch, v.Path) toFetch = append(toFetch, u)
} else { } else {
// If this is not a new import URL and we've already fetched its contents, // If this is not a new import URL and we've already fetched its contents,
// reuse them. Also, check if we also found a schema for that import URL and // reuse them. Also, check if we also found a schema for that import URL and
// record those contents for re-use as well. // record those contents for re-use as well.
if fetched[v.Path][0].Content != "" { if fetched[u][0].Content != "" {
i.Content = fetched[v.Path][0].Content i.Content = fetched[u][0].Content
if len(fetched[v.Path+schemaSuffix]) > 0 { if len(fetched[u+schemaSuffix]) > 0 {
existingSchema = fetched[v.Path+schemaSuffix][0].Content existingSchema = fetched[u+schemaSuffix][0].Content
} }
} }
} }
fetched[v.Path] = append(fetched[v.Path], i) fetched[u] = append(fetched[u], i)
if existingSchema != "" { if existingSchema != "" {
fetched[v.Path+schemaSuffix] = append(fetched[v.Path+schemaSuffix], fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix],
&ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) &common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema})
} }
} }
@ -162,7 +181,7 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
for _, i := range fetched[url] { for _, i := range fetched[url] {
schemaImportName := i.Name + schemaSuffix schemaImportName := i.Name + schemaSuffix
fetched[schemaURL] = append(fetched[schemaURL], fetched[schemaURL] = append(fetched[schemaURL],
&ImportFile{Name: schemaImportName, Content: sch}) &common.ImportFile{Name: schemaImportName, Content: sch})
} }
} }
@ -170,10 +189,37 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil
toFetch = toFetch[1:] toFetch = toFetch[1:]
} }
ret := []*ImportFile{} ret := []*common.ImportFile{}
for _, v := range fetched { for _, v := range fetched {
ret = append(ret, v...) ret = append(ret, v...)
} }
return ret, nil return ret, nil
} }
// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL
// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for
// example) will return empty string.
func (tr *typeResolver) MapFetchableURL(t string) (string, error) {
if util.IsGithubShortType(t) {
return tr.ShortTypeToDownloadURL(t)
} else if util.IsHttpUrl(t) {
return t, nil
}
return "", nil
}
// ShortTypeToDownloadURL converts a github URL into downloadable URL from github.
// Input must be of the type and is assumed to have been validated before this call:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) {
m := tr.re.FindStringSubmatch(template)
if len(m) != 6 {
return "", fmt.Errorf("Failed to parse short github url: %s", template)
}
r := tr.rp.GetGithubRegistry(m[1], m[2])
t := registry.Type{m[3], m[4], m[5]}
return r.GetURL(t)
}

186
manager/manager/typeresolver_test.go
Unescape View File

@ -15,14 +15,20 @@ package manager
import ( import (
"errors" "errors"
"fmt"
"net/http" "net/http"
"reflect" "reflect"
"regexp"
"strings" "strings"
"testing" "testing"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
"github.com/kubernetes/deployment-manager/registry"
"github.com/kubernetes/deployment-manager/common"
) )
var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)")
type responseAndError struct { type responseAndError struct {
err error err error
code int code int
@ -30,12 +36,13 @@ type responseAndError struct {
} }
type resolverTestCase struct { type resolverTestCase struct {
config string config string
imports []*ImportFile imports []*common.ImportFile
responses map[string]responseAndError responses map[string]responseAndError
urlcount int urlcount int
expectedErr error expectedErr error
importOut []*ImportFile importOut []*common.ImportFile
registryProvider registry.RegistryProvider
} }
type testGetter struct { type testGetter struct {
@ -51,11 +58,68 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) {
return ret.resp, ret.code, ret.err return ret.resp, ret.code, ret.err
} }
type urlAndError struct {
u string
e error
}
type testRegistryProvider struct {
owner string
repo string
r map[string]registry.Registry
}
func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider {
r := make(map[string]registry.Registry)
r[owner+repository] = &testGithubRegistry{tests, count}
return &testRegistryProvider{owner, repository, r}
}
func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry {
return trp.r[owner+repository]
}
type testGithubRegistry struct {
responses map[registry.Type]urlAndError
count int
}
func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) {
tgr.count = tgr.count + 1
ret := tgr.responses[t]
return ret.u, ret.e
}
func (tgr *testGithubRegistry) List() ([]registry.Type, error) {
return []registry.Type{}, fmt.Errorf("List should not be called in the test")
}
func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) {
r := &typeResolver{
re: re,
rp: c.registryProvider,
}
for in, expected := range tests {
actual, err := r.ShortTypeToDownloadURL(in)
if actual != expected.u {
t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual)
}
if err != expected.e {
t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err)
}
}
}
func testDriver(c resolverTestCase, t *testing.T) { func testDriver(c resolverTestCase, t *testing.T) {
g := &testGetter{test: t, responses: c.responses} g := &testGetter{test: t, responses: c.responses}
r := &typeResolver{getter: g, maxUrls: 5} r := &typeResolver{
getter: g,
maxUrls: 5,
re: re,
rp: c.registryProvider,
}
conf := &Configuration{} conf := &common.Configuration{}
dataErr := yaml.Unmarshal([]byte(c.config), conf) dataErr := yaml.Unmarshal([]byte(c.config), conf)
if dataErr != nil { if dataErr != nil {
panic("bad test data") panic("bad test data")
@ -73,8 +137,8 @@ func testDriver(c resolverTestCase, t *testing.T) {
t.Errorf("Expected error %s but found %s", c.expectedErr, err) t.Errorf("Expected error %s but found %s", c.expectedErr, err)
} }
resultImport := map[ImportFile]bool{} resultImport := map[common.ImportFile]bool{}
expectedImport := map[ImportFile]bool{} expectedImport := map[common.ImportFile]bool{}
for _, i := range result { for _, i := range result {
resultImport[*i] = true resultImport[*i] = true
} }
@ -106,7 +170,7 @@ resources:
` `
func TestIncludedImport(t *testing.T) { func TestIncludedImport(t *testing.T) {
imports := []*ImportFile{&ImportFile{Name: "foo.py"}} imports := []*common.ImportFile{&common.ImportFile{Name: "foo.py"}}
test := resolverTestCase{ test := resolverTestCase{
config: includeImport, config: includeImport,
imports: imports, imports: imports,
@ -121,7 +185,7 @@ resources:
` `
func TestSingleUrl(t *testing.T) { func TestSingleUrl(t *testing.T) {
finalImports := []*ImportFile{&ImportFile{Name: "http://my-fake-url", Content: "my-content"}} finalImports := []*common.ImportFile{&common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}}
responses := map[string]responseAndError{ responses := map[string]responseAndError{
"http://my-fake-url": responseAndError{nil, http.StatusOK, "my-content"}, "http://my-fake-url": responseAndError{nil, http.StatusOK, "my-content"},
@ -158,10 +222,10 @@ imports:
` `
func TestSingleUrlWithSchema(t *testing.T) { func TestSingleUrlWithSchema(t *testing.T) {
finalImports := []*ImportFile{ finalImports := []*common.ImportFile{
&ImportFile{Name: "http://my-fake-url", Content: "my-content"}, &common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
&ImportFile{Name: "schema-import", Content: "schema-import"}, &common.ImportFile{Name: "schema-import", Content: "schema-import"},
&ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, &common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
} }
responses := map[string]responseAndError{ responses := map[string]responseAndError{
@ -236,13 +300,13 @@ imports:
` `
func TestSharedImport(t *testing.T) { func TestSharedImport(t *testing.T) {
finalImports := []*ImportFile{ finalImports := []*common.ImportFile{
&ImportFile{Name: "http://my-fake-url", Content: "my-content"}, &common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"},
&ImportFile{Name: "http://my-fake-url1", Content: "my-content-1"}, &common.ImportFile{Name: "http://my-fake-url1", Path: "http://my-fake-url1", Content: "my-content-1"},
&ImportFile{Name: "schema-import", Content: "schema-import"}, &common.ImportFile{Name: "schema-import", Content: "schema-import"},
&ImportFile{Name: "schema-import-1", Content: "schema-import"}, &common.ImportFile{Name: "schema-import-1", Content: "schema-import"},
&ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, &common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1},
&ImportFile{Name: "http://my-fake-url1.schema", Content: schema2}, &common.ImportFile{Name: "http://my-fake-url1.schema", Content: schema2},
} }
responses := map[string]responseAndError{ responses := map[string]responseAndError{
@ -262,3 +326,79 @@ func TestSharedImport(t *testing.T) {
} }
testDriver(test, t) testDriver(test, t)
} }
func TestShortGithubUrlMapping(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
}
func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) {
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
tests := map[string]urlAndError{
"github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil},
"github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil},
}
test := resolverTestCase{
registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2),
}
testUrlConversionDriver(test, tests, t)
}
var templateShortGithubTemplate = `
resources:
- name: foo
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v1
- name: foo1
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
`
func TestShortGithubUrl(t *testing.T) {
finalImports := []*common.ImportFile{
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py",
Content: "my-content"},
&common.ImportFile{
Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2",
Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py",
Content: "my-content-2"},
}
responses := map[string]responseAndError{
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content-2"},
"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""},
}
githubUrlMaps := map[registry.Type]urlAndError{
registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil},
registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil},
}
test := resolverTestCase{
config: templateShortGithubTemplate,
importOut: finalImports,
urlcount: 4,
responses: responses,
registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2),
}
testDriver(test, t)
}

60
manager/repository/repository.go
Unescape View File

@ -22,37 +22,37 @@ import (
"sync" "sync"
"time" "time"
"github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common"
) )
// deploymentTypeInstanceMap stores type instances mapped by deployment name. // deploymentTypeInstanceMap stores type instances mapped by deployment name.
// This allows for simple updating and deleting of per-deployment instances // This allows for simple updating and deleting of per-deployment instances
// when deployments are created/updated/deleted. // when deployments are created/updated/deleted.
type deploymentTypeInstanceMap map[string][]*manager.TypeInstance type deploymentTypeInstanceMap map[string][]*common.TypeInstance
type typeInstanceMap map[string]deploymentTypeInstanceMap type typeInstanceMap map[string]deploymentTypeInstanceMap
type mapBasedRepository struct { type mapBasedRepository struct {
sync.RWMutex sync.RWMutex
deployments map[string]manager.Deployment deployments map[string]common.Deployment
manifests map[string]map[string]*manager.Manifest manifests map[string]map[string]*common.Manifest
instances typeInstanceMap instances typeInstanceMap
} }
// NewMapBasedRepository returns a new map based repository. // NewMapBasedRepository returns a new map based repository.
func NewMapBasedRepository() manager.Repository { func NewMapBasedRepository() common.Repository {
return &mapBasedRepository{ return &mapBasedRepository{
deployments: make(map[string]manager.Deployment, 0), deployments: make(map[string]common.Deployment, 0),
manifests: make(map[string]map[string]*manager.Manifest, 0), manifests: make(map[string]map[string]*common.Manifest, 0),
instances: typeInstanceMap{}, instances: typeInstanceMap{},
} }
} }
// ListDeployments returns of all of the deployments in the repository. // ListDeployments returns of all of the deployments in the repository.
func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) { func (r *mapBasedRepository) ListDeployments() ([]common.Deployment, error) {
r.RLock() r.RLock()
defer r.RUnlock() defer r.RUnlock()
l := []manager.Deployment{} l := []common.Deployment{}
for _, deployment := range r.deployments { for _, deployment := range r.deployments {
l = append(l, deployment) l = append(l, deployment)
} }
@ -62,7 +62,7 @@ func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) {
// GetDeployment returns the deployment with the supplied name. // GetDeployment returns the deployment with the supplied name.
// If the deployment is not found, it returns an error. // If the deployment is not found, it returns an error.
func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, error) { func (r *mapBasedRepository) GetDeployment(name string) (*common.Deployment, error) {
d, ok := r.deployments[name] d, ok := r.deployments[name]
if !ok { if !ok {
return nil, fmt.Errorf("deployment %s not found", name) return nil, fmt.Errorf("deployment %s not found", name)
@ -72,13 +72,13 @@ func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, er
// GetValidDeployment returns the deployment with the supplied name. // GetValidDeployment returns the deployment with the supplied name.
// If the deployment is not found or marked as deleted, it returns an error. // If the deployment is not found or marked as deleted, it returns an error.
func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deployment, error) { func (r *mapBasedRepository) GetValidDeployment(name string) (*common.Deployment, error) {
d, err := r.GetDeployment(name) d, err := r.GetDeployment(name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if d.Status == manager.DeletedStatus { if d.Status == common.DeletedStatus {
return nil, fmt.Errorf("deployment %s is deleted", name) return nil, fmt.Errorf("deployment %s is deleted", name)
} }
@ -86,7 +86,7 @@ func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deploymen
} }
// SetDeploymentStatus sets the DeploymentStatus of the deployment and updates ModifiedAt // SetDeploymentStatus sets the DeploymentStatus of the deployment and updates ModifiedAt
func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.DeploymentStatus) error { func (r *mapBasedRepository) SetDeploymentStatus(name string, status common.DeploymentStatus) error {
return func() error { return func() error {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -104,8 +104,8 @@ func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.Dep
} }
// CreateDeployment creates a new deployment and stores it in the repository. // CreateDeployment creates a new deployment and stores it in the repository.
func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, error) { func (r *mapBasedRepository) CreateDeployment(name string) (*common.Deployment, error) {
d, err := func() (*manager.Deployment, error) { d, err := func() (*common.Deployment, error) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -114,8 +114,8 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
return nil, fmt.Errorf("Deployment %s already exists", name) return nil, fmt.Errorf("Deployment %s already exists", name)
} }
d := manager.NewDeployment(name) d := common.NewDeployment(name)
d.Status = manager.CreatedStatus d.Status = common.CreatedStatus
d.DeployedAt = time.Now() d.DeployedAt = time.Now()
r.deployments[name] = *d r.deployments[name] = *d
return d, nil return d, nil
@ -129,7 +129,7 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment,
return d, nil return d, nil
} }
func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manager.Manifest) error { func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *common.Manifest) error {
err := func() error { err := func() error {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -168,8 +168,8 @@ func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manage
// DeleteDeployment deletes the deployment with the supplied name. // DeleteDeployment deletes the deployment with the supplied name.
// If forget is true, then the deployment is removed from the repository. // If forget is true, then the deployment is removed from the repository.
// Otherwise, it is marked as deleted and retained. // Otherwise, it is marked as deleted and retained.
func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manager.Deployment, error) { func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
d, err := func() (*manager.Deployment, error) { d, err := func() (*common.Deployment, error) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -180,7 +180,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
if !forget { if !forget {
d.DeletedAt = time.Now() d.DeletedAt = time.Now()
d.Status = manager.DeletedStatus d.Status = common.DeletedStatus
r.deployments[name] = *d r.deployments[name] = *d
} else { } else {
delete(r.deployments, name) delete(r.deployments, name)
@ -199,7 +199,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage
return d, nil return d, nil
} }
func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*manager.Manifest, error) { func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -211,17 +211,17 @@ func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*m
return r.listManifestsForDeployment(deploymentName) return r.listManifestsForDeployment(deploymentName)
} }
func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*manager.Manifest, error) { func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*common.Manifest, error) {
l, ok := r.manifests[deploymentName] l, ok := r.manifests[deploymentName]
if !ok { if !ok {
l = make(map[string]*manager.Manifest, 0) l = make(map[string]*common.Manifest, 0)
r.manifests[deploymentName] = l r.manifests[deploymentName] = l
} }
return l, nil return l, nil
} }
func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*manager.Manifest, error) { func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -233,7 +233,7 @@ func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName str
return r.getManifestForDeployment(deploymentName, manifestName) return r.getManifestForDeployment(deploymentName, manifestName)
} }
func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*manager.Manifest, error) { func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*common.Manifest, error) {
l, err := r.listManifestsForDeployment(deploymentName) l, err := r.listManifestsForDeployment(deploymentName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -249,7 +249,7 @@ func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, man
// GetLatestManifest returns the latest manifest for a given deployment, // GetLatestManifest returns the latest manifest for a given deployment,
// which by definition is the manifest with the largest time stamp. // which by definition is the manifest with the largest time stamp.
func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*manager.Manifest, error) { func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*common.Manifest, error) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
@ -273,11 +273,11 @@ func (r *mapBasedRepository) ListTypes() []string {
// GetTypeInstances returns all instances of a given type. If type is empty, // GetTypeInstances returns all instances of a given type. If type is empty,
// returns all instances for all types. // returns all instances for all types.
func (r *mapBasedRepository) GetTypeInstances(typeName string) []*manager.TypeInstance { func (r *mapBasedRepository) GetTypeInstances(typeName string) []*common.TypeInstance {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()
var instances []*manager.TypeInstance var instances []*common.TypeInstance
for t, dInstMap := range r.instances { for t, dInstMap := range r.instances {
if t == typeName || typeName == "all" { if t == typeName || typeName == "all" {
for _, i := range dInstMap { for _, i := range dInstMap {
@ -307,7 +307,7 @@ func (r *mapBasedRepository) ClearTypeInstances(deploymentName string) {
// //
// To clear the current set of instances first, caller should first use // To clear the current set of instances first, caller should first use
// ClearTypeInstances(). // ClearTypeInstances().
func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*manager.TypeInstance) { func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*common.TypeInstance) {
r.Lock() r.Lock()
defer r.Unlock() defer r.Unlock()

32
manager/repository/repository_test.go
Unescape View File

@ -14,7 +14,7 @@ limitations under the License.
package repository package repository
import ( import (
"github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common"
"fmt" "fmt"
"testing" "testing"
@ -77,7 +77,7 @@ func testCreateDeploymentWithManifests(t *testing.T, count int) {
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
var manifestName = fmt.Sprintf("manifest-%d", i) var manifestName = fmt.Sprintf("manifest-%d", i)
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
err := r.AddManifest(deploymentName, &manifest) err := r.AddManifest(deploymentName, &manifest)
if err != nil { if err != nil {
t.Fatalf("AddManifest failed: %v", err) t.Fatalf("AddManifest failed: %v", err)
@ -136,7 +136,7 @@ func TestRepositoryDeleteWorksWithNoLatestManifest(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err) t.Fatalf("DeleteDeployment failed: %v", err)
} }
if dDeleted.Status != manager.DeletedStatus { if dDeleted.Status != common.DeletedStatus {
t.Fatalf("Deployment Status is not deleted") t.Fatalf("Deployment Status is not deleted")
} }
if _, err := r.ListManifests(deploymentName); err == nil { if _, err := r.ListManifests(deploymentName); err == nil {
@ -148,7 +148,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
var deploymentName = "mydeployment" var deploymentName = "mydeployment"
var manifestName = "manifest-0" var manifestName = "manifest-0"
r := NewMapBasedRepository() r := NewMapBasedRepository()
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
_, err := r.CreateDeployment(deploymentName) _, err := r.CreateDeployment(deploymentName)
if err != nil { if err != nil {
t.Fatalf("CreateDeployment failed: %v", err) t.Fatalf("CreateDeployment failed: %v", err)
@ -161,7 +161,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err) t.Fatalf("DeleteDeployment failed: %v", err)
} }
if dDeleted.Status != manager.DeletedStatus { if dDeleted.Status != common.DeletedStatus {
t.Fatalf("Deployment Status is not deleted") t.Fatalf("Deployment Status is not deleted")
} }
} }
@ -170,7 +170,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
var deploymentName = "mydeployment" var deploymentName = "mydeployment"
var manifestName = "manifest-0" var manifestName = "manifest-0"
r := NewMapBasedRepository() r := NewMapBasedRepository()
manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} manifest := common.Manifest{Deployment: deploymentName, Name: manifestName}
_, err := r.CreateDeployment(deploymentName) _, err := r.CreateDeployment(deploymentName)
if err != nil { if err != nil {
t.Fatalf("CreateDeployment failed: %v", err) t.Fatalf("CreateDeployment failed: %v", err)
@ -183,7 +183,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("DeleteDeployment failed: %v", err) t.Fatalf("DeleteDeployment failed: %v", err)
} }
if dDeleted.Status != manager.CreatedStatus { if dDeleted.Status != common.CreatedStatus {
t.Fatalf("Deployment Status is not created") t.Fatalf("Deployment Status is not created")
} }
} }
@ -191,9 +191,9 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) {
func TestRepositoryTypeInstances(t *testing.T) { func TestRepositoryTypeInstances(t *testing.T) {
r := NewMapBasedRepository() r := NewMapBasedRepository()
d1Map := map[string][]*manager.TypeInstance{ d1Map := map[string][]*common.TypeInstance{
"t1": []*manager.TypeInstance{ "t1": []*common.TypeInstance{
&manager.TypeInstance{ &common.TypeInstance{
Name: "i1", Name: "i1",
Type: "t1", Type: "t1",
Deployment: "d1", Deployment: "d1",
@ -203,9 +203,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
}, },
} }
d2Map := map[string][]*manager.TypeInstance{ d2Map := map[string][]*common.TypeInstance{
"t2": []*manager.TypeInstance{ "t2": []*common.TypeInstance{
&manager.TypeInstance{ &common.TypeInstance{
Name: "i2", Name: "i2",
Type: "t2", Type: "t2",
Deployment: "d2", Deployment: "d2",
@ -215,9 +215,9 @@ func TestRepositoryTypeInstances(t *testing.T) {
}, },
} }
d3Map := map[string][]*manager.TypeInstance{ d3Map := map[string][]*common.TypeInstance{
"t2": []*manager.TypeInstance{ "t2": []*common.TypeInstance{
&manager.TypeInstance{ &common.TypeInstance{
Name: "i3", Name: "i3",
Type: "t2", Type: "t2",
Deployment: "d3", Deployment: "d3",

38
registry/github_registry.go
Unescape View File

@ -18,6 +18,7 @@ import (
"fmt" "fmt"
"log" "log"
"strings"
) )
// GithubRegistry implements the Registry interface that talks to github. // GithubRegistry implements the Registry interface that talks to github.
@ -66,7 +67,11 @@ func (g *GithubRegistry) List() ([]Type, error) {
// GetURL fetches the download URL for a given Type and checks for existence of a schema file. // GetURL fetches the download URL for a given Type and checks for existence of a schema file.
func (g *GithubRegistry) GetURL(t Type) (string, error) { func (g *GithubRegistry) GetURL(t Type) (string, error) {
path := g.path + "/" + t.Name + "/" + t.Version path,err := g.MakeRepositoryPath(t)
if err != nil {
return "", err
}
log.Printf("Got repository path: %s", path)
_, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil)
if err != nil { if err != nil {
log.Printf("Failed to list versions at path: %s: %v", path, err) log.Printf("Failed to list versions at path: %s: %v", path, err)
@ -114,3 +119,34 @@ func (g *GithubRegistry) getDirs(dir string) ([]string, error) {
return dirs, nil return dirs, nil
} }
func (g *GithubRegistry) mapCollection(collection string) (string, error) {
if strings.ContainsAny(collection, "/") {
return "", fmt.Errorf("collection must not contain slashes, got %s", collection)
}
// TODO(vaikas): Implement lookup from the root metadata file to map collection to a path
return collection, nil
}
// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version.
// The returned repository path will be of the form:
// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version
// Type.Collection will be mapped using mapCollection in the future, for now it's a straight
// 1:1 mapping (if given)
func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) {
log.Printf("Making repository path: %v", t)
// First map the collection
collection, err := g.mapCollection(t.Collection)
if err != nil {
return "", err
}
// Construct the return path
p := ""
if len(g.path) > 0 {
p += g.path + "/"
}
if len(collection) > 0 {
p += collection + "/"
}
return p + t.Name + "/" + t.Version, nil
}

5
registry/registry.go
Unescape View File

@ -22,6 +22,7 @@ package registry
// For example, a template registry containing two versions of redis // For example, a template registry containing two versions of redis
// (implemented in jinja), and one version of replicatedservice (implemented // (implemented in jinja), and one version of replicatedservice (implemented
// in python) would have a directory structure that looks something like this: // in python) would have a directory structure that looks something like this:
// qualifier [optional] prefix to a virtual root within the repository.
// /redis // /redis
// /v1 // /v1
// redis.jinja // redis.jinja
@ -35,6 +36,7 @@ package registry
// replicatedservice.python.schema // replicatedservice.python.schema
type Type struct { type Type struct {
Collection string
Name string Name string
Version string Version string
} }
@ -46,3 +48,6 @@ type Registry interface {
// Get the download URL for a given template and version // Get the download URL for a given template and version
GetURL(t Type) (string, error) GetURL(t Type) (string, error)
} }

26
registry/registryprovider.go
Unescape View File

@ -0,0 +1,26 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
// RegistryProvider returns factories for creating registries for a given RegistryType.
type RegistryProvider interface {
GetGithubRegistry(owner string, repository string) Registry
}
type DefaultRegistryProvider struct {
}
func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry {
return NewGithubRegistry(owner, repository, "")
}

14
resourcifier/configurations.go
Unescape View File

@ -14,7 +14,7 @@ limitations under the License.
package main package main
import ( import (
"github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common"
"github.com/kubernetes/deployment-manager/resourcifier/configurator" "github.com/kubernetes/deployment-manager/resourcifier/configurator"
"github.com/kubernetes/deployment-manager/util" "github.com/kubernetes/deployment-manager/util"
@ -76,8 +76,8 @@ func listConfigurationsHandlerFunc(w http.ResponseWriter, r *http.Request) {
return return
} }
c := &manager.Configuration{ c := &common.Configuration{
[]*manager.Resource{ []*common.Resource{
{Type: rtype}, {Type: rtype},
}, },
} }
@ -105,8 +105,8 @@ func getConfigurationHandlerFunc(w http.ResponseWriter, r *http.Request) {
return return
} }
c := &manager.Configuration{ c := &common.Configuration{
[]*manager.Resource{ []*common.Resource{
{Name: rname, Type: rtype}, {Name: rname, Type: rtype},
}, },
} }
@ -253,7 +253,7 @@ func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler s
return unescaped, nil return unescaped, nil
} }
func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *manager.Configuration { func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *common.Configuration {
b := io.LimitReader(r.Body, *maxLength*1024) b := io.LimitReader(r.Body, *maxLength*1024)
y, err := ioutil.ReadAll(b) y, err := ioutil.ReadAll(b)
if err != nil { if err != nil {
@ -276,7 +276,7 @@ func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *m
return nil return nil
} }
c := &manager.Configuration{} c := &common.Configuration{}
if err := json.Unmarshal(j, c); err != nil { if err := json.Unmarshal(j, c); err != nil {
e := errors.New(err.Error() + "\n" + string(j)) e := errors.New(err.Error() + "\n" + string(j))
util.LogAndReturnError(handler, http.StatusBadRequest, e, w) util.LogAndReturnError(handler, http.StatusBadRequest, e, w)

18
resourcifier/configurator/configurator.go
Unescape View File

@ -20,7 +20,7 @@ import (
"os/exec" "os/exec"
"strings" "strings"
"github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/common"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
) )
@ -77,7 +77,7 @@ func (e *Error) appendError(err error) error {
// action on it (create/delete/replace) and updates the State of the resource with the resulting // action on it (create/delete/replace) and updates the State of the resource with the resulting
// status. In case of errors with a resource, Resource.State.Errors is set. // status. In case of errors with a resource, Resource.State.Errors is set.
// and then updates the deployment with the completion status and completion time. // and then updates the deployment with the completion status and completion time.
func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, error) { func (a *Configurator) Configure(c *common.Configuration, o operation) (string, error) {
errors := &Error{} errors := &Error{}
var output []string var output []string
for i, resource := range c.Resources { for i, resource := range c.Resources {
@ -99,8 +99,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
if err != nil { if err != nil {
e := fmt.Errorf("yaml marshal failed for resource: %v: %v", resource.Name, err) e := fmt.Errorf("yaml marshal failed for resource: %v: %v", resource.Name, err)
log.Println(errors.appendError(e)) log.Println(errors.appendError(e))
c.Resources[i].State = &manager.ResourceState{ c.Resources[i].State = &common.ResourceState{
Status: manager.Aborted, Status: common.Aborted,
Errors: []string{e.Error()}, Errors: []string{e.Error()},
} }
continue continue
@ -122,8 +122,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
e := fmt.Errorf("cannot start kubetcl for resource: %v: %v", resource.Name, err) e := fmt.Errorf("cannot start kubetcl for resource: %v: %v", resource.Name, err)
c.Resources[i].State = &manager.ResourceState{ c.Resources[i].State = &common.ResourceState{
Status: manager.Failed, Status: common.Failed,
Errors: []string{e.Error()}, Errors: []string{e.Error()},
} }
log.Println(errors.appendError(e)) log.Println(errors.appendError(e))
@ -137,8 +137,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
log.Println(resource.Name + " not found, treating as success for delete") log.Println(resource.Name + " not found, treating as success for delete")
} else { } else {
e := fmt.Errorf("kubetcl failed for resource: %v: %v: %v", resource.Name, err, combined.String()) e := fmt.Errorf("kubetcl failed for resource: %v: %v: %v", resource.Name, err, combined.String())
c.Resources[i].State = &manager.ResourceState{ c.Resources[i].State = &common.ResourceState{
Status: manager.Failed, Status: common.Failed,
Errors: []string{e.Error()}, Errors: []string{e.Error()},
} }
log.Println(errors.appendError(e)) log.Println(errors.appendError(e))
@ -147,7 +147,7 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string,
} }
output = append(output, combined.String()) output = append(output, combined.String())
c.Resources[i].State = &manager.ResourceState{Status: manager.Created} c.Resources[i].State = &common.ResourceState{Status: common.Created}
log.Printf("kubectl succeeded for resource: %v: SysTime: %v UserTime: %v\n%v", log.Printf("kubectl succeeded for resource: %v: SysTime: %v UserTime: %v\n%v",
resource.Name, cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String()) resource.Name, cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String())
} }

34
util/templateutil.go
Unescape View File

@ -15,9 +15,39 @@ package util
import ( import (
"strings" "strings"
"log"
"github.com/kubernetes/deployment-manager/common"
) )
// IsTemplate returns whether a given type is a template. // IsTemplate returns whether a given type is a template.
func IsTemplate(t string) bool { func IsTemplate(t string, imports []*common.ImportFile) bool {
return strings.HasSuffix(t, ".py") || strings.HasSuffix(t, ".jinja") log.Printf("IsTemplate: %s : %+v", t, imports)
for _, imp := range imports {
log.Printf("Checking: %s", imp.Name)
if imp.Name == t {
return true
}
}
return false
}
// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type.
// For now, this means using github types:
// github.com/owner/repo/qualifier/type:version
// for example:
// github.com/kubernetes/application-dm-templates/storage/redis:v1
func IsGithubShortType(t string) bool {
if !strings.HasPrefix(t, "github.com/") {
return false
}
s := strings.Split(t, "/")
if len(s) != 5 {
return false
}
v := strings.Split(s[4], ":")
if len(v) != 2 {
return false
}
return true
} }

Write Preview
Loading…
Cancel
Save