diff --git a/manager/manager/types.go b/common/types.go similarity index 98% rename from manager/manager/types.go rename to common/types.go index 031892711..40605bc70 100644 --- a/manager/manager/types.go +++ b/common/types.go @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package manager +package common import ( "time" @@ -120,6 +120,7 @@ type Template struct { // ImportFile describes a base64 encoded file imported by a Template. type ImportFile struct { Name string `json:"name,omitempty"` + Path string `json:"path",omitempty` // Actual URL for the file Content string `json:"content"` } diff --git a/dm/dm.go b/dm/dm.go index 53bdfab7e..e7c5080f1 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -16,8 +16,8 @@ package main import ( "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/expandybird/expander" - "github.com/kubernetes/deployment-manager/manager/manager" "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/util" @@ -286,8 +286,8 @@ func isHttp(t string) bool { return strings.HasPrefix(t, "http://") || strings.HasPrefix(t, "https://") } -func loadTemplate(args []string) *expander.Template { - var template *expander.Template +func loadTemplate(args []string) *common.Template { + var template *common.Template var err error if len(args) < 2 { fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied") @@ -329,7 +329,7 @@ func getRegistryType(fullType string) *registry.Type { } } -func buildTemplateFromType(t registry.Type) *expander.Template { +func buildTemplateFromType(t registry.Type) *common.Template { downloadURL := getDownloadUrl(t) props := make(map[string]interface{}) @@ -355,7 +355,7 @@ func buildTemplateFromType(t registry.Type) *expander.Template { // Name the deployment after the type name. name := fmt.Sprintf("%s:%s", t.Name, t.Version) - config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{ + config := common.Configuration{Resources: []*common.Resource{&common.Resource{ Name: name, Type: downloadURL, Properties: props, @@ -366,14 +366,14 @@ func buildTemplateFromType(t registry.Type) *expander.Template { log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config) } - return &expander.Template{ + return &common.Template{ Name: name, Content: string(y), // No imports, as this is a single type from repository. } } -func marshalTemplate(template *expander.Template) io.ReadCloser { +func marshalTemplate(template *common.Template) io.ReadCloser { j, err := json.Marshal(template) if err != nil { log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err) diff --git a/expandybird/expander/expander.go b/expandybird/expander/expander.go index 5a952bfef..51f518b21 100644 --- a/expandybird/expander/expander.go +++ b/expandybird/expander/expander.go @@ -23,11 +23,12 @@ import ( "path/filepath" "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/common" ) // Expander abstracts interactions with the expander and deployer services. type Expander interface { - ExpandTemplate(template *Template) (string, error) + ExpandTemplate(template *common.Template) (string, error) } type expander struct { @@ -39,24 +40,10 @@ func NewExpander(binary string) Expander { return &expander{binary} } -// ImportFile describes a file that we import into our templates -// TODO: Encode the Content so that it doesn't get mangled. -type ImportFile struct { - Name string `json:"name,omitempty"` - Content string `json:"content"` -} - -// A Template defines a single deployment. -type Template struct { - Name string `json:"name"` - Content string `json:"content"` - Imports []*ImportFile `json:"imports"` -} - // NewTemplateFromRootTemplate creates and returns a new template whose content // and imported files are constructed from reading the root template, parsing out // the imports section and reading the imports from there -func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) { +func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) { templateDir := filepath.Dir(templateFileName) content, err := ioutil.ReadFile(templateFileName) if err != nil { @@ -85,14 +72,14 @@ func NewTemplateFromRootTemplate(templateFileName string) (*Template, error) { func NewTemplateFromFileNames( templateFileName string, importFileNames []string, -) (*Template, error) { +) (*common.Template, error) { name := path.Base(templateFileName) content, err := ioutil.ReadFile(templateFileName) if err != nil { return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName) } - imports := []*ImportFile{} + imports := []*common.ImportFile{} for _, importFileName := range importFileNames { importFileData, err := ioutil.ReadFile(importFileName) if err != nil { @@ -100,13 +87,13 @@ func NewTemplateFromFileNames( } imports = append(imports, - &ImportFile{ + &common.ImportFile{ Name: path.Base(importFileName), Content: string(importFileData), }) } - return &Template{ + return &common.Template{ Name: name, Content: string(content), Imports: imports, @@ -190,7 +177,7 @@ func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) { // ExpandTemplate passes the given configuration to the expander and returns the // expanded configuration as a string on success. -func (e *expander) ExpandTemplate(template *Template) (string, error) { +func (e *expander) ExpandTemplate(template *common.Template) (string, error) { if e.ExpansionBinary == "" { message := fmt.Sprintf("expansion binary cannot be empty") return "", fmt.Errorf("error expanding template %s: %s", template.Name, message) @@ -216,7 +203,7 @@ func (e *expander) ExpandTemplate(template *Template) (string, error) { } for _, imp := range template.Imports { - cmd.Args = append(cmd.Args, imp.Name, imp.Content) + cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content) } if err := cmd.Start(); err != nil { diff --git a/expandybird/expander/expander_test.go b/expandybird/expander/expander_test.go index 83ee552f9..26efc634d 100644 --- a/expandybird/expander/expander_test.go +++ b/expandybird/expander/expander_test.go @@ -19,6 +19,8 @@ import ( "reflect" "strings" "testing" + + "github.com/kubernetes/deployment-manager/common" ) const invalidFileName = "afilethatdoesnotexist" @@ -36,7 +38,7 @@ type ExpanderTestCase struct { ExpectedError string } -func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *Template { +func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template { template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames) if err != nil { t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err) diff --git a/expandybird/expansion/expansion.py b/expandybird/expansion/expansion.py index 162ebbcd0..7f74c43f4 100755 --- a/expandybird/expansion/expansion.py +++ b/expandybird/expansion/expansion.py @@ -136,7 +136,7 @@ def _ProcessResource(resource, imports, env, validate_schema=False): layout = {'name': resource['name'], 'type': resource['type']} - if IsTemplate(resource['type']) and resource['type'] in imports: + if resource['type'] in imports: # A template resource, which contains sub-resources. expanded_template = ExpandTemplate(resource, imports, env, validate_schema) @@ -183,11 +183,6 @@ def _ValidateUniqueNames(template_resources, template_name='config'): # If this resource doesn't have a name, we will report that error later -def IsTemplate(resource_type): - """Returns whether a given resource type is a Template.""" - return resource_type.endswith('.py') or resource_type.endswith('.jinja') - - def ExpandTemplate(resource, imports, env, validate_schema=False): """Expands a template, calling expansion mechanism based on type. @@ -206,6 +201,7 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): ExpansionError: if there is any error occurred during expansion """ source_file = resource['type'] + path = resource['type'] # Look for Template in imports. if source_file not in imports: @@ -213,6 +209,12 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): source_file, 'Unable to find source file %s in imports.' % (source_file)) + # source_file could be a short version of the template (say github short name) + # so we need to potentially map this into the fully resolvable name. + if 'path' in imports[source_file]: + if imports[source_file]['path']: + path = imports[source_file]['path'] + resource['imports'] = imports # Populate the additional environment variables. @@ -231,13 +233,13 @@ def ExpandTemplate(resource, imports, env, validate_schema=False): except schema_validation.ValidationErrors as e: raise ExpansionError(resource['name'], e.message) - if source_file.endswith('jinja'): + if path.endswith('jinja'): expanded_template = ExpandJinja( - source_file, imports[source_file], resource, imports) - elif source_file.endswith('py'): + source_file, imports[source_file]['content'], resource, imports) + elif path.endswith('py'): # This is a Python template. expanded_template = ExpandPython( - imports[source_file], source_file, resource) + imports[source_file]['content'], source_file, resource) else: # The source file is not a jinja file or a python file. # This in fact should never happen due to the IsTemplate check above. @@ -262,8 +264,8 @@ def ExpandJinja(file_name, source_template, resource, imports): source_template: string, the content of jinja file to be render resource: resource object, the resource that contains parameters to the jinja file - imports: map from string to string, the map of imported files names - and contents + imports: map from string to map {name, path}, the map of imported files names + fully resolved path and contents Returns: The final expanded template Raises: @@ -362,9 +364,10 @@ def main(): print >>sys.stderr, 'Invalid import definition at argv pos %d' % idx sys.exit(1) name = sys.argv[idx] - value = sys.argv[idx + 1] - imports[name] = value - idx += 2 + path = sys.argv[idx + 1] + value = sys.argv[idx + 2] + imports[name] = {'content': value, 'path': path} + idx += 3 env = {} env['deployment'] = os.environ['DEPLOYMENT_NAME'] diff --git a/expandybird/expansion/sandbox_loader.py b/expandybird/expansion/sandbox_loader.py index 7b4dbba4e..bbed8473a 100644 --- a/expandybird/expansion/sandbox_loader.py +++ b/expandybird/expansion/sandbox_loader.py @@ -95,7 +95,7 @@ def process_imports(imports): # Now build the hierarchical modules. for k in imports.keys(): - if imports[k].endswith('.jinja'): + if imports[k]['path'].endswith('.jinja'): continue # Normalize paths and trim .py extension, if any. normalized = os.path.splitext(os.path.normpath(k))[0] diff --git a/expandybird/service/service.go b/expandybird/service/service.go index 3e1c0a78a..760050283 100644 --- a/expandybird/service/service.go +++ b/expandybird/service/service.go @@ -15,6 +15,7 @@ package service import ( "github.com/kubernetes/deployment-manager/expandybird/expander" + "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/util" "errors" @@ -40,7 +41,7 @@ func NewService(handler restful.RouteFunction) *Service { webService.Produces(restful.MIME_JSON, restful.MIME_XML) webService.Route(webService.POST("/expand").To(handler). Doc("Expand a template."). - Reads(&expander.Template{})) + Reads(&common.Template{})) return &Service{webService} } @@ -60,7 +61,7 @@ func (s *Service) Register(container *restful.Container) { func NewExpansionHandler(backend expander.Expander) restful.RouteFunction { return func(req *restful.Request, resp *restful.Response) { util.LogHandlerEntry("expandybird: expand", req.Request) - template := &expander.Template{} + template := &common.Template{} if err := req.ReadEntity(&template); err != nil { logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp) return diff --git a/expandybird/service/service_test.go b/expandybird/service/service_test.go index cd8aec878..01d636941 100644 --- a/expandybird/service/service_test.go +++ b/expandybird/service/service_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/kubernetes/deployment-manager/expandybird/expander" + "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/util" restful "github.com/emicklei/go-restful" @@ -180,7 +181,7 @@ type mockExpander struct { // ExpandTemplate passes the given configuration to the expander and returns the // expanded configuration as a string on success. -func (e *mockExpander) ExpandTemplate(template *expander.Template) (string, error) { +func (e *mockExpander) ExpandTemplate(template *common.Template) (string, error) { switch template.Name { case "InvalidFileName.yaml": return "", fmt.Errorf("expansion error") diff --git a/manager/deployments.go b/manager/deployments.go index 54519005b..e09e672c9 100644 --- a/manager/deployments.go +++ b/manager/deployments.go @@ -22,6 +22,7 @@ import ( "log" "net" "net/http" + "net/url" "os" "strings" @@ -29,6 +30,7 @@ import ( "github.com/gorilla/mux" "github.com/kubernetes/deployment-manager/manager/manager" + "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/manager/repository" "github.com/kubernetes/deployment-manager/util" ) @@ -198,17 +200,16 @@ func putDeploymentHandlerFunc(w http.ResponseWriter, r *http.Request) { func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler string) (string, error) { vars := mux.Vars(r) - variable, ok := vars[variable] + retVariable, ok := vars[variable] if !ok { e := fmt.Errorf("%s parameter not found in URL", variable) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) return "", e } - - return variable, nil + return retVariable, nil } -func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manager.Template { +func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *common.Template { util.LogHandlerEntry(handler, r) b := io.LimitReader(r.Body, *maxLength*1024) y, err := ioutil.ReadAll(b) @@ -237,7 +238,7 @@ func getTemplate(w http.ResponseWriter, r *http.Request, handler string) *manage return nil } - t := &manager.Template{} + t := &common.Template{} if err := json.Unmarshal(j, t); err != nil { e := fmt.Errorf("%v\n%v", err, string(j)) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) diff --git a/manager/manager/deployer.go b/manager/manager/deployer.go index 0b86310b6..ae80b047a 100644 --- a/manager/manager/deployer.go +++ b/manager/manager/deployer.go @@ -25,14 +25,15 @@ import ( "strings" "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/common" ) // Deployer abstracts interactions with the expander and deployer services. type Deployer interface { - GetConfiguration(cached *Configuration) (*Configuration, error) - CreateConfiguration(configuration *Configuration) (*Configuration, error) - DeleteConfiguration(configuration *Configuration) (*Configuration, error) - PutConfiguration(configuration *Configuration) (*Configuration, error) + GetConfiguration(cached *common.Configuration) (*common.Configuration, error) + CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) + DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) + PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) } // NewDeployer returns a new initialized Deployer. @@ -54,9 +55,9 @@ type formatter func(err error) error // GetConfiguration reads and returns the actual configuration // of the resources described by a cached configuration. -func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, error) { +func (d *deployer) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) { errors := &Error{} - actual := &Configuration{} + actual := &common.Configuration{} for _, resource := range cached.Resources { rtype := url.QueryEscape(resource.Type) rname := url.QueryEscape(resource.Name) @@ -70,7 +71,7 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro } if len(body) != 0 { - result := &Resource{Name: resource.Name, Type: resource.Type} + result := &common.Resource{Name: resource.Name, Type: resource.Type} if err := yaml.Unmarshal(body, &result.Properties); err != nil { return nil, fmt.Errorf("cannot get configuration for resource (%v)", err) } @@ -88,22 +89,22 @@ func (d *deployer) GetConfiguration(cached *Configuration) (*Configuration, erro // CreateConfiguration deploys the set of resources described by a configuration and returns // the Configuration with status for each resource filled in. -func (d *deployer) CreateConfiguration(configuration *Configuration) (*Configuration, error) { +func (d *deployer) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) { return d.callServiceWithConfiguration("POST", "create", configuration) } // DeleteConfiguration deletes the set of resources described by a configuration. -func (d *deployer) DeleteConfiguration(configuration *Configuration) (*Configuration, error) { +func (d *deployer) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) { return d.callServiceWithConfiguration("DELETE", "delete", configuration) } // PutConfiguration replaces the set of resources described by a configuration and returns // the Configuration with status for each resource filled in. -func (d *deployer) PutConfiguration(configuration *Configuration) (*Configuration, error) { +func (d *deployer) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) { return d.callServiceWithConfiguration("PUT", "replace", configuration) } -func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *Configuration) (*Configuration, error) { +func (d *deployer) callServiceWithConfiguration(method, operation string, configuration *common.Configuration) (*common.Configuration, error) { callback := func(e error) error { return fmt.Errorf("cannot %s configuration: %s", operation, e) } @@ -120,7 +121,7 @@ func (d *deployer) callServiceWithConfiguration(method, operation string, config return nil, err } - result := &Configuration{} + result := &common.Configuration{} if len(resp) != 0 { if err := yaml.Unmarshal(resp, &result); err != nil { return nil, fmt.Errorf("cannot unmarshal response: (%v)", err) diff --git a/manager/manager/deployer_test.go b/manager/manager/deployer_test.go index 244255a8e..5ba9a9f03 100644 --- a/manager/manager/deployer_test.go +++ b/manager/manager/deployer_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/kubernetes/deployment-manager/util" + "github.com/kubernetes/deployment-manager/common" "github.com/ghodss/yaml" ) @@ -250,8 +251,8 @@ func TestPutConfiguration(t *testing.T) { } } -func getValidConfiguration(t *testing.T) *Configuration { - valid := &Configuration{} +func getValidConfiguration(t *testing.T) *common.Configuration { + valid := &common.Configuration{} err := yaml.Unmarshal(validConfigurationTestCaseData, valid) if err != nil { t.Errorf("cannot unmarshal test case data:%s\n", err) @@ -266,7 +267,7 @@ func deployerErrorHandler(w http.ResponseWriter, r *http.Request) { } func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) { - valid := &Configuration{} + valid := &common.Configuration{} err := yaml.Unmarshal(validConfigurationTestCaseData, valid) if err != nil { status := fmt.Sprintf("cannot unmarshal test case data:%s", err) @@ -282,7 +283,7 @@ func deployerSuccessHandler(w http.ResponseWriter, r *http.Request) { return } - result := &Configuration{} + result := &common.Configuration{} if err := yaml.Unmarshal(body, result); err != nil { status := fmt.Sprintf("cannot unmarshal request body:%s", err) http.Error(w, status, http.StatusInternalServerError) diff --git a/manager/manager/expander.go b/manager/manager/expander.go index c906e6306..2d291a530 100644 --- a/manager/manager/expander.go +++ b/manager/manager/expander.go @@ -22,6 +22,7 @@ import ( "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/util" + "github.com/kubernetes/deployment-manager/common" ) const ( @@ -31,13 +32,13 @@ const ( // ExpandedTemplate is the structure returned by the expansion service. type ExpandedTemplate struct { - Config *Configuration `json:"config"` - Layout *Layout `json:"layout"` + Config *common.Configuration `json:"config"` + Layout *common.Layout `json:"layout"` } // Expander abstracts interactions with the expander and deployer services. type Expander interface { - ExpandTemplate(t *Template) (*ExpandedTemplate, error) + ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) } // NewExpander returns a new initialized Expander. @@ -54,7 +55,7 @@ func (e *expander) getBaseURL() string { return fmt.Sprintf("%s/expand", e.expanderURL) } -func expanderError(t *Template, err error) error { +func expanderError(t *common.Template, err error) error { return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content) } @@ -90,14 +91,14 @@ func expanderError(t *Template, err error) error { // between the name#template key to exist in the layout given a particular choice of naming. // In practice, it would be nearly impossible to hit, but consider including properties/name/type // into a hash of sorts to make this robust... -func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*LayoutResource { - ret := map[string]*LayoutResource{} +func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource { + ret := map[string]*common.LayoutResource{} toVisit := l.Resources for len(toVisit) > 0 { lr := toVisit[0] nodeKey := lr.Resource.Name + layoutNodeKeySeparator + lr.Resource.Type - if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type) { + if len(lr.Layout.Resources) == 0 && util.IsTemplate(lr.Resource.Type, imports) { ret[nodeKey] = lr } else if toReplace[nodeKey] != nil { toReplace[nodeKey].Resources = lr.Resources @@ -112,20 +113,20 @@ func walkLayout(l *Layout, toReplace map[string]*LayoutResource) map[string]*Lay // ExpandTemplate expands the supplied template, and returns a configuration. // It will also update the imports in the provided template if any were added // during type resolution. -func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { +func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) { // We have a fencepost problem here. // 1. Start by trying to resolve any missing templates // 2. Expand the configuration using all the of the imports available to us at this point // 3. Expansion may yield additional templates, so we run the type resolution again // 4. If type resolution resulted in new imports being available, return to 2. - config := &Configuration{} + config := &common.Configuration{} if err := yaml.Unmarshal([]byte(t.Content), config); err != nil { e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content) return nil, e } - var finalLayout *Layout - needResolve := map[string]*LayoutResource{} + var finalLayout *common.Layout + needResolve := map[string]*common.LayoutResource{} // Start things off by attempting to resolve the templates in a first pass. newImp, err := e.typeResolver.ResolveTypes(config, t.Imports) @@ -151,7 +152,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { if finalLayout == nil { finalLayout = result.Layout } - needResolve = walkLayout(result.Layout, needResolve) + needResolve = walkLayout(result.Layout, t.Imports, needResolve) newImp, err = e.typeResolver.ResolveTypes(result.Config, t.Imports) if err != nil { @@ -170,7 +171,7 @@ func (e *expander) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { } } -func (e *expander) expandTemplate(t *Template) (*ExpandedTemplate, error) { +func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) { j, err := json.Marshal(t) if err != nil { return nil, err diff --git a/manager/manager/expander_test.go b/manager/manager/expander_test.go index 11ba5ec58..a4760c34b 100644 --- a/manager/manager/expander_test.go +++ b/manager/manager/expander_test.go @@ -24,16 +24,17 @@ import ( "testing" "github.com/kubernetes/deployment-manager/util" + "github.com/kubernetes/deployment-manager/common" "github.com/ghodss/yaml" ) type mockResolver struct { - responses [][]*ImportFile + responses [][]*common.ImportFile t *testing.T } -func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*ImportFile, error) { +func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) { if len(r.responses) < 1 { return nil, nil } @@ -43,7 +44,7 @@ func (r *mockResolver) ResolveTypes(c *Configuration, i []*ImportFile) ([]*Impor return ret, nil } -var validTemplateTestCaseData = Template{ +var validTemplateTestCaseData = common.Template{ Name: "TestTemplate", Content: string(validContentTestCaseData), Imports: validImportFilesTestCaseData, @@ -59,11 +60,19 @@ resources: test-property: test-value `) -var validImportFilesTestCaseData = []*ImportFile{ - &ImportFile{ +var validImportFilesTestCaseData = []*common.ImportFile{ + &common.ImportFile{ Name: "test-type.py", Content: "test-type.py validTemplateTestCaseData content", }, + &common.ImportFile{ + Name: "test.py", + Content: "test.py validTemplateTestCaseData content", + }, + &common.ImportFile{ + Name: "test2.py", + Content: "test2.py validTemplateTestCaseData content", + }, } var validConfigTestCaseData = []byte(` @@ -210,7 +219,7 @@ layout: test: test ` -var roundTripTemplate = Template{ +var roundTripTemplate = common.Template{ Name: "TestTemplate", Content: roundTripContent, Imports: nil, @@ -249,9 +258,9 @@ func TestExpandTemplate(t *testing.T) { "expect success for ExpandTemplate with two expansions", "", roundTripHandler, - &mockResolver{[][]*ImportFile{ + &mockResolver{[][]*common.ImportFile{ {}, - {&ImportFile{Name: "test.py"}}, + {&common.ImportFile{Name: "test.py"}}, }, t}, roundTripResponse, }, @@ -334,7 +343,7 @@ func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) { return } - template := &Template{} + template := &common.Template{} if err := json.Unmarshal(body, template); err != nil { status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body) http.Error(w, status, http.StatusInternalServerError) diff --git a/manager/manager/manager.go b/manager/manager/manager.go index 170286871..4c2f2f486 100644 --- a/manager/manager/manager.go +++ b/manager/manager/manager.go @@ -17,35 +17,37 @@ import ( "fmt" "log" "time" + + "github.com/kubernetes/deployment-manager/common" ) // Manager manages a persistent set of Deployments. type Manager interface { - ListDeployments() ([]Deployment, error) - GetDeployment(name string) (*Deployment, error) - CreateDeployment(t *Template) (*Deployment, error) - DeleteDeployment(name string, forget bool) (*Deployment, error) - PutDeployment(name string, t *Template) (*Deployment, error) - ListManifests(deploymentName string) (map[string]*Manifest, error) - GetManifest(deploymentName string, manifest string) (*Manifest, error) - Expand(t *Template) (*Manifest, error) + ListDeployments() ([]common.Deployment, error) + GetDeployment(name string) (*common.Deployment, error) + CreateDeployment(t *common.Template) (*common.Deployment, error) + DeleteDeployment(name string, forget bool) (*common.Deployment, error) + PutDeployment(name string, t *common.Template) (*common.Deployment, error) + ListManifests(deploymentName string) (map[string]*common.Manifest, error) + GetManifest(deploymentName string, manifest string) (*common.Manifest, error) + Expand(t *common.Template) (*common.Manifest, error) ListTypes() []string - ListInstances(typeName string) []*TypeInstance + ListInstances(typeName string) []*common.TypeInstance } type manager struct { expander Expander deployer Deployer - repository Repository + repository common.Repository } // NewManager returns a new initialized Manager. -func NewManager(expander Expander, deployer Deployer, repository Repository) Manager { +func NewManager(expander Expander, deployer Deployer, repository common.Repository) Manager { return &manager{expander, deployer, repository} } // ListDeployments returns the list of deployments -func (m *manager) ListDeployments() ([]Deployment, error) { +func (m *manager) ListDeployments() ([]common.Deployment, error) { l, err := m.repository.ListDeployments() if err != nil { return nil, err @@ -55,7 +57,7 @@ func (m *manager) ListDeployments() ([]Deployment, error) { // GetDeployment retrieves the configuration stored for a given deployment // as well as the current configuration from the cluster. -func (m *manager) GetDeployment(name string) (*Deployment, error) { +func (m *manager) GetDeployment(name string) (*common.Deployment, error) { d, err := m.repository.GetDeployment(name) if err != nil { return nil, err @@ -66,7 +68,7 @@ func (m *manager) GetDeployment(name string) (*Deployment, error) { // ListManifests retrieves the manifests for a given deployment // of each of the deployments in the repository and returns the deployments. -func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, error) { +func (m *manager) ListManifests(deploymentName string) (map[string]*common.Manifest, error) { l, err := m.repository.ListManifests(deploymentName) if err != nil { return nil, err @@ -76,7 +78,7 @@ func (m *manager) ListManifests(deploymentName string) (map[string]*Manifest, er } // GetManifest retrieves the specified manifest for a given deployment -func (m *manager) GetManifest(deploymentName string, manifestName string) (*Manifest, error) { +func (m *manager) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) { d, err := m.repository.GetManifest(deploymentName, manifestName) if err != nil { return nil, err @@ -88,7 +90,7 @@ func (m *manager) GetManifest(deploymentName string, manifestName string) (*Mani // CreateDeployment expands the supplied template, creates the resulting // configuration in the cluster, creates a new deployment that tracks it, // and stores the deployment in the repository. Returns the deployment. -func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { +func (m *manager) CreateDeployment(t *common.Template) (*common.Deployment, error) { log.Printf("Creating deployment: %s", t.Name) _, err := m.repository.CreateDeployment(t.Name) if err != nil { @@ -99,7 +101,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { manifest, err := m.createManifest(t) if err != nil { log.Printf("Manifest creation failed: %v", err) - m.repository.SetDeploymentStatus(t.Name, FailedStatus) + m.repository.SetDeploymentStatus(t.Name, common.FailedStatus) return nil, err } @@ -107,7 +109,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { if createErr != nil { // Deployment failed, mark as failed log.Printf("CreateConfiguration failed: %v", err) - m.repository.SetDeploymentStatus(t.Name, FailedStatus) + m.repository.SetDeploymentStatus(t.Name, common.FailedStatus) // If we failed before being able to create some of the resources, then // return the failure as such. Otherwise, we're going to add the manifest // and hence resource specific errors down below. @@ -115,7 +117,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { return nil, createErr } } else { - m.repository.SetDeploymentStatus(t.Name, DeployedStatus) + m.repository.SetDeploymentStatus(t.Name, common.DeployedStatus) } // Update the manifest with the actual state of the reified resources @@ -123,7 +125,7 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { aErr := m.repository.AddManifest(t.Name, manifest) if aErr != nil { log.Printf("AddManifest failed %v", aErr) - m.repository.SetDeploymentStatus(t.Name, FailedStatus) + m.repository.SetDeploymentStatus(t.Name, common.FailedStatus) // If there's an earlier error, return that instead since it contains // more applicable error message. Adding manifest failure is more akin // to a check fail (either deployment doesn't exist, or a manifest with the same @@ -141,14 +143,14 @@ func (m *manager) CreateDeployment(t *Template) (*Deployment, error) { return m.repository.GetValidDeployment(t.Name) } -func (m *manager) createManifest(t *Template) (*Manifest, error) { +func (m *manager) createManifest(t *common.Template) (*common.Manifest, error) { et, err := m.expander.ExpandTemplate(t) if err != nil { log.Printf("Expansion failed %v", err) return nil, err } - return &Manifest{ + return &common.Manifest{ Name: generateManifestName(), Deployment: t.Name, InputConfig: t, @@ -157,10 +159,10 @@ func (m *manager) createManifest(t *Template) (*Manifest, error) { }, nil } -func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *Layout) { +func (m *manager) addTypeInstances(deploymentName string, manifestName string, layout *common.Layout) { m.repository.ClearTypeInstances(deploymentName) - instances := make(map[string][]*TypeInstance) + instances := make(map[string][]*common.TypeInstance) for i, r := range layout.Resources { addTypeInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i)) } @@ -168,9 +170,9 @@ func (m *manager) addTypeInstances(deploymentName string, manifestName string, l m.repository.SetTypeInstances(deploymentName, instances) } -func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource, deploymentName string, manifestName string, jsonPath string) { +func addTypeInstances(instances *map[string][]*common.TypeInstance, r *common.LayoutResource, deploymentName string, manifestName string, jsonPath string) { // Add this resource. - inst := &TypeInstance{ + inst := &common.TypeInstance{ Name: r.Name, Type: r.Type, Deployment: deploymentName, @@ -189,7 +191,7 @@ func addTypeInstances(instances *map[string][]*TypeInstance, r *LayoutResource, // the supplied identifier from the cluster.repository. If forget is true, then // the deployment is removed from the repository. Otherwise, it is marked // as deleted and retained. -func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error) { +func (m *manager) DeleteDeployment(name string, forget bool) (*common.Deployment, error) { log.Printf("Deleting deployment: %s", name) d, err := m.repository.GetValidDeployment(name) if err != nil { @@ -210,7 +212,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error } // Create an empty manifest since resources have been deleted. - err = m.repository.AddManifest(name, &Manifest{Deployment: name, Name: generateManifestName()}) + err = m.repository.AddManifest(name, &common.Manifest{Deployment: name, Name: generateManifestName()}) if err != nil { log.Printf("Failed to add empty manifest") return nil, err @@ -230,7 +232,7 @@ func (m *manager) DeleteDeployment(name string, forget bool) (*Deployment, error // PutDeployment replaces the configuration of the deployment with // the supplied identifier in the cluster, and returns the deployment. -func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) { +func (m *manager) PutDeployment(name string, t *common.Template) (*common.Deployment, error) { _, err := m.repository.GetValidDeployment(name) if err != nil { return nil, err @@ -239,20 +241,20 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) { manifest, err := m.createManifest(t) if err != nil { log.Printf("Manifest creation failed: %v", err) - m.repository.SetDeploymentStatus(name, FailedStatus) + m.repository.SetDeploymentStatus(name, common.FailedStatus) return nil, err } actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig) if err != nil { - m.repository.SetDeploymentStatus(name, FailedStatus) + m.repository.SetDeploymentStatus(name, common.FailedStatus) return nil, err } manifest.ExpandedConfig = actualConfig err = m.repository.AddManifest(t.Name, manifest) if err != nil { - m.repository.SetDeploymentStatus(name, FailedStatus) + m.repository.SetDeploymentStatus(name, common.FailedStatus) return nil, err } @@ -262,14 +264,14 @@ func (m *manager) PutDeployment(name string, t *Template) (*Deployment, error) { return m.repository.GetValidDeployment(t.Name) } -func (m *manager) Expand(t *Template) (*Manifest, error) { +func (m *manager) Expand(t *common.Template) (*common.Manifest, error) { et, err := m.expander.ExpandTemplate(t) if err != nil { log.Printf("Expansion failed %v", err) return nil, err } - return &Manifest{ + return &common.Manifest{ ExpandedConfig: et.Config, Layout: et.Layout, }, nil @@ -279,7 +281,7 @@ func (m *manager) ListTypes() []string { return m.repository.ListTypes() } -func (m *manager) ListInstances(typeName string) []*TypeInstance { +func (m *manager) ListInstances(typeName string) []*common.TypeInstance { return m.repository.GetTypeInstances(typeName) } diff --git a/manager/manager/manager_test.go b/manager/manager/manager_test.go index f31d56801..67e1441c0 100644 --- a/manager/manager/manager_test.go +++ b/manager/manager/manager_test.go @@ -18,25 +18,27 @@ import ( "reflect" "strings" "testing" + + "github.com/kubernetes/deployment-manager/common" ) -var template = Template{Name: "test", Content: "test"} +var template = common.Template{Name: "test", Content: "test"} -var layout = Layout{ - Resources: []*LayoutResource{&LayoutResource{Resource: Resource{Name: "test", Type: "test"}}}, +var layout = common.Layout{ + Resources: []*common.LayoutResource{&common.LayoutResource{Resource: common.Resource{Name: "test", Type: "test"}}}, } -var configuration = Configuration{ - Resources: []*Resource{&Resource{Name: "test", Type: "test"}}, +var configuration = common.Configuration{ + Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test"}}, } -var resourcesWithSuccessState = Configuration{ - Resources: []*Resource{&Resource{Name: "test", Type: "test", State: &ResourceState{Status: Created}}}, +var resourcesWithSuccessState = common.Configuration{ + Resources: []*common.Resource{&common.Resource{Name: "test", Type: "test", State: &common.ResourceState{Status: common.Created}}}, } -var resourcesWithFailureState = Configuration{ - Resources: []*Resource{&Resource{ +var resourcesWithFailureState = common.Configuration{ + Resources: []*common.Resource{&common.Resource{ Name: "test", Type: "test", - State: &ResourceState{ - Status: Failed, + State: &common.ResourceState{ + Status: common.Failed, Errors: []string{"test induced error"}, }, }}, @@ -49,14 +51,14 @@ var expandedConfig = ExpandedTemplate{ var deploymentName = "deployment" var manifestName = "manifest-2" -var manifest = Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout} -var manifestMap = map[string]*Manifest{manifest.Name: &manifest} +var manifest = common.Manifest{Name: manifestName, ExpandedConfig: &configuration, Layout: &layout} +var manifestMap = map[string]*common.Manifest{manifest.Name: &manifest} -var deployment = Deployment{ +var deployment = common.Deployment{ Name: "test", } -var deploymentList = []Deployment{deployment, {Name: "test2"}} +var deploymentList = []common.Deployment{deployment, {Name: "test2"}} var typeInstMap = map[string][]string{"test": []string{"test"}} @@ -64,7 +66,7 @@ var errTest = errors.New("test") type expanderStub struct{} -func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, error) { +func (expander *expanderStub) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) { if reflect.DeepEqual(*t, template) { return &expandedConfig, nil } @@ -74,17 +76,17 @@ func (expander *expanderStub) ExpandTemplate(t *Template) (*ExpandedTemplate, er type deployerStub struct { FailCreate bool - Created []*Configuration + Created []*common.Configuration FailDelete bool - Deleted []*Configuration + Deleted []*common.Configuration FailCreateResource bool } func (deployer *deployerStub) reset() { deployer.FailCreate = false - deployer.Created = make([]*Configuration, 0) + deployer.Created = make([]*common.Configuration, 0) deployer.FailDelete = false - deployer.Deleted = make([]*Configuration, 0) + deployer.Deleted = make([]*common.Configuration, 0) deployer.FailCreateResource = false } @@ -93,11 +95,11 @@ func newDeployerStub() *deployerStub { return ret } -func (deployer *deployerStub) GetConfiguration(cached *Configuration) (*Configuration, error) { +func (deployer *deployerStub) GetConfiguration(cached *common.Configuration) (*common.Configuration, error) { return nil, nil } -func (deployer *deployerStub) CreateConfiguration(configuration *Configuration) (*Configuration, error) { +func (deployer *deployerStub) CreateConfiguration(configuration *common.Configuration) (*common.Configuration, error) { if deployer.FailCreate { return nil, errTest } @@ -109,7 +111,7 @@ func (deployer *deployerStub) CreateConfiguration(configuration *Configuration) return &resourcesWithSuccessState, nil } -func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration) (*Configuration, error) { +func (deployer *deployerStub) DeleteConfiguration(configuration *common.Configuration) (*common.Configuration, error) { if deployer.FailDelete { return nil, errTest } @@ -117,34 +119,34 @@ func (deployer *deployerStub) DeleteConfiguration(configuration *Configuration) return nil, nil } -func (deployer *deployerStub) PutConfiguration(configuration *Configuration) (*Configuration, error) { +func (deployer *deployerStub) PutConfiguration(configuration *common.Configuration) (*common.Configuration, error) { return nil, nil } type repositoryStub struct { FailListDeployments bool Created []string - ManifestAdd map[string]*Manifest + ManifestAdd map[string]*common.Manifest Deleted []string GetValid []string TypeInstances map[string][]string TypeInstancesCleared bool GetTypeInstancesCalled bool ListTypesCalled bool - DeploymentStatuses []DeploymentStatus + DeploymentStatuses []common.DeploymentStatus } func (repository *repositoryStub) reset() { repository.FailListDeployments = false repository.Created = make([]string, 0) - repository.ManifestAdd = make(map[string]*Manifest) + repository.ManifestAdd = make(map[string]*common.Manifest) repository.Deleted = make([]string, 0) repository.GetValid = make([]string, 0) repository.TypeInstances = make(map[string][]string) repository.TypeInstancesCleared = false repository.GetTypeInstancesCalled = false repository.ListTypesCalled = false - repository.DeploymentStatuses = make([]DeploymentStatus, 0) + repository.DeploymentStatuses = make([]common.DeploymentStatus, 0) } func newRepositoryStub() *repositoryStub { @@ -152,14 +154,14 @@ func newRepositoryStub() *repositoryStub { return ret } -func (repository *repositoryStub) ListDeployments() ([]Deployment, error) { +func (repository *repositoryStub) ListDeployments() ([]common.Deployment, error) { if repository.FailListDeployments { return deploymentList, errTest } return deploymentList, nil } -func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) { +func (repository *repositoryStub) GetDeployment(d string) (*common.Deployment, error) { if d == deploymentName { return &deployment, nil } @@ -167,32 +169,32 @@ func (repository *repositoryStub) GetDeployment(d string) (*Deployment, error) { return nil, errTest } -func (repository *repositoryStub) GetValidDeployment(d string) (*Deployment, error) { +func (repository *repositoryStub) GetValidDeployment(d string) (*common.Deployment, error) { repository.GetValid = append(repository.GetValid, d) return &deployment, nil } -func (repository *repositoryStub) SetDeploymentStatus(name string, status DeploymentStatus) error { +func (repository *repositoryStub) SetDeploymentStatus(name string, status common.DeploymentStatus) error { repository.DeploymentStatuses = append(repository.DeploymentStatuses, status) return nil } -func (repository *repositoryStub) CreateDeployment(d string) (*Deployment, error) { +func (repository *repositoryStub) CreateDeployment(d string) (*common.Deployment, error) { repository.Created = append(repository.Created, d) return &deployment, nil } -func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*Deployment, error) { +func (repository *repositoryStub) DeleteDeployment(d string, forget bool) (*common.Deployment, error) { repository.Deleted = append(repository.Deleted, d) return &deployment, nil } -func (repository *repositoryStub) AddManifest(d string, manifest *Manifest) error { +func (repository *repositoryStub) AddManifest(d string, manifest *common.Manifest) error { repository.ManifestAdd[d] = manifest return nil } -func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error) { +func (repository *repositoryStub) GetLatestManifest(d string) (*common.Manifest, error) { if d == deploymentName { return repository.ManifestAdd[d], nil } @@ -200,7 +202,7 @@ func (repository *repositoryStub) GetLatestManifest(d string) (*Manifest, error) return nil, errTest } -func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest, error) { +func (repository *repositoryStub) ListManifests(d string) (map[string]*common.Manifest, error) { if d == deploymentName { return manifestMap, nil } @@ -208,7 +210,7 @@ func (repository *repositoryStub) ListManifests(d string) (map[string]*Manifest, return nil, errTest } -func (repository *repositoryStub) GetManifest(d string, m string) (*Manifest, error) { +func (repository *repositoryStub) GetManifest(d string, m string) (*common.Manifest, error) { if d == deploymentName && m == manifestName { return &manifest, nil } @@ -221,16 +223,16 @@ func (r *repositoryStub) ListTypes() []string { return []string{} } -func (r *repositoryStub) GetTypeInstances(t string) []*TypeInstance { +func (r *repositoryStub) GetTypeInstances(t string) []*common.TypeInstance { r.GetTypeInstancesCalled = true - return []*TypeInstance{} + return []*common.TypeInstance{} } func (r *repositoryStub) ClearTypeInstances(d string) { r.TypeInstancesCleared = true } -func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*TypeInstance) { +func (r *repositoryStub) SetTypeInstances(d string, is map[string][]*common.TypeInstance) { for k, _ := range is { r.TypeInstances[d] = append(r.TypeInstances[d], k) } @@ -326,7 +328,7 @@ func TestCreateDeployment(t *testing.T) { testDeployer.Created[0], configuration) } - if testRepository.DeploymentStatuses[0] != DeployedStatus { + if testRepository.DeploymentStatuses[0] != common.DeployedStatus { t.Fatal("CreateDeployment success did not mark deployment as deployed") } @@ -355,7 +357,7 @@ func TestCreateDeploymentCreationFailure(t *testing.T) { testRepository.Created[0]) } - if testRepository.DeploymentStatuses[0] != FailedStatus { + if testRepository.DeploymentStatuses[0] != common.FailedStatus { t.Fatal("CreateDeployment failure did not mark deployment as failed") } @@ -385,7 +387,7 @@ func TestCreateDeploymentCreationResourceFailure(t *testing.T) { testRepository.Created[0]) } - if testRepository.DeploymentStatuses[0] != FailedStatus { + if testRepository.DeploymentStatuses[0] != common.FailedStatus { t.Fatal("CreateDeployment failure did not mark deployment as failed") } diff --git a/manager/manager/typeresolver.go b/manager/manager/typeresolver.go index ab8dfd036..ba9397e08 100644 --- a/manager/manager/typeresolver.go +++ b/manager/manager/typeresolver.go @@ -16,8 +16,11 @@ package manager import ( "fmt" "net/http" + "regexp" "time" + "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/util" "github.com/ghodss/yaml" @@ -31,12 +34,14 @@ const ( // TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file // or primitive, and attempts to replace them with a template from a URL. type TypeResolver interface { - ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) + ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) } type typeResolver struct { getter util.HTTPClient maxUrls int + re *regexp.Regexp + rp registry.RegistryProvider } // NewTypeResolver returns a new initialized TypeResolver. @@ -48,10 +53,12 @@ func NewTypeResolver() TypeResolver { client.Timeout = timeout ret.getter = util.NewHTTPClient(3, client, util.NewSleeper()) ret.maxUrls = maxURLImports + ret.re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") + ret.rp = ®istry.DefaultRegistryProvider{} return ret } -func resolverError(c *Configuration, err error) error { +func resolverError(c *common.Configuration, err error) error { return fmt.Errorf("cannot resolve types in configuration %s due to: \n%s\n", c, err) } @@ -78,21 +85,23 @@ func performHTTPGet(g util.HTTPClient, u string, allowMissing bool) (content str // resolved type definitions in t.ImportFiles. Types can be either // primitive (i.e., built in), resolved (i.e., already t.ImportFiles), or remote // (i.e., described by a URL that must be fetched to resolve the type). -func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFile) ([]*ImportFile, error) { +func (tr *typeResolver) ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error) { existing := map[string]bool{} for _, v := range imports { existing[v.Name] = true } - fetched := map[string][]*ImportFile{} + fetched := map[string][]*common.ImportFile{} toFetch := make([]string, 0, tr.maxUrls) for _, r := range config.Resources { - // Only fetch HTTP URLs that we haven't already imported. - if util.IsHttpUrl(r.Type) && !existing[r.Type] { - toFetch = append(toFetch, r.Type) - - fetched[r.Type] = append(fetched[r.Type], &ImportFile{Name: r.Type}) - + // Map the type to a fetchable URL (if applicable) or skip it if it's a non-fetchable type (primitive for example). + u, err := tr.MapFetchableURL(r.Type) + if err != nil { + return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", r.Type, err)) + } + if len(u) > 0 && !existing[r.Type] { + toFetch = append(toFetch, u) + fetched[u] = append(fetched[u], &common.ImportFile{Name: r.Type, Path: u}) // Add to existing map so it is not fetched multiple times. existing[r.Type] = true } @@ -100,13 +109,14 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil count := 0 for len(toFetch) > 0 { - //1. Fetch import URL. Exit if no URLs left - //2. Check/handle HTTP status - //3. Store results in all ImportFiles from that URL - //4. Check for the optional schema file at import URL + .schema - //5. Repeat 2,3 for schema file - //6. Add each schema import to fetch if not already done - //7. Mark URL done. Return to 1. + //1. If short github URL, resolve to a download URL + //2. Fetch import URL. Exit if no URLs left + //3. Check/handle HTTP status + //4. Store results in all ImportFiles from that URL + //5. Check for the optional schema file at import URL + .schema + //6. Repeat 2,3 for schema file + //7. Add each schema import to fetch if not already done + //8. Mark URL done. Return to 1. if count >= tr.maxUrls { return nil, resolverError(config, fmt.Errorf("Number of imports exceeds maximum of %d", tr.maxUrls)) @@ -129,32 +139,41 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil } if sch != "" { - var s Schema + var s common.Schema if err := yaml.Unmarshal([]byte(sch), &s); err != nil { return nil, resolverError(config, err) } // Here we handle any nested imports in the schema we've just fetched. for _, v := range s.Imports { - i := &ImportFile{Name: v.Name} + i := &common.ImportFile{Name: v.Name} var existingSchema string - if len(fetched[v.Path]) == 0 { + u, conversionErr := tr.MapFetchableURL(v.Path) + if conversionErr != nil { + return nil, resolverError(config, fmt.Errorf("Failed to understand download url for %s: %v", v.Path, conversionErr)) + } + // If it's not a fetchable URL, we need to use the type name as is, since it is a short name + // for a schema. + if len(u) == 0 { + u = v.Path + } + if len(fetched[u]) == 0 { // If this import URL is new to us, add it to the URLs to fetch. - toFetch = append(toFetch, v.Path) + toFetch = append(toFetch, u) } else { // If this is not a new import URL and we've already fetched its contents, // reuse them. Also, check if we also found a schema for that import URL and // record those contents for re-use as well. - if fetched[v.Path][0].Content != "" { - i.Content = fetched[v.Path][0].Content - if len(fetched[v.Path+schemaSuffix]) > 0 { - existingSchema = fetched[v.Path+schemaSuffix][0].Content + if fetched[u][0].Content != "" { + i.Content = fetched[u][0].Content + if len(fetched[u+schemaSuffix]) > 0 { + existingSchema = fetched[u+schemaSuffix][0].Content } } } - fetched[v.Path] = append(fetched[v.Path], i) + fetched[u] = append(fetched[u], i) if existingSchema != "" { - fetched[v.Path+schemaSuffix] = append(fetched[v.Path+schemaSuffix], - &ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) + fetched[u+schemaSuffix] = append(fetched[u+schemaSuffix], + &common.ImportFile{Name: v.Name + schemaSuffix, Content: existingSchema}) } } @@ -162,7 +181,7 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil for _, i := range fetched[url] { schemaImportName := i.Name + schemaSuffix fetched[schemaURL] = append(fetched[schemaURL], - &ImportFile{Name: schemaImportName, Content: sch}) + &common.ImportFile{Name: schemaImportName, Content: sch}) } } @@ -170,10 +189,37 @@ func (tr *typeResolver) ResolveTypes(config *Configuration, imports []*ImportFil toFetch = toFetch[1:] } - ret := []*ImportFile{} + ret := []*common.ImportFile{} for _, v := range fetched { ret = append(ret, v...) } return ret, nil } + +// MapFetchableUrl checks a type to see if it is either a short git hub url or a fully specified URL +// and returns the URL that should be used to fetch it. If the url is not fetchable (primitive type for +// example) will return empty string. +func (tr *typeResolver) MapFetchableURL(t string) (string, error) { + if util.IsGithubShortType(t) { + return tr.ShortTypeToDownloadURL(t) + } else if util.IsHttpUrl(t) { + return t, nil + } + return "", nil +} + +// ShortTypeToDownloadURL converts a github URL into downloadable URL from github. +// Input must be of the type and is assumed to have been validated before this call: +// github.com/owner/repo/qualifier/type:version +// for example: +// github.com/kubernetes/application-dm-templates/storage/redis:v1 +func (tr *typeResolver) ShortTypeToDownloadURL(template string) (string, error) { + m := tr.re.FindStringSubmatch(template) + if len(m) != 6 { + return "", fmt.Errorf("Failed to parse short github url: %s", template) + } + r := tr.rp.GetGithubRegistry(m[1], m[2]) + t := registry.Type{m[3], m[4], m[5]} + return r.GetURL(t) +} diff --git a/manager/manager/typeresolver_test.go b/manager/manager/typeresolver_test.go index a9d80269b..b9667d959 100644 --- a/manager/manager/typeresolver_test.go +++ b/manager/manager/typeresolver_test.go @@ -15,14 +15,20 @@ package manager import ( "errors" + "fmt" "net/http" "reflect" + "regexp" "strings" "testing" "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/registry" + "github.com/kubernetes/deployment-manager/common" ) +var re = regexp.MustCompile("github.com/(.*)/(.*)/(.*)/(.*):(.*)") + type responseAndError struct { err error code int @@ -30,12 +36,13 @@ type responseAndError struct { } type resolverTestCase struct { - config string - imports []*ImportFile - responses map[string]responseAndError - urlcount int - expectedErr error - importOut []*ImportFile + config string + imports []*common.ImportFile + responses map[string]responseAndError + urlcount int + expectedErr error + importOut []*common.ImportFile + registryProvider registry.RegistryProvider } type testGetter struct { @@ -51,11 +58,68 @@ func (tg *testGetter) Get(url string) (body string, code int, err error) { return ret.resp, ret.code, ret.err } +type urlAndError struct { + u string + e error +} + +type testRegistryProvider struct { + owner string + repo string + r map[string]registry.Registry +} + +func newTestRegistryProvider(owner string, repository string, tests map[registry.Type]urlAndError, count int) registry.RegistryProvider { + r := make(map[string]registry.Registry) + r[owner+repository] = &testGithubRegistry{tests, count} + return &testRegistryProvider{owner, repository, r} +} + +func (trp *testRegistryProvider) GetGithubRegistry(owner string, repository string) registry.Registry { + return trp.r[owner+repository] +} + +type testGithubRegistry struct { + responses map[registry.Type]urlAndError + count int +} + +func (tgr *testGithubRegistry) GetURL(t registry.Type) (string, error) { + tgr.count = tgr.count + 1 + ret := tgr.responses[t] + return ret.u, ret.e +} + +func (tgr *testGithubRegistry) List() ([]registry.Type, error) { + return []registry.Type{}, fmt.Errorf("List should not be called in the test") +} + +func testUrlConversionDriver(c resolverTestCase, tests map[string]urlAndError, t *testing.T) { + r := &typeResolver{ + re: re, + rp: c.registryProvider, + } + for in, expected := range tests { + actual, err := r.ShortTypeToDownloadURL(in) + if actual != expected.u { + t.Errorf("failed on: %s : expected %s but got %s", in, expected.u, actual) + } + if err != expected.e { + t.Errorf("failed on: %s : expected error %v but got %v", in, expected.e, err) + } + } +} + func testDriver(c resolverTestCase, t *testing.T) { g := &testGetter{test: t, responses: c.responses} - r := &typeResolver{getter: g, maxUrls: 5} + r := &typeResolver{ + getter: g, + maxUrls: 5, + re: re, + rp: c.registryProvider, + } - conf := &Configuration{} + conf := &common.Configuration{} dataErr := yaml.Unmarshal([]byte(c.config), conf) if dataErr != nil { panic("bad test data") @@ -73,8 +137,8 @@ func testDriver(c resolverTestCase, t *testing.T) { t.Errorf("Expected error %s but found %s", c.expectedErr, err) } - resultImport := map[ImportFile]bool{} - expectedImport := map[ImportFile]bool{} + resultImport := map[common.ImportFile]bool{} + expectedImport := map[common.ImportFile]bool{} for _, i := range result { resultImport[*i] = true } @@ -106,7 +170,7 @@ resources: ` func TestIncludedImport(t *testing.T) { - imports := []*ImportFile{&ImportFile{Name: "foo.py"}} + imports := []*common.ImportFile{&common.ImportFile{Name: "foo.py"}} test := resolverTestCase{ config: includeImport, imports: imports, @@ -121,7 +185,7 @@ resources: ` func TestSingleUrl(t *testing.T) { - finalImports := []*ImportFile{&ImportFile{Name: "http://my-fake-url", Content: "my-content"}} + finalImports := []*common.ImportFile{&common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}} responses := map[string]responseAndError{ "http://my-fake-url": responseAndError{nil, http.StatusOK, "my-content"}, @@ -158,10 +222,10 @@ imports: ` func TestSingleUrlWithSchema(t *testing.T) { - finalImports := []*ImportFile{ - &ImportFile{Name: "http://my-fake-url", Content: "my-content"}, - &ImportFile{Name: "schema-import", Content: "schema-import"}, - &ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, + finalImports := []*common.ImportFile{ + &common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}, + &common.ImportFile{Name: "schema-import", Content: "schema-import"}, + &common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, } responses := map[string]responseAndError{ @@ -236,13 +300,13 @@ imports: ` func TestSharedImport(t *testing.T) { - finalImports := []*ImportFile{ - &ImportFile{Name: "http://my-fake-url", Content: "my-content"}, - &ImportFile{Name: "http://my-fake-url1", Content: "my-content-1"}, - &ImportFile{Name: "schema-import", Content: "schema-import"}, - &ImportFile{Name: "schema-import-1", Content: "schema-import"}, - &ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, - &ImportFile{Name: "http://my-fake-url1.schema", Content: schema2}, + finalImports := []*common.ImportFile{ + &common.ImportFile{Name: "http://my-fake-url", Path: "http://my-fake-url", Content: "my-content"}, + &common.ImportFile{Name: "http://my-fake-url1", Path: "http://my-fake-url1", Content: "my-content-1"}, + &common.ImportFile{Name: "schema-import", Content: "schema-import"}, + &common.ImportFile{Name: "schema-import-1", Content: "schema-import"}, + &common.ImportFile{Name: "http://my-fake-url.schema", Content: schema1}, + &common.ImportFile{Name: "http://my-fake-url1.schema", Content: schema2}, } responses := map[string]responseAndError{ @@ -262,3 +326,79 @@ func TestSharedImport(t *testing.T) { } testDriver(test, t) } + +func TestShortGithubUrlMapping(t *testing.T) { + githubUrlMaps := map[registry.Type]urlAndError{ + registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + } + + tests := map[string]urlAndError{ + "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + "github.com/kubernetes/application-dm-templates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/storage/redis/v1/redis.jinja", nil}, + } + + test := resolverTestCase{ + registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + } + testUrlConversionDriver(test, tests, t) +} + +func TestShortGithubUrlMappingDifferentOwnerAndRepo(t *testing.T) { + githubUrlMaps := map[registry.Type]urlAndError{ + registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.Type{"storage", "redis", "v1"}: urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + } + + tests := map[string]urlAndError{ + "github.com/example/mytemplates/common/replicatedservice:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + "github.com/example/mytemplates/storage/redis:v1": urlAndError{"https://raw.githubusercontent.com/example/mytemplates/master/storage/redis/v1/redis.jinja", nil}, + } + + test := resolverTestCase{ + registryProvider: newTestRegistryProvider("example", "mytemplates", githubUrlMaps, 2), + } + testUrlConversionDriver(test, tests, t) +} + +var templateShortGithubTemplate = ` +resources: +- name: foo + type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v1 +- name: foo1 + type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2 +` + +func TestShortGithubUrl(t *testing.T) { + finalImports := []*common.ImportFile{ + &common.ImportFile{ + Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v1", + Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", + Content: "my-content"}, + &common.ImportFile{ + Name: "github.com/kubernetes/application-dm-templates/common/replicatedservice:v2", + Path: "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", + Content: "my-content-2"}, + } + + responses := map[string]responseAndError{ + "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content"}, + "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""}, + "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py": responseAndError{nil, http.StatusOK, "my-content-2"}, + "https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py.schema": responseAndError{nil, http.StatusNotFound, ""}, + } + + githubUrlMaps := map[registry.Type]urlAndError{ + registry.Type{"common", "replicatedservice", "v1"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v1/replicatedservice.py", nil}, + registry.Type{"common", "replicatedservice", "v2"}: urlAndError{"https://raw.githubusercontent.com/kubernetes/application-dm-templates/master/common/replicatedservice/v2/replicatedservice.py", nil}, + } + + test := resolverTestCase{ + config: templateShortGithubTemplate, + importOut: finalImports, + urlcount: 4, + responses: responses, + registryProvider: newTestRegistryProvider("kubernetes", "application-dm-templates", githubUrlMaps, 2), + } + testDriver(test, t) +} diff --git a/manager/repository/repository.go b/manager/repository/repository.go index 9e40a3507..6d8f98a4d 100644 --- a/manager/repository/repository.go +++ b/manager/repository/repository.go @@ -22,37 +22,37 @@ import ( "sync" "time" - "github.com/kubernetes/deployment-manager/manager/manager" + "github.com/kubernetes/deployment-manager/common" ) // deploymentTypeInstanceMap stores type instances mapped by deployment name. // This allows for simple updating and deleting of per-deployment instances // when deployments are created/updated/deleted. -type deploymentTypeInstanceMap map[string][]*manager.TypeInstance +type deploymentTypeInstanceMap map[string][]*common.TypeInstance type typeInstanceMap map[string]deploymentTypeInstanceMap type mapBasedRepository struct { sync.RWMutex - deployments map[string]manager.Deployment - manifests map[string]map[string]*manager.Manifest + deployments map[string]common.Deployment + manifests map[string]map[string]*common.Manifest instances typeInstanceMap } // NewMapBasedRepository returns a new map based repository. -func NewMapBasedRepository() manager.Repository { +func NewMapBasedRepository() common.Repository { return &mapBasedRepository{ - deployments: make(map[string]manager.Deployment, 0), - manifests: make(map[string]map[string]*manager.Manifest, 0), + deployments: make(map[string]common.Deployment, 0), + manifests: make(map[string]map[string]*common.Manifest, 0), instances: typeInstanceMap{}, } } // ListDeployments returns of all of the deployments in the repository. -func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) { +func (r *mapBasedRepository) ListDeployments() ([]common.Deployment, error) { r.RLock() defer r.RUnlock() - l := []manager.Deployment{} + l := []common.Deployment{} for _, deployment := range r.deployments { l = append(l, deployment) } @@ -62,7 +62,7 @@ func (r *mapBasedRepository) ListDeployments() ([]manager.Deployment, error) { // GetDeployment returns the deployment with the supplied name. // If the deployment is not found, it returns an error. -func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, error) { +func (r *mapBasedRepository) GetDeployment(name string) (*common.Deployment, error) { d, ok := r.deployments[name] if !ok { return nil, fmt.Errorf("deployment %s not found", name) @@ -72,13 +72,13 @@ func (r *mapBasedRepository) GetDeployment(name string) (*manager.Deployment, er // GetValidDeployment returns the deployment with the supplied name. // If the deployment is not found or marked as deleted, it returns an error. -func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deployment, error) { +func (r *mapBasedRepository) GetValidDeployment(name string) (*common.Deployment, error) { d, err := r.GetDeployment(name) if err != nil { return nil, err } - if d.Status == manager.DeletedStatus { + if d.Status == common.DeletedStatus { return nil, fmt.Errorf("deployment %s is deleted", name) } @@ -86,7 +86,7 @@ func (r *mapBasedRepository) GetValidDeployment(name string) (*manager.Deploymen } // SetDeploymentStatus sets the DeploymentStatus of the deployment and updates ModifiedAt -func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.DeploymentStatus) error { +func (r *mapBasedRepository) SetDeploymentStatus(name string, status common.DeploymentStatus) error { return func() error { r.Lock() defer r.Unlock() @@ -104,8 +104,8 @@ func (r *mapBasedRepository) SetDeploymentStatus(name string, status manager.Dep } // CreateDeployment creates a new deployment and stores it in the repository. -func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, error) { - d, err := func() (*manager.Deployment, error) { +func (r *mapBasedRepository) CreateDeployment(name string) (*common.Deployment, error) { + d, err := func() (*common.Deployment, error) { r.Lock() defer r.Unlock() @@ -114,8 +114,8 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, return nil, fmt.Errorf("Deployment %s already exists", name) } - d := manager.NewDeployment(name) - d.Status = manager.CreatedStatus + d := common.NewDeployment(name) + d.Status = common.CreatedStatus d.DeployedAt = time.Now() r.deployments[name] = *d return d, nil @@ -129,7 +129,7 @@ func (r *mapBasedRepository) CreateDeployment(name string) (*manager.Deployment, return d, nil } -func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manager.Manifest) error { +func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *common.Manifest) error { err := func() error { r.Lock() defer r.Unlock() @@ -168,8 +168,8 @@ func (r *mapBasedRepository) AddManifest(deploymentName string, manifest *manage // DeleteDeployment deletes the deployment with the supplied name. // If forget is true, then the deployment is removed from the repository. // Otherwise, it is marked as deleted and retained. -func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manager.Deployment, error) { - d, err := func() (*manager.Deployment, error) { +func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*common.Deployment, error) { + d, err := func() (*common.Deployment, error) { r.Lock() defer r.Unlock() @@ -180,7 +180,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage if !forget { d.DeletedAt = time.Now() - d.Status = manager.DeletedStatus + d.Status = common.DeletedStatus r.deployments[name] = *d } else { delete(r.deployments, name) @@ -199,7 +199,7 @@ func (r *mapBasedRepository) DeleteDeployment(name string, forget bool) (*manage return d, nil } -func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*manager.Manifest, error) { +func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*common.Manifest, error) { r.Lock() defer r.Unlock() @@ -211,17 +211,17 @@ func (r *mapBasedRepository) ListManifests(deploymentName string) (map[string]*m return r.listManifestsForDeployment(deploymentName) } -func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*manager.Manifest, error) { +func (r *mapBasedRepository) listManifestsForDeployment(deploymentName string) (map[string]*common.Manifest, error) { l, ok := r.manifests[deploymentName] if !ok { - l = make(map[string]*manager.Manifest, 0) + l = make(map[string]*common.Manifest, 0) r.manifests[deploymentName] = l } return l, nil } -func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*manager.Manifest, error) { +func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) { r.Lock() defer r.Unlock() @@ -233,7 +233,7 @@ func (r *mapBasedRepository) GetManifest(deploymentName string, manifestName str return r.getManifestForDeployment(deploymentName, manifestName) } -func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*manager.Manifest, error) { +func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, manifestName string) (*common.Manifest, error) { l, err := r.listManifestsForDeployment(deploymentName) if err != nil { return nil, err @@ -249,7 +249,7 @@ func (r *mapBasedRepository) getManifestForDeployment(deploymentName string, man // GetLatestManifest returns the latest manifest for a given deployment, // which by definition is the manifest with the largest time stamp. -func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*manager.Manifest, error) { +func (r *mapBasedRepository) GetLatestManifest(deploymentName string) (*common.Manifest, error) { r.Lock() defer r.Unlock() @@ -273,11 +273,11 @@ func (r *mapBasedRepository) ListTypes() []string { // GetTypeInstances returns all instances of a given type. If type is empty, // returns all instances for all types. -func (r *mapBasedRepository) GetTypeInstances(typeName string) []*manager.TypeInstance { +func (r *mapBasedRepository) GetTypeInstances(typeName string) []*common.TypeInstance { r.Lock() defer r.Unlock() - var instances []*manager.TypeInstance + var instances []*common.TypeInstance for t, dInstMap := range r.instances { if t == typeName || typeName == "all" { for _, i := range dInstMap { @@ -307,7 +307,7 @@ func (r *mapBasedRepository) ClearTypeInstances(deploymentName string) { // // To clear the current set of instances first, caller should first use // ClearTypeInstances(). -func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*manager.TypeInstance) { +func (r *mapBasedRepository) SetTypeInstances(deploymentName string, instances map[string][]*common.TypeInstance) { r.Lock() defer r.Unlock() diff --git a/manager/repository/repository_test.go b/manager/repository/repository_test.go index 47f9af313..2f90fd4fe 100644 --- a/manager/repository/repository_test.go +++ b/manager/repository/repository_test.go @@ -14,7 +14,7 @@ limitations under the License. package repository import ( - "github.com/kubernetes/deployment-manager/manager/manager" + "github.com/kubernetes/deployment-manager/common" "fmt" "testing" @@ -77,7 +77,7 @@ func testCreateDeploymentWithManifests(t *testing.T, count int) { for i := 0; i < count; i++ { var manifestName = fmt.Sprintf("manifest-%d", i) - manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} + manifest := common.Manifest{Deployment: deploymentName, Name: manifestName} err := r.AddManifest(deploymentName, &manifest) if err != nil { t.Fatalf("AddManifest failed: %v", err) @@ -136,7 +136,7 @@ func TestRepositoryDeleteWorksWithNoLatestManifest(t *testing.T) { if err != nil { t.Fatalf("DeleteDeployment failed: %v", err) } - if dDeleted.Status != manager.DeletedStatus { + if dDeleted.Status != common.DeletedStatus { t.Fatalf("Deployment Status is not deleted") } if _, err := r.ListManifests(deploymentName); err == nil { @@ -148,7 +148,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) { var deploymentName = "mydeployment" var manifestName = "manifest-0" r := NewMapBasedRepository() - manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} + manifest := common.Manifest{Deployment: deploymentName, Name: manifestName} _, err := r.CreateDeployment(deploymentName) if err != nil { t.Fatalf("CreateDeployment failed: %v", err) @@ -161,7 +161,7 @@ func TestRepositoryDeleteDeploymentWorksNoForget(t *testing.T) { if err != nil { t.Fatalf("DeleteDeployment failed: %v", err) } - if dDeleted.Status != manager.DeletedStatus { + if dDeleted.Status != common.DeletedStatus { t.Fatalf("Deployment Status is not deleted") } } @@ -170,7 +170,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) { var deploymentName = "mydeployment" var manifestName = "manifest-0" r := NewMapBasedRepository() - manifest := manager.Manifest{Deployment: deploymentName, Name: manifestName} + manifest := common.Manifest{Deployment: deploymentName, Name: manifestName} _, err := r.CreateDeployment(deploymentName) if err != nil { t.Fatalf("CreateDeployment failed: %v", err) @@ -183,7 +183,7 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) { if err != nil { t.Fatalf("DeleteDeployment failed: %v", err) } - if dDeleted.Status != manager.CreatedStatus { + if dDeleted.Status != common.CreatedStatus { t.Fatalf("Deployment Status is not created") } } @@ -191,9 +191,9 @@ func TestRepositoryDeleteDeploymentWorksForget(t *testing.T) { func TestRepositoryTypeInstances(t *testing.T) { r := NewMapBasedRepository() - d1Map := map[string][]*manager.TypeInstance{ - "t1": []*manager.TypeInstance{ - &manager.TypeInstance{ + d1Map := map[string][]*common.TypeInstance{ + "t1": []*common.TypeInstance{ + &common.TypeInstance{ Name: "i1", Type: "t1", Deployment: "d1", @@ -203,9 +203,9 @@ func TestRepositoryTypeInstances(t *testing.T) { }, } - d2Map := map[string][]*manager.TypeInstance{ - "t2": []*manager.TypeInstance{ - &manager.TypeInstance{ + d2Map := map[string][]*common.TypeInstance{ + "t2": []*common.TypeInstance{ + &common.TypeInstance{ Name: "i2", Type: "t2", Deployment: "d2", @@ -215,9 +215,9 @@ func TestRepositoryTypeInstances(t *testing.T) { }, } - d3Map := map[string][]*manager.TypeInstance{ - "t2": []*manager.TypeInstance{ - &manager.TypeInstance{ + d3Map := map[string][]*common.TypeInstance{ + "t2": []*common.TypeInstance{ + &common.TypeInstance{ Name: "i3", Type: "t2", Deployment: "d3", diff --git a/registry/github_registry.go b/registry/github_registry.go index fff813bd1..b0378d7a4 100644 --- a/registry/github_registry.go +++ b/registry/github_registry.go @@ -18,6 +18,7 @@ import ( "fmt" "log" + "strings" ) // GithubRegistry implements the Registry interface that talks to github. @@ -66,7 +67,11 @@ func (g *GithubRegistry) List() ([]Type, error) { // GetURL fetches the download URL for a given Type and checks for existence of a schema file. func (g *GithubRegistry) GetURL(t Type) (string, error) { - path := g.path + "/" + t.Name + "/" + t.Version + path,err := g.MakeRepositoryPath(t) + if err != nil { + return "", err + } + log.Printf("Got repository path: %s", path) _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, path, nil) if err != nil { log.Printf("Failed to list versions at path: %s: %v", path, err) @@ -114,3 +119,34 @@ func (g *GithubRegistry) getDirs(dir string) ([]string, error) { return dirs, nil } + +func (g *GithubRegistry) mapCollection(collection string) (string, error) { + if strings.ContainsAny(collection, "/") { + return "", fmt.Errorf("collection must not contain slashes, got %s", collection) + } + // TODO(vaikas): Implement lookup from the root metadata file to map collection to a path + return collection, nil +} + +// MakeRepositoryPath constructs a github path to a given type based on a repository, and type name and version. +// The returned repository path will be of the form: +// [GithubRegistry.path/][Type.Collection]/Type.Name/Type.Version +// Type.Collection will be mapped using mapCollection in the future, for now it's a straight +// 1:1 mapping (if given) +func (g *GithubRegistry) MakeRepositoryPath(t Type) (string, error) { + log.Printf("Making repository path: %v", t) + // First map the collection + collection, err := g.mapCollection(t.Collection) + if err != nil { + return "", err + } + // Construct the return path + p := "" + if len(g.path) > 0 { + p += g.path + "/" + } + if len(collection) > 0 { + p += collection + "/" + } + return p + t.Name + "/" + t.Version, nil +} diff --git a/registry/registry.go b/registry/registry.go index cbaab8a06..609c17155 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -22,6 +22,7 @@ package registry // For example, a template registry containing two versions of redis // (implemented in jinja), and one version of replicatedservice (implemented // in python) would have a directory structure that looks something like this: +// qualifier [optional] prefix to a virtual root within the repository. // /redis // /v1 // redis.jinja @@ -35,6 +36,7 @@ package registry // replicatedservice.python.schema type Type struct { + Collection string Name string Version string } @@ -46,3 +48,6 @@ type Registry interface { // Get the download URL for a given template and version GetURL(t Type) (string, error) } + + + diff --git a/registry/registryprovider.go b/registry/registryprovider.go new file mode 100644 index 000000000..2381e3995 --- /dev/null +++ b/registry/registryprovider.go @@ -0,0 +1,26 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +// RegistryProvider returns factories for creating registries for a given RegistryType. +type RegistryProvider interface { + GetGithubRegistry(owner string, repository string) Registry +} + +type DefaultRegistryProvider struct { +} + +func (drp *DefaultRegistryProvider) GetGithubRegistry(owner string, repository string) Registry { + return NewGithubRegistry(owner, repository, "") +} diff --git a/resourcifier/configurations.go b/resourcifier/configurations.go index 90dccfe80..34f5ec42c 100644 --- a/resourcifier/configurations.go +++ b/resourcifier/configurations.go @@ -14,7 +14,7 @@ limitations under the License. package main import ( - "github.com/kubernetes/deployment-manager/manager/manager" + "github.com/kubernetes/deployment-manager/common" "github.com/kubernetes/deployment-manager/resourcifier/configurator" "github.com/kubernetes/deployment-manager/util" @@ -76,8 +76,8 @@ func listConfigurationsHandlerFunc(w http.ResponseWriter, r *http.Request) { return } - c := &manager.Configuration{ - []*manager.Resource{ + c := &common.Configuration{ + []*common.Resource{ {Type: rtype}, }, } @@ -105,8 +105,8 @@ func getConfigurationHandlerFunc(w http.ResponseWriter, r *http.Request) { return } - c := &manager.Configuration{ - []*manager.Resource{ + c := &common.Configuration{ + []*common.Resource{ {Name: rname, Type: rtype}, }, } @@ -253,7 +253,7 @@ func getPathVariable(w http.ResponseWriter, r *http.Request, variable, handler s return unescaped, nil } -func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *manager.Configuration { +func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *common.Configuration { b := io.LimitReader(r.Body, *maxLength*1024) y, err := ioutil.ReadAll(b) if err != nil { @@ -276,7 +276,7 @@ func getConfiguration(w http.ResponseWriter, r *http.Request, handler string) *m return nil } - c := &manager.Configuration{} + c := &common.Configuration{} if err := json.Unmarshal(j, c); err != nil { e := errors.New(err.Error() + "\n" + string(j)) util.LogAndReturnError(handler, http.StatusBadRequest, e, w) diff --git a/resourcifier/configurator/configurator.go b/resourcifier/configurator/configurator.go index e4e6fec89..c6c72d272 100644 --- a/resourcifier/configurator/configurator.go +++ b/resourcifier/configurator/configurator.go @@ -20,7 +20,7 @@ import ( "os/exec" "strings" - "github.com/kubernetes/deployment-manager/manager/manager" + "github.com/kubernetes/deployment-manager/common" "github.com/ghodss/yaml" ) @@ -77,7 +77,7 @@ func (e *Error) appendError(err error) error { // action on it (create/delete/replace) and updates the State of the resource with the resulting // status. In case of errors with a resource, Resource.State.Errors is set. // and then updates the deployment with the completion status and completion time. -func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, error) { +func (a *Configurator) Configure(c *common.Configuration, o operation) (string, error) { errors := &Error{} var output []string for i, resource := range c.Resources { @@ -99,8 +99,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, if err != nil { e := fmt.Errorf("yaml marshal failed for resource: %v: %v", resource.Name, err) log.Println(errors.appendError(e)) - c.Resources[i].State = &manager.ResourceState{ - Status: manager.Aborted, + c.Resources[i].State = &common.ResourceState{ + Status: common.Aborted, Errors: []string{e.Error()}, } continue @@ -122,8 +122,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, if err := cmd.Start(); err != nil { e := fmt.Errorf("cannot start kubetcl for resource: %v: %v", resource.Name, err) - c.Resources[i].State = &manager.ResourceState{ - Status: manager.Failed, + c.Resources[i].State = &common.ResourceState{ + Status: common.Failed, Errors: []string{e.Error()}, } log.Println(errors.appendError(e)) @@ -137,8 +137,8 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, log.Println(resource.Name + " not found, treating as success for delete") } else { e := fmt.Errorf("kubetcl failed for resource: %v: %v: %v", resource.Name, err, combined.String()) - c.Resources[i].State = &manager.ResourceState{ - Status: manager.Failed, + c.Resources[i].State = &common.ResourceState{ + Status: common.Failed, Errors: []string{e.Error()}, } log.Println(errors.appendError(e)) @@ -147,7 +147,7 @@ func (a *Configurator) Configure(c *manager.Configuration, o operation) (string, } output = append(output, combined.String()) - c.Resources[i].State = &manager.ResourceState{Status: manager.Created} + c.Resources[i].State = &common.ResourceState{Status: common.Created} log.Printf("kubectl succeeded for resource: %v: SysTime: %v UserTime: %v\n%v", resource.Name, cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), combined.String()) } diff --git a/util/templateutil.go b/util/templateutil.go index efc5e8fcc..a64b05af0 100644 --- a/util/templateutil.go +++ b/util/templateutil.go @@ -15,9 +15,39 @@ package util import ( "strings" + "log" + + "github.com/kubernetes/deployment-manager/common" ) // IsTemplate returns whether a given type is a template. -func IsTemplate(t string) bool { - return strings.HasSuffix(t, ".py") || strings.HasSuffix(t, ".jinja") +func IsTemplate(t string, imports []*common.ImportFile) bool { + log.Printf("IsTemplate: %s : %+v", t, imports) + for _, imp := range imports { + log.Printf("Checking: %s", imp.Name) + if imp.Name == t { + return true + } + } + return false +} + +// IsGithubShortType returns whether a given type is a type description in a short format to a github repository type. +// For now, this means using github types: +// github.com/owner/repo/qualifier/type:version +// for example: +// github.com/kubernetes/application-dm-templates/storage/redis:v1 +func IsGithubShortType(t string) bool { + if !strings.HasPrefix(t, "github.com/") { + return false + } + s := strings.Split(t, "/") + if len(s) != 5 { + return false + } + v := strings.Split(s[4], ":") + if len(v) != 2 { + return false + } + return true }