From 9ae0a4056725b5806ae4d6e8b81ba7f3c1b184db Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 16 Feb 2016 12:34:32 -0800 Subject: [PATCH] mechanical move of NewTemplate* methods from expander to util package --- dm/dm.go | 12 +- expandybird/expander/expander.go | 171 ---------------------- expandybird/expander/expander_test.go | 129 +---------------- expandybird/service/service_test.go | 6 +- util/templateutil.go | 196 ++++++++++++++++++++++++++ util/templateutil_test.go | 148 +++++++++++++++++++ 6 files changed, 357 insertions(+), 305 deletions(-) create mode 100644 util/templateutil.go create mode 100644 util/templateutil_test.go diff --git a/dm/dm.go b/dm/dm.go index 3e0534197..365b90358 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -20,7 +20,7 @@ import ( "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/common" - "github.com/kubernetes/deployment-manager/expandybird/expander" + "github.com/kubernetes/deployment-manager/util" "archive/tar" "bytes" @@ -337,14 +337,14 @@ func loadTemplate(args []string) *common.Template { } r := bytes.NewReader(input) - template, err = expander.NewTemplateFromArchive(args[1], r, args[2:]) + template, err = util.NewTemplateFromArchive(args[1], r, args[2:]) if err != nil { if err != tar.ErrHeader { panic(err) } r := bytes.NewReader(input) - template, err = expander.NewTemplateFromReader(args[1], r, args[2:]) + template, err = util.NewTemplateFromReader(args[1], r, args[2:]) if err != nil { panic(fmt.Errorf("cannot create configuration from supplied arguments: %s\n", err)) } @@ -354,9 +354,9 @@ func loadTemplate(args []string) *common.Template { // it's a local file, it's configuration. if _, err := os.Stat(args[1]); err == nil { if len(args) > 2 { - template, err = expander.NewTemplateFromFileNames(args[1], args[2:]) + template, err = util.NewTemplateFromFileNames(args[1], args[2:]) } else { - template, err = expander.NewTemplateFromRootTemplate(args[1]) + template, err = util.NewTemplateFromRootTemplate(args[1]) } } else { template = buildTemplateFromType(args[1]) @@ -397,7 +397,7 @@ func buildTemplateFromType(t string) *common.Template { } // Name the deployment after the type name. - template, err := expander.NewTemplateFromType(t, t, props) + template, err := util.NewTemplateFromType(t, t, props) if err != nil { panic(fmt.Errorf("cannot create configuration from type (%s): %s\n", t, err)) } diff --git a/expandybird/expander/expander.go b/expandybird/expander/expander.go index 0dfb6ac7d..01176301f 100644 --- a/expandybird/expander/expander.go +++ b/expandybird/expander/expander.go @@ -17,16 +17,11 @@ limitations under the License. package expander import ( - "archive/tar" "bytes" "fmt" - "io" - "io/ioutil" "log" "os" "os/exec" - "path" - "path/filepath" "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/common" @@ -46,172 +41,6 @@ func NewExpander(binary string) Expander { return &expander{binary} } -// NewTemplateFromType creates and returns a new template whose content -// is a YAML marshaled resource assembled from the supplied arguments. -func NewTemplateFromType(name, typeName string, properties map[string]interface{}) (*common.Template, error) { - resource := &common.Resource{ - Name: name, - Type: typeName, - Properties: properties, - } - - config := common.Configuration{Resources: []*common.Resource{resource}} - content, err := yaml.Marshal(config) - if err != nil { - return nil, fmt.Errorf("error: %s\ncannot marshal configuration: %v\n", err, config) - } - - template := &common.Template{ - Name: name, - Content: string(content), - Imports: []*common.ImportFile{}, - } - - return template, nil -} - -// NewTemplateFromArchive creates and returns a new template whose content -// and imported files are read from the supplied archive. -func NewTemplateFromArchive(name string, r io.Reader, importFileNames []string) (*common.Template, error) { - var content []byte - imports, err := collectImportFiles(importFileNames) - if err != nil { - return nil, err - } - - tr := tar.NewReader(r) - for i := 0; true; i++ { - hdr, err := tr.Next() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - if hdr.Name != name { - importFileData, err := ioutil.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("cannot read archive file %s: %s", hdr.Name, err) - } - - imports = append(imports, - &common.ImportFile{ - Name: path.Base(hdr.Name), - Content: string(importFileData), - }) - } else { - content, err = ioutil.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("cannot read %s from archive: %s", name, err) - } - } - } - - if len(content) < 1 { - return nil, fmt.Errorf("cannot find %s in archive", name) - } - - return &common.Template{ - Name: name, - Content: string(content), - Imports: imports, - }, nil -} - -// NewTemplateFromReader creates and returns a new template whose content -// is read from the supplied reader. -func NewTemplateFromReader(name string, r io.Reader, importFileNames []string) (*common.Template, error) { - content, err := ioutil.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("cannot read archive %s: %s", name, err) - } - - return newTemplateFromContentAndImports(name, string(content), importFileNames) -} - -// NewTemplateFromRootTemplate creates and returns a new template whose content -// and imported files are constructed from reading the root template, parsing out -// the imports section and reading the imports from there -func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) { - templateDir := filepath.Dir(templateFileName) - content, err := ioutil.ReadFile(templateFileName) - if err != nil { - return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName) - } - - var c map[string]interface{} - err = yaml.Unmarshal([]byte(content), &c) - if err != nil { - log.Fatalf("Cannot parse template: %v", err) - } - - // For each of the imports, grab the import file - var imports []string - if c["imports"] != nil { - for _, importFile := range c["imports"].([]interface{}) { - var fileName = importFile.(map[string]interface{})["path"].(string) - imports = append(imports, templateDir+"/"+fileName) - } - } - - return NewTemplateFromFileNames(templateFileName, imports[0:]) -} - -// NewTemplateFromFileNames creates and returns a new template whose content -// and imported files are read from the supplied file names. -func NewTemplateFromFileNames( - templateFileName string, - importFileNames []string, -) (*common.Template, error) { - content, err := ioutil.ReadFile(templateFileName) - if err != nil { - return nil, fmt.Errorf("cannot read template file %s: %s", templateFileName, err) - } - - name := path.Base(templateFileName) - return newTemplateFromContentAndImports(name, string(content), importFileNames) -} - -func newTemplateFromContentAndImports( - name, content string, - importFileNames []string, -) (*common.Template, error) { - if len(content) < 1 { - return nil, fmt.Errorf("supplied configuration is empty") - } - - imports, err := collectImportFiles(importFileNames) - if err != nil { - return nil, err - } - - return &common.Template{ - Name: name, - Content: content, - Imports: imports, - }, nil -} - -func collectImportFiles(importFileNames []string) ([]*common.ImportFile, error) { - imports := []*common.ImportFile{} - for _, importFileName := range importFileNames { - importFileData, err := ioutil.ReadFile(importFileName) - if err != nil { - return nil, fmt.Errorf("cannot read import file %s: %s", importFileName, err) - } - - imports = append(imports, - &common.ImportFile{ - Name: path.Base(importFileName), - Content: string(importFileData), - }) - } - - return imports, nil -} - // ExpansionResult describes the unmarshalled output of ExpandTemplate. type ExpansionResult struct { Config map[string]interface{} diff --git a/expandybird/expander/expander_test.go b/expandybird/expander/expander_test.go index 9c4677f78..bb12577c1 100644 --- a/expandybird/expander/expander_test.go +++ b/expandybird/expander/expander_test.go @@ -17,8 +17,6 @@ limitations under the License. package expander import ( - "archive/tar" - "bytes" "fmt" "io" "io/ioutil" @@ -28,12 +26,10 @@ import ( "strings" "testing" - "github.com/ghodss/yaml" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/util" ) -const invalidFileName = "afilethatdoesnotexist" - var importFileNames = []string{ "../test/replicatedservice.py", } @@ -51,7 +47,7 @@ type ExpanderTestCase struct { } func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template { - template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames) + template, err := util.NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames) if err != nil { t.Fatalf("cannot create template for test case '%s': %s", etc.Description, err) } @@ -108,131 +104,14 @@ func testExpandTemplateFromFile(t *testing.T, fileName, baseName string, importF expandAndVerifyOutput(t, actualOutput, description) } -var ( - testTemplateName = "expandybird" - testTemplateType = "replicatedservice.py" - testTemplateProperties = ` -service_port: 8080 -target_port: 8080 -container_port: 8080 -external_service: true -replicas: 3 -image: gcr.io/dm-k8s-testing/expandybird -labels: - app: expandybird -` -) - -func TestNewTemplateFromType(t *testing.T) { - var properties map[string]interface{} - if err := yaml.Unmarshal([]byte(testTemplateProperties), &properties); err != nil { - t.Fatalf("cannot unmarshal test data: %s", err) - } - - _, err := NewTemplateFromType(testTemplateName, testTemplateType, properties) - if err != nil { - t.Fatalf("cannot create template from type %s: %s", testTemplateType, err) - } -} - -func TestNewTemplateFromReader(t *testing.T) { - r := bytes.NewReader([]byte{}) - if _, err := NewTemplateFromReader("test", r, nil); err == nil { - t.Fatalf("expected error did not occur for empty input: %s", err) - } - - r = bytes.NewReader([]byte("test")) - if _, err := NewTemplateFromReader("test", r, nil); err != nil { - t.Fatalf("cannot read test template: %s", err) - } -} - -type archiveBuilder []struct { - Name, Body string -} - -var invalidFiles = archiveBuilder{ - {"testFile1.yaml", ""}, -} - -var validFiles = archiveBuilder{ - {"testFile1.yaml", "testFile:1"}, - {"testFile2.yaml", "testFile:2"}, -} - -func generateArchive(t *testing.T, files archiveBuilder) *bytes.Reader { - buffer := new(bytes.Buffer) - tw := tar.NewWriter(buffer) - for _, file := range files { - hdr := &tar.Header{ - Name: file.Name, - Mode: 0600, - Size: int64(len(file.Body)), - } - - if err := tw.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - - if _, err := tw.Write([]byte(file.Body)); err != nil { - t.Fatal(err) - } - } - - if err := tw.Close(); err != nil { - t.Fatal(err) - } - - r := bytes.NewReader(buffer.Bytes()) - return r -} - -func TestNewTemplateFromArchive(t *testing.T) { - r := bytes.NewReader([]byte{}) - if _, err := NewTemplateFromArchive("", r, nil); err == nil { - t.Fatalf("expected error did not occur for empty input: %s", err) - } - - r = bytes.NewReader([]byte("test")) - if _, err := NewTemplateFromArchive("", r, nil); err == nil { - t.Fatalf("expected error did not occur for non archive file:%s", err) - } - - r = generateArchive(t, invalidFiles) - if _, err := NewTemplateFromArchive(invalidFiles[0].Name, r, nil); err == nil { - t.Fatalf("expected error did not occur for empty file in archive") - } - - r = generateArchive(t, validFiles) - if _, err := NewTemplateFromArchive("", r, nil); err == nil { - t.Fatalf("expected error did not occur for missing file in archive") - } - - r = generateArchive(t, validFiles) - if _, err := NewTemplateFromArchive(validFiles[1].Name, r, nil); err != nil { - t.Fatalf("cannnot create template from valid archive") - } -} - -func TestNewTemplateFromFileNames(t *testing.T) { - if _, err := NewTemplateFromFileNames(invalidFileName, importFileNames); err == nil { - t.Fatalf("expected error did not occur for invalid template file name") - } - - _, err := NewTemplateFromFileNames(invalidFileName, []string{"afilethatdoesnotexist"}) - if err == nil { - t.Fatalf("expected error did not occur for invalid import file names") - } -} - func TestExpandTemplateFromReader(t *testing.T) { baseName := path.Base(validFileName) - testExpandTemplateFromFile(t, validFileName, baseName, importFileNames, NewTemplateFromReader) + testExpandTemplateFromFile(t, validFileName, baseName, importFileNames, util.NewTemplateFromReader) } func TestExpandTemplateFromArchive(t *testing.T) { baseName := path.Base(validFileName) - testExpandTemplateFromFile(t, archiveFileName, baseName, nil, NewTemplateFromArchive) + testExpandTemplateFromFile(t, archiveFileName, baseName, nil, util.NewTemplateFromArchive) } var ExpanderTestCases = []ExpanderTestCase{ diff --git a/expandybird/service/service_test.go b/expandybird/service/service_test.go index c2a921737..ee5f029ae 100644 --- a/expandybird/service/service_test.go +++ b/expandybird/service/service_test.go @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,15 +26,15 @@ import ( "reflect" "testing" - "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/common" + "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/util" restful "github.com/emicklei/go-restful" ) func GetTemplateReader(t *testing.T, description string, templateFileName string) io.Reader { - template, err := expander.NewTemplateFromFileNames(templateFileName, importFileNames) + template, err := util.NewTemplateFromFileNames(templateFileName, importFileNames) if err != nil { t.Errorf("cannot create template for test case (%s): %s\n", err, description) } diff --git a/util/templateutil.go b/util/templateutil.go new file mode 100644 index 000000000..35f507afd --- /dev/null +++ b/util/templateutil.go @@ -0,0 +1,196 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "log" + "path" + "path/filepath" + + "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/common" +) + +// NewTemplateFromType creates and returns a new template whose content +// is a YAML marshaled resource assembled from the supplied arguments. +func NewTemplateFromType(name, typeName string, properties map[string]interface{}) (*common.Template, error) { + resource := &common.Resource{ + Name: name, + Type: typeName, + Properties: properties, + } + + config := common.Configuration{Resources: []*common.Resource{resource}} + content, err := yaml.Marshal(config) + if err != nil { + return nil, fmt.Errorf("error: %s\ncannot marshal configuration: %v\n", err, config) + } + + template := &common.Template{ + Name: name, + Content: string(content), + Imports: []*common.ImportFile{}, + } + + return template, nil +} + +// NewTemplateFromArchive creates and returns a new template whose content +// and imported files are read from the supplied archive. +func NewTemplateFromArchive(name string, r io.Reader, importFileNames []string) (*common.Template, error) { + var content []byte + imports, err := collectImportFiles(importFileNames) + if err != nil { + return nil, err + } + + tr := tar.NewReader(r) + for i := 0; true; i++ { + hdr, err := tr.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if hdr.Name != name { + importFileData, err := ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("cannot read archive file %s: %s", hdr.Name, err) + } + + imports = append(imports, + &common.ImportFile{ + Name: path.Base(hdr.Name), + Content: string(importFileData), + }) + } else { + content, err = ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("cannot read %s from archive: %s", name, err) + } + } + } + + if len(content) < 1 { + return nil, fmt.Errorf("cannot find %s in archive", name) + } + + return &common.Template{ + Name: name, + Content: string(content), + Imports: imports, + }, nil +} + +// NewTemplateFromReader creates and returns a new template whose content +// is read from the supplied reader. +func NewTemplateFromReader(name string, r io.Reader, importFileNames []string) (*common.Template, error) { + content, err := ioutil.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("cannot read archive %s: %s", name, err) + } + + return newTemplateFromContentAndImports(name, string(content), importFileNames) +} + +// NewTemplateFromRootTemplate creates and returns a new template whose content +// and imported files are constructed from reading the root template, parsing out +// the imports section and reading the imports from there +func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, error) { + templateDir := filepath.Dir(templateFileName) + content, err := ioutil.ReadFile(templateFileName) + if err != nil { + return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName) + } + + var c map[string]interface{} + err = yaml.Unmarshal([]byte(content), &c) + if err != nil { + log.Fatalf("Cannot parse template: %v", err) + } + + // For each of the imports, grab the import file + var imports []string + if c["imports"] != nil { + for _, importFile := range c["imports"].([]interface{}) { + var fileName = importFile.(map[string]interface{})["path"].(string) + imports = append(imports, templateDir+"/"+fileName) + } + } + + return NewTemplateFromFileNames(templateFileName, imports[0:]) +} + +// NewTemplateFromFileNames creates and returns a new template whose content +// and imported files are read from the supplied file names. +func NewTemplateFromFileNames( + templateFileName string, + importFileNames []string, +) (*common.Template, error) { + content, err := ioutil.ReadFile(templateFileName) + if err != nil { + return nil, fmt.Errorf("cannot read template file %s: %s", templateFileName, err) + } + + name := path.Base(templateFileName) + return newTemplateFromContentAndImports(name, string(content), importFileNames) +} + +func newTemplateFromContentAndImports( + name, content string, + importFileNames []string, +) (*common.Template, error) { + if len(content) < 1 { + return nil, fmt.Errorf("supplied configuration is empty") + } + + imports, err := collectImportFiles(importFileNames) + if err != nil { + return nil, err + } + + return &common.Template{ + Name: name, + Content: content, + Imports: imports, + }, nil +} + +func collectImportFiles(importFileNames []string) ([]*common.ImportFile, error) { + imports := []*common.ImportFile{} + for _, importFileName := range importFileNames { + importFileData, err := ioutil.ReadFile(importFileName) + if err != nil { + return nil, fmt.Errorf("cannot read import file %s: %s", importFileName, err) + } + + imports = append(imports, + &common.ImportFile{ + Name: path.Base(importFileName), + Content: string(importFileData), + }) + } + + return imports, nil +} diff --git a/util/templateutil_test.go b/util/templateutil_test.go new file mode 100644 index 000000000..1c5a8784f --- /dev/null +++ b/util/templateutil_test.go @@ -0,0 +1,148 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/tar" + "bytes" + "testing" + + "github.com/ghodss/yaml" +) + +const invalidFileName = "afilethatdoesnotexist" + +var importFileNames = []string{ + "../test/replicatedservice.py", +} + +var ( + testTemplateName = "expandybird" + testTemplateType = "replicatedservice.py" + testTemplateProperties = ` +service_port: 8080 +target_port: 8080 +container_port: 8080 +external_service: true +replicas: 3 +image: gcr.io/dm-k8s-testing/expandybird +labels: + app: expandybird +` +) + +func TestNewTemplateFromType(t *testing.T) { + var properties map[string]interface{} + if err := yaml.Unmarshal([]byte(testTemplateProperties), &properties); err != nil { + t.Fatalf("cannot unmarshal test data: %s", err) + } + + _, err := NewTemplateFromType(testTemplateName, testTemplateType, properties) + if err != nil { + t.Fatalf("cannot create template from type %s: %s", testTemplateType, err) + } +} + +func TestNewTemplateFromReader(t *testing.T) { + r := bytes.NewReader([]byte{}) + if _, err := NewTemplateFromReader("test", r, nil); err == nil { + t.Fatalf("expected error did not occur for empty input: %s", err) + } + + r = bytes.NewReader([]byte("test")) + if _, err := NewTemplateFromReader("test", r, nil); err != nil { + t.Fatalf("cannot read test template: %s", err) + } +} + +type archiveBuilder []struct { + Name, Body string +} + +var invalidFiles = archiveBuilder{ + {"testFile1.yaml", ""}, +} + +var validFiles = archiveBuilder{ + {"testFile1.yaml", "testFile:1"}, + {"testFile2.yaml", "testFile:2"}, +} + +func generateArchive(t *testing.T, files archiveBuilder) *bytes.Reader { + buffer := new(bytes.Buffer) + tw := tar.NewWriter(buffer) + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + if _, err := tw.Write([]byte(file.Body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + r := bytes.NewReader(buffer.Bytes()) + return r +} + +func TestNewTemplateFromArchive(t *testing.T) { + r := bytes.NewReader([]byte{}) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for empty input: %s", err) + } + + r = bytes.NewReader([]byte("test")) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for non archive file:%s", err) + } + + r = generateArchive(t, invalidFiles) + if _, err := NewTemplateFromArchive(invalidFiles[0].Name, r, nil); err == nil { + t.Fatalf("expected error did not occur for empty file in archive") + } + + r = generateArchive(t, validFiles) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for missing file in archive") + } + + r = generateArchive(t, validFiles) + if _, err := NewTemplateFromArchive(validFiles[1].Name, r, nil); err != nil { + t.Fatalf("cannnot create template from valid archive") + } +} + +func TestNewTemplateFromFileNames(t *testing.T) { + if _, err := NewTemplateFromFileNames(invalidFileName, importFileNames); err == nil { + t.Fatalf("expected error did not occur for invalid template file name") + } + + _, err := NewTemplateFromFileNames(invalidFileName, []string{"afilethatdoesnotexist"}) + if err == nil { + t.Fatalf("expected error did not occur for invalid import file names") + } +}