diff --git a/dm/dm.go b/dm/dm.go index d7a38d7e2..4b3417a06 100644 --- a/dm/dm.go +++ b/dm/dm.go @@ -21,6 +21,7 @@ import ( "github.com/kubernetes/deployment-manager/registry" "github.com/kubernetes/deployment-manager/util" + "archive/tar" "bytes" "encoding/json" "flag" @@ -61,7 +62,7 @@ var commands = []string{ } var usage = func() { - message := "Usage: %s [] ( | | ( [...]))\n" + message := "Usage: %s [] [( | | ( [...]))]\n" fmt.Fprintf(os.Stderr, message, os.Args[0]) fmt.Fprintln(os.Stderr, "Commands:") for _, command := range commands { @@ -72,13 +73,15 @@ var usage = func() { fmt.Fprintln(os.Stderr, "Flags:") flag.PrintDefaults() fmt.Fprintln(os.Stderr) - os.Exit(1) + fmt.Fprintln(os.Stderr, "--stdin requires a file name and either the file contents or a tar archive containing the named file.") + fmt.Fprintln(os.Stderr, " a tar archive may include any additional files referenced directly or indirectly by the named file.") + panic("\n") } func getGitRegistry() *registry.GithubRegistry { s := strings.Split(*template_registry, "/") if len(s) < 2 { - log.Fatalf("invalid template registry: %s", *template_registry) + panic(fmt.Errorf("invalid template registry: %s", *template_registry)) } var path = "" @@ -90,6 +93,18 @@ func getGitRegistry() *registry.GithubRegistry { } func main() { + defer func() { + result := recover() + if result != nil { + log.Fatalln(result) + } + }() + + execute() + os.Exit(0) +} + +func execute() { flag.Parse() args := flag.Args() if len(args) < 1 { @@ -97,18 +112,12 @@ func main() { usage() } - if *stdin { - fmt.Printf("reading from stdin is not yet implemented") - os.Exit(0) - } - - command := args[0] - switch command { + switch args[0] { case "templates": git := getGitRegistry() templates, err := git.List() if err != nil { - log.Fatalf("Cannot list %v", err) + panic(fmt.Errorf("Cannot list %v", err)) } fmt.Printf("Templates:\n") @@ -200,12 +209,12 @@ func callService(path, method, action string, reader io.ReadCloser) { resp := callHttp(u, method, action, reader) var j interface{} if err := json.Unmarshal([]byte(resp), &j); err != nil { - log.Fatalf("Failed to parse JSON response from service: %s", resp) + panic(fmt.Errorf("Failed to parse JSON response from service: %s", resp)) } y, err := yaml.Marshal(j) if err != nil { - log.Fatalf("Failed to serialize JSON response from service: %s", resp) + panic(fmt.Errorf("Failed to serialize JSON response from service: %s", resp)) } fmt.Println(string(y)) @@ -221,19 +230,19 @@ func callHttp(path, method, action string, reader io.ReadCloser) string { response, err := client.Do(request) if err != nil { - log.Fatalf("cannot %s: %s\n", action, err) + panic(fmt.Errorf("cannot %s: %s\n", action, err)) } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { - log.Fatalf("cannot %s: %s\n", action, err) + panic(fmt.Errorf("cannot %s: %s\n", action, err)) } if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { message := fmt.Sprintf("status code: %d status: %s : %s", response.StatusCode, response.Status, body) - log.Fatalf("cannot %s: %s\n", action, message) + panic(fmt.Errorf("cannot %s: %s\n", action, message)) } return string(body) @@ -250,7 +259,7 @@ func describeType(args []string) { tUrl := getTypeUrl(args[1]) if tUrl == "" { - log.Fatalf("Invalid type name, must be a template URL or in the form \":\": %s", args[1]) + panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \":\": %s", args[1])) } schemaUrl := tUrl + ".schema" fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil)) @@ -277,7 +286,7 @@ func getDownloadUrl(t registry.Type) string { git := getGitRegistry() url, err := git.GetURL(t) if err != nil { - log.Fatalf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err) + panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)) } return url @@ -295,18 +304,43 @@ func loadTemplate(args []string) *common.Template { usage() } - if len(args) < 3 { - if t := getRegistryType(args[1]); t != nil { - template = buildTemplateFromType(*t) - } else { - template, err = expander.NewTemplateFromRootTemplate(args[1]) + if *stdin { + if len(args) < 2 { + usage() + } + + input, err := ioutil.ReadAll(os.Stdin) + if err != nil { + panic(err) + } + + r := bytes.NewReader(input) + template, err = expander.NewTemplateFromArchive(args[1], r, args[2:]) + if err != nil { + if err != tar.ErrHeader { + panic(err) + } + + r := bytes.NewReader(input) + template, err = expander.NewTemplateFromReader(args[1], r, args[2:]) + if err != nil { + panic(fmt.Errorf("cannot create configuration from supplied arguments: %s\n", err)) + } } } else { - template, err = expander.NewTemplateFromFileNames(args[1], args[2:]) - } + if len(args) < 3 { + if t := getRegistryType(args[1]); t != nil { + template = buildTemplateFromType(*t) + } else { + template, err = expander.NewTemplateFromRootTemplate(args[1]) + } + } else { + template, err = expander.NewTemplateFromFileNames(args[1], args[2:]) + } - if err != nil { - log.Fatalf("cannot create configuration from supplied arguments: %s\n", err) + if err != nil { + panic(fmt.Errorf("cannot create configuration from supplied arguments: %s\n", err)) + } } // Override name if set from flags. @@ -339,7 +373,7 @@ func buildTemplateFromType(t registry.Type) *common.Template { for _, p := range plist { ppair := strings.Split(p, "=") if len(ppair) != 2 { - log.Fatalf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p) + panic(fmt.Errorf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p)) } // support ints @@ -364,7 +398,7 @@ func buildTemplateFromType(t registry.Type) *common.Template { y, err := yaml.Marshal(config) if err != nil { - log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config) + panic(fmt.Errorf("error: %s\ncannot create configuration for deployment: %v\n", err, config)) } return &common.Template{ @@ -377,7 +411,7 @@ func buildTemplateFromType(t registry.Type) *common.Template { func marshalTemplate(template *common.Template) io.ReadCloser { j, err := json.Marshal(template) if err != nil { - log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err) + panic(fmt.Errorf("cannot deploy configuration %s: %s\n", template.Name, err)) } return ioutil.NopCloser(bytes.NewReader(j)) diff --git a/expandybird/expander/expander.go b/expandybird/expander/expander.go index 51f518b21..d39390869 100644 --- a/expandybird/expander/expander.go +++ b/expandybird/expander/expander.go @@ -14,8 +14,10 @@ limitations under the License. package expander import ( + "archive/tar" "bytes" "fmt" + "io" "io/ioutil" "log" "os/exec" @@ -40,6 +42,67 @@ func NewExpander(binary string) Expander { return &expander{binary} } +// NewTemplateFromArchive creates and returns a new template whose content +// and imported files are read from the supplied archive. +func NewTemplateFromArchive(name string, r io.Reader, importFileNames []string) (*common.Template, error) { + var content []byte + imports, err := collectImportFiles(importFileNames) + if err != nil { + return nil, err + } + + tr := tar.NewReader(r) + for i := 0; true; i++ { + hdr, err := tr.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if hdr.Name != name { + importFileData, err := ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("cannot read archive file %s: %s", hdr.Name, err) + } + + imports = append(imports, + &common.ImportFile{ + Name: path.Base(hdr.Name), + Content: string(importFileData), + }) + } else { + content, err = ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("cannot read %s from archive: %s", name, err) + } + } + } + + if len(content) < 1 { + return nil, fmt.Errorf("cannot find %s in archive", name) + } + + return &common.Template{ + Name: name, + Content: string(content), + Imports: imports, + }, nil +} + +// NewTemplateFromReader creates and returns a new template whose content +// is read from the supplied reader. +func NewTemplateFromReader(name string, r io.Reader, importFileNames []string) (*common.Template, error) { + content, err := ioutil.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("cannot read archive %s: %s", name, err) + } + + return newTemplateFromContentAndImports(name, string(content), importFileNames) +} + // NewTemplateFromRootTemplate creates and returns a new template whose content // and imported files are constructed from reading the root template, parsing out // the imports section and reading the imports from there @@ -64,6 +127,7 @@ func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, err imports = append(imports, templateDir+"/"+fileName) } } + return NewTemplateFromFileNames(templateFileName, imports[0:]) } @@ -73,17 +137,41 @@ func NewTemplateFromFileNames( templateFileName string, importFileNames []string, ) (*common.Template, error) { - name := path.Base(templateFileName) content, err := ioutil.ReadFile(templateFileName) if err != nil { - return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName) + return nil, fmt.Errorf("cannot read template file %s: %s", templateFileName, err) } + name := path.Base(templateFileName) + return newTemplateFromContentAndImports(name, string(content), importFileNames) +} + +func newTemplateFromContentAndImports( + name, content string, + importFileNames []string, +) (*common.Template, error) { + if len(content) < 1 { + return nil, fmt.Errorf("supplied configuration is empty") + } + + imports, err := collectImportFiles(importFileNames) + if err != nil { + return nil, err + } + + return &common.Template{ + Name: name, + Content: content, + Imports: imports, + }, nil +} + +func collectImportFiles(importFileNames []string) ([]*common.ImportFile, error) { imports := []*common.ImportFile{} for _, importFileName := range importFileNames { importFileData, err := ioutil.ReadFile(importFileName) if err != nil { - return nil, fmt.Errorf("cannot read import file (%s): %s", err, importFileName) + return nil, fmt.Errorf("cannot read import file %s: %s", importFileName, err) } imports = append(imports, @@ -93,11 +181,7 @@ func NewTemplateFromFileNames( }) } - return &common.Template{ - Name: name, - Content: string(content), - Imports: imports, - }, nil + return imports, nil } // ExpansionResult describes the unmarshalled output of ExpandTemplate. diff --git a/expandybird/expander/expander_test.go b/expandybird/expander/expander_test.go index 485ed197e..738730b5b 100644 --- a/expandybird/expander/expander_test.go +++ b/expandybird/expander/expander_test.go @@ -14,8 +14,13 @@ limitations under the License. package expander import ( + "archive/tar" + "bytes" "fmt" + "io" "io/ioutil" + "os" + "path" "reflect" "strings" "testing" @@ -29,7 +34,10 @@ var importFileNames = []string{ "../test/replicatedservice.py", } +var validFileName = "../test/ValidContent.yaml" var outputFileName = "../test/ExpectedOutput.yaml" +var archiveFileName = "../test/TestArchive.tar" +var expanderName = "../expansion/expansion.py" type ExpanderTestCase struct { Description string @@ -41,7 +49,7 @@ type ExpanderTestCase struct { func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template { template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames) if err != nil { - t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err) + t.Fatalf("cannot create template for test case '%s': %s", etc.Description, err) } return template @@ -50,23 +58,152 @@ func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template { func GetOutputString(t *testing.T, description string) string { output, err := ioutil.ReadFile(outputFileName) if err != nil { - t.Errorf("cannot read output file for test case '%s': %s\n", description, err) + t.Fatalf("cannot read output file for test case '%s': %s", description, err) } return string(output) } +func expandAndVerifyOutput(t *testing.T, actualOutput, description string) { + actualResult, err := NewExpansionResult(actualOutput) + if err != nil { + t.Fatalf("error in test case '%s': %s\n", description, err) + } + + expectedOutput := GetOutputString(t, description) + expectedResult, err := NewExpansionResult(expectedOutput) + if err != nil { + t.Fatalf("error in test case '%s': %s\n", description, err) + } + + if !reflect.DeepEqual(actualResult, expectedResult) { + message := fmt.Sprintf("want:\n%s\nhave:\n%s\n", expectedOutput, actualOutput) + t.Fatalf("error in test case '%s':\n%s\n", description, message) + } +} + +func testExpandTemplateFromFile(t *testing.T, fileName, baseName string, importFileNames []string, + constructor func(string, io.Reader, []string) (*common.Template, error)) { + file, err := os.Open(fileName) + if err != nil { + t.Fatalf("cannot open file %s: %s", fileName, err) + } + + template, err := constructor(baseName, file, importFileNames) + if err != nil { + t.Fatalf("cannot create template from file %s: %s", fileName, err) + } + + backend := NewExpander(expanderName) + actualOutput, err := backend.ExpandTemplate(template) + if err != nil { + t.Fatalf("cannot expand template from file %s: %s", fileName, err) + } + + description := fmt.Sprintf("test expand template from file: %s", fileName) + expandAndVerifyOutput(t, actualOutput, description) +} + +func TestNewTemplateFromReader(t *testing.T) { + r := bytes.NewReader([]byte{}) + if _, err := NewTemplateFromReader("test", r, nil); err == nil { + t.Fatalf("expected error did not occur for empty input: %s", err) + } + + r = bytes.NewReader([]byte("test")) + if _, err := NewTemplateFromReader("test", r, nil); err != nil { + t.Fatalf("cannot read test template: %s", err) + } +} + +type archiveBuilder []struct { + Name, Body string +} + +var invalidFiles = archiveBuilder{ + {"testFile1.yaml", ""}, +} + +var validFiles = archiveBuilder{ + {"testFile1.yaml", "testFile:1"}, + {"testFile2.yaml", "testFile:2"}, +} + +func generateArchive(t *testing.T, files archiveBuilder) *bytes.Reader { + buffer := new(bytes.Buffer) + tw := tar.NewWriter(buffer) + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + if _, err := tw.Write([]byte(file.Body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + r := bytes.NewReader(buffer.Bytes()) + return r +} + +func TestNewTemplateFromArchive(t *testing.T) { + r := bytes.NewReader([]byte{}) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for empty input: %s", err) + } + + r = bytes.NewReader([]byte("test")) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for non archive file:%s", err) + } + + r = generateArchive(t, invalidFiles) + if _, err := NewTemplateFromArchive(invalidFiles[0].Name, r, nil); err == nil { + t.Fatalf("expected error did not occur for empty file in archive") + } + + r = generateArchive(t, validFiles) + if _, err := NewTemplateFromArchive("", r, nil); err == nil { + t.Fatalf("expected error did not occur for missing file in archive") + } + + r = generateArchive(t, validFiles) + if _, err := NewTemplateFromArchive(validFiles[1].Name, r, nil); err != nil { + t.Fatalf("cannnot create template from valid archive") + } +} + func TestNewTemplateFromFileNames(t *testing.T) { if _, err := NewTemplateFromFileNames(invalidFileName, importFileNames); err == nil { - t.Errorf("expected error did not occur for invalid template file name") + t.Fatalf("expected error did not occur for invalid template file name") } _, err := NewTemplateFromFileNames(invalidFileName, []string{"afilethatdoesnotexist"}) if err == nil { - t.Errorf("expected error did not occur for invalid import file names") + t.Fatalf("expected error did not occur for invalid import file names") } } +func TestExpandTemplateFromReader(t *testing.T) { + baseName := path.Base(validFileName) + testExpandTemplateFromFile(t, validFileName, baseName, importFileNames, NewTemplateFromReader) +} + +func TestExpandTemplateFromArchive(t *testing.T) { + baseName := path.Base(validFileName) + testExpandTemplateFromFile(t, archiveFileName, baseName, nil, NewTemplateFromArchive) +} + var ExpanderTestCases = []ExpanderTestCase{ { "expect error for invalid file name", @@ -106,43 +243,29 @@ var ExpanderTestCases = []ExpanderTestCase{ }, { "expect success", - "../test/ValidContent.yaml", + validFileName, importFileNames, "", }, } func TestExpandTemplate(t *testing.T) { - backend := NewExpander("../expansion/expansion.py") + backend := NewExpander(expanderName) for _, etc := range ExpanderTestCases { template := etc.GetTemplate(t) actualOutput, err := backend.ExpandTemplate(template) if err != nil { message := err.Error() if !strings.Contains(message, etc.ExpectedError) { - t.Errorf("error in test case '%s': %s\n", etc.Description, message) + t.Fatalf("error in test case '%s': %s\n", etc.Description, message) } } else { if etc.ExpectedError != "" { - t.Errorf("expected error did not occur in test case '%s': %s\n", + t.Fatalf("expected error did not occur in test case '%s': %s\n", etc.Description, etc.ExpectedError) } - actualResult, err := NewExpansionResult(actualOutput) - if err != nil { - t.Errorf("error in test case '%s': %s\n", etc.Description, err) - } - - expectedOutput := GetOutputString(t, etc.Description) - expectedResult, err := NewExpansionResult(expectedOutput) - if err != nil { - t.Errorf("error in test case '%s': %s\n", etc.Description, err) - } - - if !reflect.DeepEqual(actualResult, expectedResult) { - message := fmt.Sprintf("want: %s\nhave: %s\n", expectedOutput, actualOutput) - t.Errorf("error in test case '%s': %s\n", etc.Description, message) - } + expandAndVerifyOutput(t, actualOutput, etc.Description) } } } diff --git a/expandybird/test/ExpectedOutput.yaml b/expandybird/test/ExpectedOutput.yaml index 9828d448d..24bd519a3 100644 --- a/expandybird/test/ExpectedOutput.yaml +++ b/expandybird/test/ExpectedOutput.yaml @@ -51,7 +51,7 @@ config: spec: containers: - env: [] - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird name: expandybird ports: - containerPort: 8080 @@ -63,7 +63,7 @@ layout: properties: container_port: 8080 external_service: true - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird labels: app: expandybird replicas: 3 diff --git a/expandybird/test/InvalidFileName.yaml b/expandybird/test/InvalidFileName.yaml index bde69b1a1..712426be2 100644 --- a/expandybird/test/InvalidFileName.yaml +++ b/expandybird/test/InvalidFileName.yaml @@ -19,4 +19,4 @@ resources: properties: service_port: 8080 target_port: 8080 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/InvalidProperty.yaml b/expandybird/test/InvalidProperty.yaml index 4ecd975e1..7eeb78538 100644 --- a/expandybird/test/InvalidProperty.yaml +++ b/expandybird/test/InvalidProperty.yaml @@ -19,4 +19,4 @@ resources: properties: service_port: 8080 target_port: 8080 - invalidproperty: b.gcr.io/dm-k8s-testing/expandybird + invalidproperty: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/InvalidTypeName.yaml b/expandybird/test/InvalidTypeName.yaml index 719891aa6..387e4a172 100644 --- a/expandybird/test/InvalidTypeName.yaml +++ b/expandybird/test/InvalidTypeName.yaml @@ -19,4 +19,4 @@ resources: properties: service_port: 8080 target_port: 8080 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/MissingImports.yaml b/expandybird/test/MissingImports.yaml index 900af8d0c..e6a24e6f6 100644 --- a/expandybird/test/MissingImports.yaml +++ b/expandybird/test/MissingImports.yaml @@ -18,4 +18,4 @@ resources: properties: service_port: 8080 target_port: 8080 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/MissingResourceName.yaml b/expandybird/test/MissingResourceName.yaml index 0fc117a7d..8ae93dace 100644 --- a/expandybird/test/MissingResourceName.yaml +++ b/expandybird/test/MissingResourceName.yaml @@ -18,4 +18,4 @@ resources: properties: service_port: 8080 target_port: 8080 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/MissingTypeName.yaml b/expandybird/test/MissingTypeName.yaml index 3816cc7a9..65bd433a4 100644 --- a/expandybird/test/MissingTypeName.yaml +++ b/expandybird/test/MissingTypeName.yaml @@ -18,4 +18,4 @@ resources: properties: service_port: 8080 target_port: 8080 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird diff --git a/expandybird/test/TestArchive.tar b/expandybird/test/TestArchive.tar new file mode 100644 index 000000000..2c364b000 Binary files /dev/null and b/expandybird/test/TestArchive.tar differ diff --git a/expandybird/test/ValidContent.yaml b/expandybird/test/ValidContent.yaml index 53b113e8e..37a32e153 100644 --- a/expandybird/test/ValidContent.yaml +++ b/expandybird/test/ValidContent.yaml @@ -22,6 +22,6 @@ resources: container_port: 8080 external_service: true replicas: 3 - image: b.gcr.io/dm-k8s-testing/expandybird + image: gcr.io/dm-k8s-testing/expandybird labels: app: expandybird \ No newline at end of file