Merge pull request #131 from jackgr/implement-stdin

Implement --stdin flag on command line.

dm/dm.go

@@ -21,6 +21,7 @@ import (
 	"github.com/kubernetes/deployment-manager/registry"
 	"github.com/kubernetes/deployment-manager/util"
 
+	"archive/tar"
 	"bytes"
 	"encoding/json"
 	"flag"
@@ -61,7 +62,7 @@ var commands = []string{
 }
 
 var usage = func() {
-	message := "Usage: %s [<flags>] <command> (<template-name> | <deployment-name> | (<configuration> [<import1>...<importN>]))\n"
+	message := "Usage: %s [<flags>] <command> [(<template-name> | <deployment-name> | (<configuration> [<import1>...<importN>]))]\n"
 	fmt.Fprintf(os.Stderr, message, os.Args[0])
 	fmt.Fprintln(os.Stderr, "Commands:")
 	for _, command := range commands {
@@ -72,13 +73,15 @@ var usage = func() {
 	fmt.Fprintln(os.Stderr, "Flags:")
 	flag.PrintDefaults()
 	fmt.Fprintln(os.Stderr)
-	os.Exit(1)
+	fmt.Fprintln(os.Stderr, "--stdin requires a file name and either the file contents or a tar archive containing the named file.")
+	fmt.Fprintln(os.Stderr, "        a tar archive may include any additional files referenced directly or indirectly by the named file.")
+	panic("\n")
 }
 
 func getGitRegistry() *registry.GithubRegistry {
 	s := strings.Split(*template_registry, "/")
 	if len(s) < 2 {
-		log.Fatalf("invalid template registry: %s", *template_registry)
+		panic(fmt.Errorf("invalid template registry: %s", *template_registry))
 	}
 
 	var path = ""
@@ -90,6 +93,18 @@ func getGitRegistry() *registry.GithubRegistry {
 }
 
 func main() {
+	defer func() {
+		result := recover()
+		if result != nil {
+			log.Fatalln(result)
+		}
+	}()
+
+	execute()
+	os.Exit(0)
+}
+
+func execute() {
 	flag.Parse()
 	args := flag.Args()
 	if len(args) < 1 {
@@ -97,18 +112,12 @@ func main() {
 		usage()
 	}
 
-	if *stdin {
+	switch args[0] {
-		fmt.Printf("reading from stdin is not yet implemented")
-		os.Exit(0)
-	}
-
-	command := args[0]
-	switch command {
 	case "templates":
 		git := getGitRegistry()
 		templates, err := git.List()
 		if err != nil {
-			log.Fatalf("Cannot list %v", err)
+			panic(fmt.Errorf("Cannot list %v", err))
 		}
 
 		fmt.Printf("Templates:\n")
@@ -200,12 +209,12 @@ func callService(path, method, action string, reader io.ReadCloser) {
 	resp := callHttp(u, method, action, reader)
 	var j interface{}
 	if err := json.Unmarshal([]byte(resp), &j); err != nil {
-		log.Fatalf("Failed to parse JSON response from service: %s", resp)
+		panic(fmt.Errorf("Failed to parse JSON response from service: %s", resp))
 	}
 
 	y, err := yaml.Marshal(j)
 	if err != nil {
-		log.Fatalf("Failed to serialize JSON response from service: %s", resp)
+		panic(fmt.Errorf("Failed to serialize JSON response from service: %s", resp))
 	}
 
 	fmt.Println(string(y))
@@ -221,19 +230,19 @@ func callHttp(path, method, action string, reader io.ReadCloser) string {
 
 	response, err := client.Do(request)
 	if err != nil {
-		log.Fatalf("cannot %s: %s\n", action, err)
+		panic(fmt.Errorf("cannot %s: %s\n", action, err))
 	}
 
 	defer response.Body.Close()
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		log.Fatalf("cannot %s: %s\n", action, err)
+		panic(fmt.Errorf("cannot %s: %s\n", action, err))
 	}
 
 	if response.StatusCode < http.StatusOK ||
 		response.StatusCode >= http.StatusMultipleChoices {
 		message := fmt.Sprintf("status code: %d status: %s : %s", response.StatusCode, response.Status, body)
-		log.Fatalf("cannot %s: %s\n", action, message)
+		panic(fmt.Errorf("cannot %s: %s\n", action, message))
 	}
 
 	return string(body)
@@ -250,7 +259,7 @@ func describeType(args []string) {
 
 	tUrl := getTypeUrl(args[1])
 	if tUrl == "" {
-		log.Fatalf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1])
+		panic(fmt.Errorf("Invalid type name, must be a template URL or in the form \"<type-name>:<version>\": %s", args[1]))
 	}
 	schemaUrl := tUrl + ".schema"
 	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil))
@@ -277,7 +286,7 @@ func getDownloadUrl(t registry.Type) string {
 	git := getGitRegistry()
 	url, err := git.GetURL(t)
 	if err != nil {
-		log.Fatalf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err)
+		panic(fmt.Errorf("Failed to fetch type information for \"%s:%s\": %s", t.Name, t.Version, err))
 	}
 
 	return url
@@ -295,18 +304,43 @@ func loadTemplate(args []string) *common.Template {
 		usage()
 	}
 
-	if len(args) < 3 {
+	if *stdin {
-		if t := getRegistryType(args[1]); t != nil {
+		if len(args) < 2 {
-			template = buildTemplateFromType(*t)
+			usage()
-		} else {
+		}
-			template, err = expander.NewTemplateFromRootTemplate(args[1])
+
+		input, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			panic(err)
+		}
+
+		r := bytes.NewReader(input)
+		template, err = expander.NewTemplateFromArchive(args[1], r, args[2:])
+		if err != nil {
+			if err != tar.ErrHeader {
+				panic(err)
+			}
+
+			r := bytes.NewReader(input)
+			template, err = expander.NewTemplateFromReader(args[1], r, args[2:])
+			if err != nil {
+				panic(fmt.Errorf("cannot create configuration from supplied arguments: %s\n", err))
+			}
 		}
 	} else {
-		template, err = expander.NewTemplateFromFileNames(args[1], args[2:])
+		if len(args) < 3 {
-	}
+			if t := getRegistryType(args[1]); t != nil {
+				template = buildTemplateFromType(*t)
+			} else {
+				template, err = expander.NewTemplateFromRootTemplate(args[1])
+			}
+		} else {
+			template, err = expander.NewTemplateFromFileNames(args[1], args[2:])
+		}
 
-	if err != nil {
+		if err != nil {
-		log.Fatalf("cannot create configuration from supplied arguments: %s\n", err)
+			panic(fmt.Errorf("cannot create configuration from supplied arguments: %s\n", err))
+		}
 	}
 
 	// Override name if set from flags.
@@ -339,7 +373,7 @@ func buildTemplateFromType(t registry.Type) *common.Template {
 		for _, p := range plist {
 			ppair := strings.Split(p, "=")
 			if len(ppair) != 2 {
-				log.Fatalf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p)
+				panic(fmt.Errorf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p))
 			}
 
 			// support ints
@@ -364,7 +398,7 @@ func buildTemplateFromType(t registry.Type) *common.Template {
 
 	y, err := yaml.Marshal(config)
 	if err != nil {
-		log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
+		panic(fmt.Errorf("error: %s\ncannot create configuration for deployment: %v\n", err, config))
 	}
 
 	return &common.Template{
@@ -377,7 +411,7 @@ func buildTemplateFromType(t registry.Type) *common.Template {
 func marshalTemplate(template *common.Template) io.ReadCloser {
 	j, err := json.Marshal(template)
 	if err != nil {
-		log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)
+		panic(fmt.Errorf("cannot deploy configuration %s: %s\n", template.Name, err))
 	}
 
 	return ioutil.NopCloser(bytes.NewReader(j))

expandybird/expander/expander.go

@@ -14,8 +14,10 @@ limitations under the License.
 package expander
 
 import (
+	"archive/tar"
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os/exec"
@@ -40,6 +42,67 @@ func NewExpander(binary string) Expander {
 	return &expander{binary}
 }
 
+// NewTemplateFromArchive creates and returns a new template whose content
+// and imported files are read from the supplied archive.
+func NewTemplateFromArchive(name string, r io.Reader, importFileNames []string) (*common.Template, error) {
+	var content []byte
+	imports, err := collectImportFiles(importFileNames)
+	if err != nil {
+		return nil, err
+	}
+
+	tr := tar.NewReader(r)
+	for i := 0; true; i++ {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		if hdr.Name != name {
+			importFileData, err := ioutil.ReadAll(tr)
+			if err != nil {
+				return nil, fmt.Errorf("cannot read archive file %s: %s", hdr.Name, err)
+			}
+
+			imports = append(imports,
+				&common.ImportFile{
+					Name:    path.Base(hdr.Name),
+					Content: string(importFileData),
+				})
+		} else {
+			content, err = ioutil.ReadAll(tr)
+			if err != nil {
+				return nil, fmt.Errorf("cannot read %s from archive: %s", name, err)
+			}
+		}
+	}
+
+	if len(content) < 1 {
+		return nil, fmt.Errorf("cannot find %s in archive", name)
+	}
+
+	return &common.Template{
+		Name:    name,
+		Content: string(content),
+		Imports: imports,
+	}, nil
+}
+
+// NewTemplateFromReader creates and returns a new template whose content
+// is read from the supplied reader.
+func NewTemplateFromReader(name string, r io.Reader, importFileNames []string) (*common.Template, error) {
+	content, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read archive %s: %s", name, err)
+	}
+
+	return newTemplateFromContentAndImports(name, string(content), importFileNames)
+}
+
 // NewTemplateFromRootTemplate creates and returns a new template whose content
 // and imported files are constructed from reading the root template, parsing out
 // the imports section and reading the imports from there
@@ -64,6 +127,7 @@ func NewTemplateFromRootTemplate(templateFileName string) (*common.Template, err
 			imports = append(imports, templateDir+"/"+fileName)
 		}
 	}
+
 	return NewTemplateFromFileNames(templateFileName, imports[0:])
 }
 
@@ -73,17 +137,41 @@ func NewTemplateFromFileNames(
 	templateFileName string,
 	importFileNames []string,
 ) (*common.Template, error) {
-	name := path.Base(templateFileName)
 	content, err := ioutil.ReadFile(templateFileName)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read template file (%s): %s", err, templateFileName)
+		return nil, fmt.Errorf("cannot read template file %s: %s", templateFileName, err)
 	}
 
+	name := path.Base(templateFileName)
+	return newTemplateFromContentAndImports(name, string(content), importFileNames)
+}
+
+func newTemplateFromContentAndImports(
+	name, content string,
+	importFileNames []string,
+) (*common.Template, error) {
+	if len(content) < 1 {
+		return nil, fmt.Errorf("supplied configuration is empty")
+	}
+
+	imports, err := collectImportFiles(importFileNames)
+	if err != nil {
+		return nil, err
+	}
+
+	return &common.Template{
+		Name:    name,
+		Content: content,
+		Imports: imports,
+	}, nil
+}
+
+func collectImportFiles(importFileNames []string) ([]*common.ImportFile, error) {
 	imports := []*common.ImportFile{}
 	for _, importFileName := range importFileNames {
 		importFileData, err := ioutil.ReadFile(importFileName)
 		if err != nil {
-			return nil, fmt.Errorf("cannot read import file (%s): %s", err, importFileName)
+			return nil, fmt.Errorf("cannot read import file %s: %s", importFileName, err)
 		}
 
 		imports = append(imports,
@@ -93,11 +181,7 @@ func NewTemplateFromFileNames(
 			})
 	}
 
-	return &common.Template{
+	return imports, nil
-		Name:    name,
-		Content: string(content),
-		Imports: imports,
-	}, nil
 }
 
 // ExpansionResult describes the unmarshalled output of ExpandTemplate.

expandybird/expander/expander_test.go

@@ -14,8 +14,13 @@ limitations under the License.
 package expander
 
 import (
+	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"os"
+	"path"
 	"reflect"
 	"strings"
 	"testing"
@@ -29,7 +34,10 @@ var importFileNames = []string{
 	"../test/replicatedservice.py",
 }
 
+var validFileName = "../test/ValidContent.yaml"
 var outputFileName = "../test/ExpectedOutput.yaml"
+var archiveFileName = "../test/TestArchive.tar"
+var expanderName = "../expansion/expansion.py"
 
 type ExpanderTestCase struct {
 	Description      string
@@ -41,7 +49,7 @@ type ExpanderTestCase struct {
 func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template {
 	template, err := NewTemplateFromFileNames(etc.TemplateFileName, etc.ImportFileNames)
 	if err != nil {
-		t.Errorf("cannot create template for test case '%s': %s\n", etc.Description, err)
+		t.Fatalf("cannot create template for test case '%s': %s", etc.Description, err)
 	}
 
 	return template
@@ -50,23 +58,152 @@ func (etc *ExpanderTestCase) GetTemplate(t *testing.T) *common.Template {
 func GetOutputString(t *testing.T, description string) string {
 	output, err := ioutil.ReadFile(outputFileName)
 	if err != nil {
-		t.Errorf("cannot read output file for test case '%s': %s\n", description, err)
+		t.Fatalf("cannot read output file for test case '%s': %s", description, err)
 	}
 
 	return string(output)
 }
 
+func expandAndVerifyOutput(t *testing.T, actualOutput, description string) {
+	actualResult, err := NewExpansionResult(actualOutput)
+	if err != nil {
+		t.Fatalf("error in test case '%s': %s\n", description, err)
+	}
+
+	expectedOutput := GetOutputString(t, description)
+	expectedResult, err := NewExpansionResult(expectedOutput)
+	if err != nil {
+		t.Fatalf("error in test case '%s': %s\n", description, err)
+	}
+
+	if !reflect.DeepEqual(actualResult, expectedResult) {
+		message := fmt.Sprintf("want:\n%s\nhave:\n%s\n", expectedOutput, actualOutput)
+		t.Fatalf("error in test case '%s':\n%s\n", description, message)
+	}
+}
+
+func testExpandTemplateFromFile(t *testing.T, fileName, baseName string, importFileNames []string,
+	constructor func(string, io.Reader, []string) (*common.Template, error)) {
+	file, err := os.Open(fileName)
+	if err != nil {
+		t.Fatalf("cannot open file %s: %s", fileName, err)
+	}
+
+	template, err := constructor(baseName, file, importFileNames)
+	if err != nil {
+		t.Fatalf("cannot create template from file %s: %s", fileName, err)
+	}
+
+	backend := NewExpander(expanderName)
+	actualOutput, err := backend.ExpandTemplate(template)
+	if err != nil {
+		t.Fatalf("cannot expand template from file %s: %s", fileName, err)
+	}
+
+	description := fmt.Sprintf("test expand template from file: %s", fileName)
+	expandAndVerifyOutput(t, actualOutput, description)
+}
+
+func TestNewTemplateFromReader(t *testing.T) {
+	r := bytes.NewReader([]byte{})
+	if _, err := NewTemplateFromReader("test", r, nil); err == nil {
+		t.Fatalf("expected error did not occur for empty input: %s", err)
+	}
+
+	r = bytes.NewReader([]byte("test"))
+	if _, err := NewTemplateFromReader("test", r, nil); err != nil {
+		t.Fatalf("cannot read test template: %s", err)
+	}
+}
+
+type archiveBuilder []struct {
+	Name, Body string
+}
+
+var invalidFiles = archiveBuilder{
+	{"testFile1.yaml", ""},
+}
+
+var validFiles = archiveBuilder{
+	{"testFile1.yaml", "testFile:1"},
+	{"testFile2.yaml", "testFile:2"},
+}
+
+func generateArchive(t *testing.T, files archiveBuilder) *bytes.Reader {
+	buffer := new(bytes.Buffer)
+	tw := tar.NewWriter(buffer)
+	for _, file := range files {
+		hdr := &tar.Header{
+			Name: file.Name,
+			Mode: 0600,
+			Size: int64(len(file.Body)),
+		}
+
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := tw.Write([]byte(file.Body)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r := bytes.NewReader(buffer.Bytes())
+	return r
+}
+
+func TestNewTemplateFromArchive(t *testing.T) {
+	r := bytes.NewReader([]byte{})
+	if _, err := NewTemplateFromArchive("", r, nil); err == nil {
+		t.Fatalf("expected error did not occur for empty input: %s", err)
+	}
+
+	r = bytes.NewReader([]byte("test"))
+	if _, err := NewTemplateFromArchive("", r, nil); err == nil {
+		t.Fatalf("expected error did not occur for non archive file:%s", err)
+	}
+
+	r = generateArchive(t, invalidFiles)
+	if _, err := NewTemplateFromArchive(invalidFiles[0].Name, r, nil); err == nil {
+		t.Fatalf("expected error did not occur for empty file in archive")
+	}
+
+	r = generateArchive(t, validFiles)
+	if _, err := NewTemplateFromArchive("", r, nil); err == nil {
+		t.Fatalf("expected error did not occur for missing file in archive")
+	}
+
+	r = generateArchive(t, validFiles)
+	if _, err := NewTemplateFromArchive(validFiles[1].Name, r, nil); err != nil {
+		t.Fatalf("cannnot create template from valid archive")
+	}
+}
+
 func TestNewTemplateFromFileNames(t *testing.T) {
 	if _, err := NewTemplateFromFileNames(invalidFileName, importFileNames); err == nil {
-		t.Errorf("expected error did not occur for invalid template file name")
+		t.Fatalf("expected error did not occur for invalid template file name")
 	}
 
 	_, err := NewTemplateFromFileNames(invalidFileName, []string{"afilethatdoesnotexist"})
 	if err == nil {
-		t.Errorf("expected error did not occur for invalid import file names")
+		t.Fatalf("expected error did not occur for invalid import file names")
 	}
 }
 
+func TestExpandTemplateFromReader(t *testing.T) {
+	baseName := path.Base(validFileName)
+	testExpandTemplateFromFile(t, validFileName, baseName, importFileNames, NewTemplateFromReader)
+}
+
+func TestExpandTemplateFromArchive(t *testing.T) {
+	baseName := path.Base(validFileName)
+	testExpandTemplateFromFile(t, archiveFileName, baseName, nil, NewTemplateFromArchive)
+}
+
 var ExpanderTestCases = []ExpanderTestCase{
 	{
 		"expect error for invalid file name",
@@ -106,43 +243,29 @@ var ExpanderTestCases = []ExpanderTestCase{
 	},
 	{
 		"expect success",
-		"../test/ValidContent.yaml",
+		validFileName,
 		importFileNames,
 		"",
 	},
 }
 
 func TestExpandTemplate(t *testing.T) {
-	backend := NewExpander("../expansion/expansion.py")
+	backend := NewExpander(expanderName)
 	for _, etc := range ExpanderTestCases {
 		template := etc.GetTemplate(t)
 		actualOutput, err := backend.ExpandTemplate(template)
 		if err != nil {
 			message := err.Error()
 			if !strings.Contains(message, etc.ExpectedError) {
-				t.Errorf("error in test case '%s': %s\n", etc.Description, message)
+				t.Fatalf("error in test case '%s': %s\n", etc.Description, message)
 			}
 		} else {
 			if etc.ExpectedError != "" {
-				t.Errorf("expected error did not occur in test case '%s': %s\n",
+				t.Fatalf("expected error did not occur in test case '%s': %s\n",
 					etc.Description, etc.ExpectedError)
 			}
 
-			actualResult, err := NewExpansionResult(actualOutput)
+			expandAndVerifyOutput(t, actualOutput, etc.Description)
-			if err != nil {
-				t.Errorf("error in test case '%s': %s\n", etc.Description, err)
-			}
-
-			expectedOutput := GetOutputString(t, etc.Description)
-			expectedResult, err := NewExpansionResult(expectedOutput)
-			if err != nil {
-				t.Errorf("error in test case '%s': %s\n", etc.Description, err)
-			}
-
-			if !reflect.DeepEqual(actualResult, expectedResult) {
-				message := fmt.Sprintf("want: %s\nhave: %s\n", expectedOutput, actualOutput)
-				t.Errorf("error in test case '%s': %s\n", etc.Description, message)
-			}
 		}
 	}
 }

expandybird/test/ExpectedOutput.yaml

@@ -51,7 +51,7 @@ config:
           spec:
             containers:
             - env: []
-              image: b.gcr.io/dm-k8s-testing/expandybird
+              image: gcr.io/dm-k8s-testing/expandybird
               name: expandybird
               ports:
               - containerPort: 8080
@@ -63,7 +63,7 @@ layout:
     properties:
       container_port: 8080
       external_service: true
-      image: b.gcr.io/dm-k8s-testing/expandybird
+      image: gcr.io/dm-k8s-testing/expandybird
       labels:
         app: expandybird
       replicas: 3

expandybird/test/InvalidFileName.yaml

@@ -19,4 +19,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird

expandybird/test/InvalidProperty.yaml

@@ -19,4 +19,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    invalidproperty: b.gcr.io/dm-k8s-testing/expandybird
+    invalidproperty: gcr.io/dm-k8s-testing/expandybird

expandybird/test/InvalidTypeName.yaml

@@ -19,4 +19,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird

expandybird/test/MissingImports.yaml

@@ -18,4 +18,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird

expandybird/test/MissingResourceName.yaml

@@ -18,4 +18,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird

expandybird/test/MissingTypeName.yaml

@@ -18,4 +18,4 @@ resources:
   properties:
     service_port: 8080
     target_port: 8080
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird

expandybird/test/ValidContent.yaml

@@ -22,6 +22,6 @@ resources:
     container_port: 8080
     external_service: true
     replicas: 3
-    image: b.gcr.io/dm-k8s-testing/expandybird
+    image: gcr.io/dm-k8s-testing/expandybird
     labels:
       app: expandybird