Merge pull request #484 from sparkprime/go_expansion

Go expansion

cmd/goexpander/expander/expander.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package expander
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	"github.com/cloudfoundry-incubator/candiedyaml"
+	"github.com/ghodss/yaml"
+	"github.com/kubernetes/helm/pkg/expansion"
+)
+
+// parseYAMLStream takes an encoded YAML stream and turns it into a slice of JSON-marshalable
+// objects, one for each document in the stream.
+func parseYAMLStream(in io.Reader) ([]interface{}, error) {
+	// Use candiedyaml because it's the only one that supports streams.
+	decoder := candiedyaml.NewDecoder(in)
+	var document interface{}
+	stream := []interface{}{}
+	for {
+		err := decoder.Decode(&document)
+		if err != nil {
+			if strings.Contains(err.Error(), "Expected document start at line") {
+				return stream, nil
+			}
+			return nil, err
+		}
+		// Now it's held in document but we have to do a bit of a dance to get it in a form that can
+		// be marshaled as JSON for our API response.  The fundamental problem is that YAML is a
+		// superset of JSON in that it can represent non-string keys, and full IEEE floating point
+		// values (NaN etc).  JSON only allows string keys and its definition of a number is based
+		// around a sequence of digits.
+
+		// Kubernetes does not make use of these features, as it uses YAML as just "pretty JSON".
+		// Consequently this does not affect Helm either.  However, both candiedyaml and go-yaml
+		// return types that are too wide for JSON marshalling (i.e. map[interface{}]interface{}
+		// instead of map[string]interface{}), so we have to do some explicit conversion.  Luckily,
+		// ghodss/yaml has code to help with this, since decoding from YAML to JSON-marshalable
+		// values is exactly the problem that it was designed to solve.
+
+		// 1) Marshal it back to YAML string.
+		yamlBytes, err := candiedyaml.Marshal(document)
+		if err != nil {
+			return nil, err
+		}
+
+		// 2) Use ghodss/yaml to unmarshal that string into JSON-compatible data structures.
+		var jsonObj interface{}
+		if err := yaml.Unmarshal(yamlBytes, &jsonObj); err != nil {
+			return nil, err
+		}
+
+		// Now it's suitable for embedding in an API response.
+		stream = append(stream, jsonObj)
+	}
+}
+
+type expander struct {
+}
+
+// NewExpander returns an Go Templating expander.
+func NewExpander() expansion.Expander {
+	return &expander{}
+}
+
+// ExpandChart resolves the given files to a sequence of JSON-marshalable values.
+func (e *expander) ExpandChart(request *expansion.ServiceRequest) (*expansion.ServiceResponse, error) {
+
+	err := expansion.ValidateRequest(request)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO(dcunnin): Validate via JSONschema.
+
+	chartInv := request.ChartInvocation
+	chartMembers := request.Chart.Members
+
+	resources := []interface{}{}
+	for _, file := range chartMembers {
+		name := file.Path
+		content := file.Content
+		tmpl := template.New(name).Funcs(sprig.HermeticTxtFuncMap())
+
+		for _, otherFile := range chartMembers {
+			otherName := otherFile.Path
+			otherContent := otherFile.Content
+			if name == otherName {
+				continue
+			}
+			_, err := tmpl.Parse(string(otherContent))
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Have to put something in that resolves non-empty or Go templates get confused.
+		_, err := tmpl.Parse("# Content begins now")
+		if err != nil {
+			return nil, err
+		}
+
+		tmpl, err = tmpl.Parse(string(content))
+		if err != nil {
+			return nil, err
+		}
+
+		generated := bytes.NewBuffer(nil)
+		if err := tmpl.ExecuteTemplate(generated, name, chartInv.Properties); err != nil {
+			return nil, err
+		}
+
+		stream, err := parseYAMLStream(generated)
+		if err != nil {
+			return nil, fmt.Errorf("%s\nContent:\n%s", err.Error(), generated)
+		}
+
+		for _, doc := range stream {
+			resources = append(resources, doc)
+		}
+	}
+
+	return &expansion.ServiceResponse{Resources: resources}, nil
+}

cmd/goexpander/expander/expander_test.go

@@ -0,0 +1,262 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package expander
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/kubernetes/helm/pkg/chart"
+	"github.com/kubernetes/helm/pkg/common"
+	"github.com/kubernetes/helm/pkg/expansion"
+)
+
+// content provides an easy way to provide file content verbatim in tests.
+func content(lines []string) []byte {
+	return []byte(strings.Join(lines, "\n") + "\n")
+}
+
+func testExpansion(t *testing.T, req *expansion.ServiceRequest,
+	expResponse *expansion.ServiceResponse, expError string) {
+	backend := NewExpander()
+	response, err := backend.ExpandChart(req)
+	if err != nil {
+		message := err.Error()
+		if expResponse != nil || !strings.Contains(message, expError) {
+			t.Fatalf("unexpected error: %s\n", message)
+		}
+	} else {
+		if expResponse == nil {
+			t.Fatalf("expected error did not occur: %s\n", expError)
+		}
+		if !reflect.DeepEqual(response, expResponse) {
+			message := fmt.Sprintf(
+				"want:\n%s\nhave:\n%s\n", expResponse, response)
+			t.Fatalf("output mismatch:\n%s\n", message)
+		}
+	}
+}
+
+var goExpander = &chart.Expander{
+	Name:       "GoTemplating",
+	Entrypoint: "templates/main.py",
+}
+
+func TestEmpty(t *testing.T) {
+	testExpansion(
+		t,
+		&expansion.ServiceRequest{
+			ChartInvocation: &common.Resource{
+				Name: "test_invocation",
+				Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
+			},
+			Chart: &chart.Content{
+				Chartfile: &chart.Chartfile{
+					Name:     "Test",
+					Expander: goExpander,
+				},
+			},
+		},
+		&expansion.ServiceResponse{
+			Resources: []interface{}{},
+		},
+		"", // Error
+	)
+}
+
+func TestSingle(t *testing.T) {
+	testExpansion(
+		t,
+		&expansion.ServiceRequest{
+			ChartInvocation: &common.Resource{
+				Name: "test_invocation",
+				Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
+			},
+			Chart: &chart.Content{
+				Chartfile: &chart.Chartfile{
+					Name:     "Test",
+					Expander: goExpander,
+				},
+				Members: []*chart.Member{
+					{
+						Path: "templates/main.yaml",
+						Content: content([]string{
+							"name: foo",
+							"type: bar",
+						}),
+					},
+				},
+			},
+		},
+		&expansion.ServiceResponse{
+			Resources: []interface{}{
+				map[string]interface{}{
+					"name": "foo",
+					"type": "bar",
+				},
+			},
+		},
+		"", // Error
+	)
+}
+
+func TestProperties(t *testing.T) {
+	testExpansion(
+		t,
+		&expansion.ServiceRequest{
+			ChartInvocation: &common.Resource{
+				Name: "test_invocation",
+				Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
+				Properties: map[string]interface{}{
+					"prop1": 3.0,
+					"prop2": "foo",
+				},
+			},
+			Chart: &chart.Content{
+				Chartfile: &chart.Chartfile{
+					Name:     "Test",
+					Expander: goExpander,
+				},
+				Members: []*chart.Member{
+					{
+						Path: "templates/main.yaml",
+						Content: content([]string{
+							"name: foo",
+							"type: {{ .prop2 }}",
+							"properties:",
+							"  something: {{ .prop1 }}",
+						}),
+					},
+				},
+			},
+		},
+		&expansion.ServiceResponse{
+			Resources: []interface{}{
+				map[string]interface{}{
+					"name": "foo",
+					"properties": map[string]interface{}{
+						"something": 3.0,
+					},
+					"type": "foo",
+				},
+			},
+		},
+		"", // Error
+	)
+}
+
+func TestComplex(t *testing.T) {
+	testExpansion(
+		t,
+		&expansion.ServiceRequest{
+			ChartInvocation: &common.Resource{
+				Name: "test_invocation",
+				Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
+				Properties: map[string]interface{}{
+					"DatabaseName": "mydb",
+					"NumRepicas":   3,
+				},
+			},
+			Chart: &chart.Content{
+				Chartfile: &chart.Chartfile{
+					Name:     "Test",
+					Expander: goExpander,
+				},
+				Members: []*chart.Member{
+					{
+						Path: "templates/bar.tmpl",
+						Content: content([]string{
+							`{{ template "banana" . }}`,
+						}),
+					},
+					{
+						Path: "templates/base.tmpl",
+						Content: content([]string{
+							`{{ define "apple" }}`,
+							`name: Abby`,
+							`kind: Apple`,
+							`dbname: {{default "whatdb" .DatabaseName}}`,
+							`{{ end }}`,
+							``,
+							`{{ define "banana" }}`,
+							`name: Bobby`,
+							`kind: Banana`,
+							`dbname: {{default "whatdb" .DatabaseName}}`,
+							`{{ end }}`,
+						}),
+					},
+					{
+						Path: "templates/foo.tmpl",
+						Content: content([]string{
+							`---`,
+							`foo:`,
+							`  bar: baz`,
+							`---`,
+							`{{ template "apple" . }}`,
+							`---`,
+							`{{ template "apple" . }}`,
+							`...`,
+						}),
+					},
+					{
+						Path: "templates/docs.txt",
+						Content: content([]string{
+							`{{/*`,
+							`File contains only a comment.`,
+							`Suitable for documentation within templates/`,
+							`*/}}`,
+						}),
+					},
+					{
+						Path: "templates/docs2.txt",
+						Content: content([]string{
+							`# File contains only a comment.`,
+							`# Suitable for documentation within templates/`,
+						}),
+					},
+				},
+			},
+		},
+		&expansion.ServiceResponse{
+			Resources: []interface{}{
+				map[string]interface{}{
+					"name":   "Bobby",
+					"kind":   "Banana",
+					"dbname": "mydb",
+				},
+				map[string]interface{}{
+					"foo": map[string]interface{}{
+						"bar": "baz",
+					},
+				},
+				map[string]interface{}{
+					"name":   "Abby",
+					"kind":   "Apple",
+					"dbname": "mydb",
+				},
+				map[string]interface{}{
+					"name":   "Abby",
+					"kind":   "Apple",
+					"dbname": "mydb",
+				},
+			},
+		},
+		"", // Error
+	)
+}

cmd/goexpander/main.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/kubernetes/helm/cmd/goexpander/expander"
+	"github.com/kubernetes/helm/pkg/expansion"
+	"github.com/kubernetes/helm/pkg/version"
+
+	"flag"
+	"log"
+)
+
+// interface that we are going to listen on
+var address = flag.String("address", "", "Interface to listen on")
+
+// port that we are going to listen on
+var port = flag.Int("port", 8080, "Port to listen on")
+
+func main() {
+	flag.Parse()
+	backend := expander.NewExpander()
+	service := expansion.NewService(*address, *port, backend)
+	log.Printf("Version: %s", version.Version)
+	log.Printf("Listening on http://%s:%d/expand", *address, *port)
+	log.Fatal(service.ListenAndServe())
+}

glide.lock

@@ -1,8 +1,10 @@
-hash: 5a891985ff2f22aed4eaa4f7f3819f0866750c065c7acd0118415c34788bc665
+hash: a72f13699934fc94c8df5b56c76fc66d2279b2a4ff1396c2f1bff944456a8adf
-updated: 2016-03-30T15:52:37.011995847-06:00
+updated: 2016-03-30T23:15:39.980396345-04:00
 imports:
 - name: github.com/aokoli/goutils
   version: 45307ec16e3cd47cd841506c081f7afd8237d210
+- name: github.com/cloudfoundry-incubator/candiedyaml
+  version: 479485e9bfc69ee37d074b36ce36da5e4fba7941
 - name: github.com/codegangsta/cli
   version: a2943485b110df8842045ae0600047f88a3a56a1
 - name: github.com/emicklei/go-restful
@@ -46,6 +48,10 @@ imports:
   - gensupport
   - googleapi
   - googleapi/internal/uritemplates
+- name: google.golang.org/appengine
+  version: a503df954af258b9a70918df2a524d6a85ecefdb
+  subpackages:
+  - urlfetch
 - name: google.golang.org/cloud
   version: fb10e8da373d97f6ba5e648299a10b3b91f14cd5
   subpackages:
@@ -55,6 +61,7 @@ imports:
   version: d90005c5262a3463800497ea5a89aed5fe22c886
   subpackages:
   - bson
+  - internal/sasl
   - internal/scram
 - name: gopkg.in/yaml.v2
   version: f7716cbe52baa25d2e9b0d0da546fcf909fc16b4

glide.yaml

@@ -7,6 +7,7 @@ import:
   version: ^1.1.0
 - package: github.com/aokoli/goutils
   version: ^1.0.0
+- package: github.com/cloudfoundry-incubator/candiedyaml
 - package: github.com/codegangsta/cli
 - package: github.com/emicklei/go-restful
 - package: github.com/ghodss/yaml

scripts/common.sh

@@ -18,7 +18,7 @@ set -o pipefail
 
 readonly ROOTFS="${DIR}/rootfs"
 
-readonly STATIC_TARGETS=(cmd/expandybird cmd/manager cmd/resourcifier)
+readonly STATIC_TARGETS=(cmd/expandybird cmd/goexpander cmd/manager cmd/resourcifier)
 readonly ALL_TARGETS=(${STATIC_TARGETS[@]} cmd/helm)
 
 error_exit() {