diff --git a/cmd/goexpander/expander/expander.go b/cmd/goexpander/expander/expander.go new file mode 100644 index 000000000..0913bd230 --- /dev/null +++ b/cmd/goexpander/expander/expander.go @@ -0,0 +1,143 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expander + +import ( + "bytes" + "fmt" + "io" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + "github.com/cloudfoundry-incubator/candiedyaml" + "github.com/ghodss/yaml" + "github.com/kubernetes/helm/pkg/expansion" +) + +// parseYAMLStream takes an encoded YAML stream and turns it into a slice of JSON-marshalable +// objects, one for each document in the stream. +func parseYAMLStream(in io.Reader) ([]interface{}, error) { + // Use candiedyaml because it's the only one that supports streams. + decoder := candiedyaml.NewDecoder(in) + var document interface{} + stream := []interface{}{} + for { + err := decoder.Decode(&document) + if err != nil { + if strings.Contains(err.Error(), "Expected document start at line") { + return stream, nil + } + return nil, err + } + // Now it's held in document but we have to do a bit of a dance to get it in a form that can + // be marshaled as JSON for our API response. The fundamental problem is that YAML is a + // superset of JSON in that it can represent non-string keys, and full IEEE floating point + // values (NaN etc). JSON only allows string keys and its definition of a number is based + // around a sequence of digits. + + // Kubernetes does not make use of these features, as it uses YAML as just "pretty JSON". + // Consequently this does not affect Helm either. However, both candiedyaml and go-yaml + // return types that are too wide for JSON marshalling (i.e. map[interface{}]interface{} + // instead of map[string]interface{}), so we have to do some explicit conversion. Luckily, + // ghodss/yaml has code to help with this, since decoding from YAML to JSON-marshalable + // values is exactly the problem that it was designed to solve. + + // 1) Marshal it back to YAML string. + yamlBytes, err := candiedyaml.Marshal(document) + if err != nil { + return nil, err + } + + // 2) Use ghodss/yaml to unmarshal that string into JSON-compatible data structures. + var jsonObj interface{} + if err := yaml.Unmarshal(yamlBytes, &jsonObj); err != nil { + return nil, err + } + + // Now it's suitable for embedding in an API response. + stream = append(stream, jsonObj) + } +} + +type expander struct { +} + +// NewExpander returns an Go Templating expander. +func NewExpander() expansion.Expander { + return &expander{} +} + +// ExpandChart resolves the given files to a sequence of JSON-marshalable values. +func (e *expander) ExpandChart(request *expansion.ServiceRequest) (*expansion.ServiceResponse, error) { + + err := expansion.ValidateRequest(request) + if err != nil { + return nil, err + } + + // TODO(dcunnin): Validate via JSONschema. + + chartInv := request.ChartInvocation + chartMembers := request.Chart.Members + + resources := []interface{}{} + for _, file := range chartMembers { + name := file.Path + content := file.Content + tmpl := template.New(name).Funcs(sprig.HermeticTxtFuncMap()) + + for _, otherFile := range chartMembers { + otherName := otherFile.Path + otherContent := otherFile.Content + if name == otherName { + continue + } + _, err := tmpl.Parse(string(otherContent)) + if err != nil { + return nil, err + } + } + + // Have to put something in that resolves non-empty or Go templates get confused. + _, err := tmpl.Parse("# Content begins now") + if err != nil { + return nil, err + } + + tmpl, err = tmpl.Parse(string(content)) + if err != nil { + return nil, err + } + + generated := bytes.NewBuffer(nil) + if err := tmpl.ExecuteTemplate(generated, name, chartInv.Properties); err != nil { + return nil, err + } + + stream, err := parseYAMLStream(generated) + if err != nil { + return nil, fmt.Errorf("%s\nContent:\n%s", err.Error(), generated) + } + + for _, doc := range stream { + resources = append(resources, doc) + } + } + + return &expansion.ServiceResponse{Resources: resources}, nil +} diff --git a/cmd/goexpander/expander/expander_test.go b/cmd/goexpander/expander/expander_test.go new file mode 100644 index 000000000..95ce37c9d --- /dev/null +++ b/cmd/goexpander/expander/expander_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expander + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/kubernetes/helm/pkg/chart" + "github.com/kubernetes/helm/pkg/common" + "github.com/kubernetes/helm/pkg/expansion" +) + +// content provides an easy way to provide file content verbatim in tests. +func content(lines []string) []byte { + return []byte(strings.Join(lines, "\n") + "\n") +} + +func testExpansion(t *testing.T, req *expansion.ServiceRequest, + expResponse *expansion.ServiceResponse, expError string) { + backend := NewExpander() + response, err := backend.ExpandChart(req) + if err != nil { + message := err.Error() + if expResponse != nil || !strings.Contains(message, expError) { + t.Fatalf("unexpected error: %s\n", message) + } + } else { + if expResponse == nil { + t.Fatalf("expected error did not occur: %s\n", expError) + } + if !reflect.DeepEqual(response, expResponse) { + message := fmt.Sprintf( + "want:\n%s\nhave:\n%s\n", expResponse, response) + t.Fatalf("output mismatch:\n%s\n", message) + } + } +} + +var goExpander = &chart.Expander{ + Name: "GoTemplating", + Entrypoint: "templates/main.py", +} + +func TestEmpty(t *testing.T) { + testExpansion( + t, + &expansion.ServiceRequest{ + ChartInvocation: &common.Resource{ + Name: "test_invocation", + Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz", + }, + Chart: &chart.Content{ + Chartfile: &chart.Chartfile{ + Name: "Test", + Expander: goExpander, + }, + }, + }, + &expansion.ServiceResponse{ + Resources: []interface{}{}, + }, + "", // Error + ) +} + +func TestSingle(t *testing.T) { + testExpansion( + t, + &expansion.ServiceRequest{ + ChartInvocation: &common.Resource{ + Name: "test_invocation", + Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz", + }, + Chart: &chart.Content{ + Chartfile: &chart.Chartfile{ + Name: "Test", + Expander: goExpander, + }, + Members: []*chart.Member{ + { + Path: "templates/main.yaml", + Content: content([]string{ + "name: foo", + "type: bar", + }), + }, + }, + }, + }, + &expansion.ServiceResponse{ + Resources: []interface{}{ + map[string]interface{}{ + "name": "foo", + "type": "bar", + }, + }, + }, + "", // Error + ) +} + +func TestProperties(t *testing.T) { + testExpansion( + t, + &expansion.ServiceRequest{ + ChartInvocation: &common.Resource{ + Name: "test_invocation", + Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz", + Properties: map[string]interface{}{ + "prop1": 3.0, + "prop2": "foo", + }, + }, + Chart: &chart.Content{ + Chartfile: &chart.Chartfile{ + Name: "Test", + Expander: goExpander, + }, + Members: []*chart.Member{ + { + Path: "templates/main.yaml", + Content: content([]string{ + "name: foo", + "type: {{ .prop2 }}", + "properties:", + " something: {{ .prop1 }}", + }), + }, + }, + }, + }, + &expansion.ServiceResponse{ + Resources: []interface{}{ + map[string]interface{}{ + "name": "foo", + "properties": map[string]interface{}{ + "something": 3.0, + }, + "type": "foo", + }, + }, + }, + "", // Error + ) +} + +func TestComplex(t *testing.T) { + testExpansion( + t, + &expansion.ServiceRequest{ + ChartInvocation: &common.Resource{ + Name: "test_invocation", + Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz", + Properties: map[string]interface{}{ + "DatabaseName": "mydb", + "NumRepicas": 3, + }, + }, + Chart: &chart.Content{ + Chartfile: &chart.Chartfile{ + Name: "Test", + Expander: goExpander, + }, + Members: []*chart.Member{ + { + Path: "templates/bar.tmpl", + Content: content([]string{ + `{{ template "banana" . }}`, + }), + }, + { + Path: "templates/base.tmpl", + Content: content([]string{ + `{{ define "apple" }}`, + `name: Abby`, + `kind: Apple`, + `dbname: {{default "whatdb" .DatabaseName}}`, + `{{ end }}`, + ``, + `{{ define "banana" }}`, + `name: Bobby`, + `kind: Banana`, + `dbname: {{default "whatdb" .DatabaseName}}`, + `{{ end }}`, + }), + }, + { + Path: "templates/foo.tmpl", + Content: content([]string{ + `---`, + `foo:`, + ` bar: baz`, + `---`, + `{{ template "apple" . }}`, + `---`, + `{{ template "apple" . }}`, + `...`, + }), + }, + { + Path: "templates/docs.txt", + Content: content([]string{ + `{{/*`, + `File contains only a comment.`, + `Suitable for documentation within templates/`, + `*/}}`, + }), + }, + { + Path: "templates/docs2.txt", + Content: content([]string{ + `# File contains only a comment.`, + `# Suitable for documentation within templates/`, + }), + }, + }, + }, + }, + &expansion.ServiceResponse{ + Resources: []interface{}{ + map[string]interface{}{ + "name": "Bobby", + "kind": "Banana", + "dbname": "mydb", + }, + map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": "baz", + }, + }, + map[string]interface{}{ + "name": "Abby", + "kind": "Apple", + "dbname": "mydb", + }, + map[string]interface{}{ + "name": "Abby", + "kind": "Apple", + "dbname": "mydb", + }, + }, + }, + "", // Error + ) +} diff --git a/cmd/goexpander/main.go b/cmd/goexpander/main.go new file mode 100644 index 000000000..ded71a35c --- /dev/null +++ b/cmd/goexpander/main.go @@ -0,0 +1,41 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/kubernetes/helm/cmd/goexpander/expander" + "github.com/kubernetes/helm/pkg/expansion" + "github.com/kubernetes/helm/pkg/version" + + "flag" + "log" +) + +// interface that we are going to listen on +var address = flag.String("address", "", "Interface to listen on") + +// port that we are going to listen on +var port = flag.Int("port", 8080, "Port to listen on") + +func main() { + flag.Parse() + backend := expander.NewExpander() + service := expansion.NewService(*address, *port, backend) + log.Printf("Version: %s", version.Version) + log.Printf("Listening on http://%s:%d/expand", *address, *port) + log.Fatal(service.ListenAndServe()) +} diff --git a/glide.lock b/glide.lock index 18a6901fa..3351f2257 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,10 @@ -hash: 5a891985ff2f22aed4eaa4f7f3819f0866750c065c7acd0118415c34788bc665 -updated: 2016-03-30T15:52:37.011995847-06:00 +hash: a72f13699934fc94c8df5b56c76fc66d2279b2a4ff1396c2f1bff944456a8adf +updated: 2016-03-30T23:15:39.980396345-04:00 imports: - name: github.com/aokoli/goutils version: 45307ec16e3cd47cd841506c081f7afd8237d210 +- name: github.com/cloudfoundry-incubator/candiedyaml + version: 479485e9bfc69ee37d074b36ce36da5e4fba7941 - name: github.com/codegangsta/cli version: a2943485b110df8842045ae0600047f88a3a56a1 - name: github.com/emicklei/go-restful @@ -46,6 +48,10 @@ imports: - gensupport - googleapi - googleapi/internal/uritemplates +- name: google.golang.org/appengine + version: a503df954af258b9a70918df2a524d6a85ecefdb + subpackages: + - urlfetch - name: google.golang.org/cloud version: fb10e8da373d97f6ba5e648299a10b3b91f14cd5 subpackages: @@ -55,6 +61,7 @@ imports: version: d90005c5262a3463800497ea5a89aed5fe22c886 subpackages: - bson + - internal/sasl - internal/scram - name: gopkg.in/yaml.v2 version: f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 diff --git a/glide.yaml b/glide.yaml index 7c048362c..653dc557d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -7,6 +7,7 @@ import: version: ^1.1.0 - package: github.com/aokoli/goutils version: ^1.0.0 +- package: github.com/cloudfoundry-incubator/candiedyaml - package: github.com/codegangsta/cli - package: github.com/emicklei/go-restful - package: github.com/ghodss/yaml diff --git a/scripts/common.sh b/scripts/common.sh index e6f92c960..817613122 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -18,7 +18,7 @@ set -o pipefail readonly ROOTFS="${DIR}/rootfs" -readonly STATIC_TARGETS=(cmd/expandybird cmd/manager cmd/resourcifier) +readonly STATIC_TARGETS=(cmd/expandybird cmd/goexpander cmd/manager cmd/resourcifier) readonly ALL_TARGETS=(${STATIC_TARGETS[@]} cmd/helm) error_exit() {